From c063286410d6e4c1fa03b479b7961df755198f16 Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Thu, 17 Sep 2026 12:27:38 -0500 Subject: [PATCH 01/17] fix(protocol): deliver AUTO_INCREMENT insert ids and rule errors on every client path Step 1 of the narrow-id work: close the defects that would have made the allocator's behaviour unobservable or wrong before it exists. D1 Transpile swallowed every ApplyAST error (`if err == nil && applied`). Rule errors now propagate verbatim; transform.CodedError carries a MySQL code, UnsupportedStatementError maps coded -> own code, other rule errors -> 1105, parser failures -> 1064. No string matching. D2 NeedsIDInjection and ApplyAST decided independently and could disagree. One analyzeInsert drives both; column-less positional inserts substitute at the auto-increment ordinal (index into non-generated columns, not cid); INSERT ... SELECT and INSERT ... VALUES ROW(...) are refused with 1235 in MySQL's message template instead of letting SQLite assign a key. D3 The OK packet's insert id came from SQLite's connection-wide last-rowid register, shared by every client on the single hookDB connection. It now comes from the statement's own CDC entries by column name: first generated id, 0 when none. ExecuteLocalWithHooks takes one request. ConnectionSession.RecordInsertId is the single home of the rule that 0 never overwrites LAST_INSERT_ID(); the replica path stored zeros unconditionally until now. D4 Dead code: int_to_bigint rule (never registered), MergeCDCEntries, PendingLocalExecution, MustFromWireType, HasPrimaryKey, 37 orphaned regex vars and extractTableName in protocol/parser.go, transpile_test. parser_vitess.go renamed to vitess_parser_instance.go for its 21 lines. D5 An unknown wire StatementType panicked the receiving peer through MustFromWireType with no recover. protocolStatementFromProto returns an error and both replication call sites answer Success:false. Replica error path (found by review): ForwardQueryResponse now carries error_code and sql_state (fields 7, 8) so a rule's 1235/42000 reaches a replica client intact; the leader fills both from the protocol mapper, the replica raises the typed error. Code 0 keeps 1105/HY000 for an older leader. The old string-formatted "ERROR 1105 (HY000): ..." also re-classified any leader message containing "syntax error" as 1064. Replica-raised link failures report 1105/HY000 (master's observable behaviour); 2003/2013 are client-library codes and are never put in an ERR packet. Generated files regenerated with protoc 33.6 (protobuf@33) and protoc-gen-go v1.36.10; equivalence from the unchanged proto is header-only. Evidence: red-before-green on master-identical files for D3; 20 mutation arms fired at named assertions across three review rounds; unit suite 24 ok, race clean on replica/grpc/protocol/coordinator, integration 30/30, benchmarks within a measured 2-4% noise floor with identical allocs/op. Not in this change (pre-existing, recorded): three db data races on master (transaction records shared by pointer); ./db/ no-tag stub build broken on master; go vet findings db/vector_index_manager_test.go:315 and grpc/forward_session.go:356; cmd/vec-bench/main_test.go is gitignored; protocol/handlers/metadata.go:49 still formats an error code into a string. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- common/statement_code_map.go | 9 - common/statement_code_map_test.go | 32 +- coordinator/cdc_merge_test.go | 119 --- coordinator/handler.go | 60 +- db/database_manager.go | 23 +- db/database_manager_transpiler_schema_test.go | 62 ++ db/db_integration.go | 102 +- db/insert_or_replace_test.go | 32 +- db/last_insert_id_test.go | 350 +++++++ db/preupdate_hook.go | 44 +- db/preupdate_hook_stub.go | 4 +- db/statement_insert_id.go | 48 + grpc/common/marmot_common.pb.go | 2 +- grpc/forward_error_code_test.go | 81 ++ grpc/forward_handler.go | 47 +- grpc/marmot.pb.go | 38 +- grpc/marmot.proto | 11 + grpc/marmot_grpc.pb.go | 2 +- grpc/replication_handler.go | 23 +- grpc/vector_statement.go | 49 +- grpc/vector_statement_test.go | 3 +- grpc/vector_statement_wiretype_test.go | 197 ++++ protocol/error_mapper.go | 56 ++ protocol/mysql_errors.go | 2 + protocol/parser.go | 79 +- protocol/parser_vitess.go | 887 ------------------ protocol/parser_vitess_test.go | 84 -- protocol/query/context.go | 9 +- protocol/query/pipeline.go | 3 + protocol/query/pipeline_autoincrement_test.go | 40 +- protocol/query/rules/autoincrement_id.go | 215 +++-- protocol/query/rules/autoincrement_id_test.go | 304 ++++++ protocol/query/rules/int_to_bigint.go | 63 -- protocol/query/rules/int_to_bigint_test.go | 143 --- protocol/query/transform/errors.go | 37 + protocol/query/transform/rule.go | 11 +- protocol/query/transpiler.go | 8 +- protocol/query/transpiler_rule_error_test.go | 95 ++ protocol/server.go | 27 +- protocol/transaction.go | 6 + protocol/transpile_test.go | 142 --- protocol/unsupported_statement_error_test.go | 92 ++ protocol/vitess_parser_instance.go | 21 + replica/handler.go | 83 +- replica/handler_test.go | 384 ++++++++ 45 files changed, 2253 insertions(+), 1876 deletions(-) delete mode 100644 coordinator/cdc_merge_test.go create mode 100644 db/last_insert_id_test.go create mode 100644 db/statement_insert_id.go create mode 100644 grpc/forward_error_code_test.go create mode 100644 grpc/vector_statement_wiretype_test.go delete mode 100644 protocol/parser_vitess.go create mode 100644 protocol/query/rules/autoincrement_id_test.go delete mode 100644 protocol/query/rules/int_to_bigint.go delete mode 100644 protocol/query/rules/int_to_bigint_test.go create mode 100644 protocol/query/transform/errors.go create mode 100644 protocol/query/transpiler_rule_error_test.go delete mode 100644 protocol/transpile_test.go create mode 100644 protocol/unsupported_statement_error_test.go create mode 100644 protocol/vitess_parser_instance.go diff --git a/common/statement_code_map.go b/common/statement_code_map.go index 0f96bcd0..57dbf520 100644 --- a/common/statement_code_map.go +++ b/common/statement_code_map.go @@ -63,12 +63,3 @@ func MustToWireType(code StatementCode) pb.StatementType { } panic(fmt.Sprintf("unknown StatementCode %d: cannot convert to wire type", code)) } - -// MustFromWireType converts wire StatementType to internal StatementCode. -// Panics on unknown type - use at gRPC receive boundary where invalid data is unacceptable. -func MustFromWireType(st pb.StatementType) StatementCode { - if code, ok := wireToCode[st]; ok { - return code - } - panic(fmt.Sprintf("unknown wire StatementType %d: cannot convert to internal code", st)) -} diff --git a/common/statement_code_map_test.go b/common/statement_code_map_test.go index 3ea39c63..3dd8630a 100644 --- a/common/statement_code_map_test.go +++ b/common/statement_code_map_test.go @@ -86,6 +86,7 @@ func TestFromWireType(t *testing.T) { {"DROP_DATABASE", pb.StatementType_DROP_DATABASE, StatementDropDatabase, true}, {"LOAD_DATA", pb.StatementType_LOAD_DATA, StatementLoadData, true}, {"VECTOR_INDEX", pb.StatementType_VECTOR_INDEX, StatementVectorIndexControl, true}, + {"UNKNOWN_WIRE_TYPE", pb.StatementType(999), StatementCode(0), false}, } for _, tt := range tests { @@ -105,12 +106,6 @@ func TestMustToWireTypePanicsOnUnknown(t *testing.T) { }, "MustToWireType should panic on code without wire representation") } -func TestMustFromWireTypePanicsOnUnknown(t *testing.T) { - assert.Panics(t, func() { - MustFromWireType(pb.StatementType(999)) - }, "MustFromWireType should panic on unknown wire type") -} - func TestMustToWireType(t *testing.T) { tests := []struct { name string @@ -135,28 +130,3 @@ func TestMustToWireType(t *testing.T) { }) } } - -func TestMustFromWireType(t *testing.T) { - tests := []struct { - name string - wire pb.StatementType - want StatementCode - }{ - {"INSERT", pb.StatementType_INSERT, StatementInsert}, - {"UPDATE", pb.StatementType_UPDATE, StatementUpdate}, - {"DELETE", pb.StatementType_DELETE, StatementDelete}, - {"REPLACE", pb.StatementType_REPLACE, StatementReplace}, - {"DDL", pb.StatementType_DDL, StatementDDL}, - {"CREATE_DATABASE", pb.StatementType_CREATE_DATABASE, StatementCreateDatabase}, - {"DROP_DATABASE", pb.StatementType_DROP_DATABASE, StatementDropDatabase}, - {"LOAD_DATA", pb.StatementType_LOAD_DATA, StatementLoadData}, - {"VECTOR_INDEX", pb.StatementType_VECTOR_INDEX, StatementVectorIndexControl}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := MustFromWireType(tt.wire) - assert.Equal(t, tt.want, got) - }) - } -} diff --git a/coordinator/cdc_merge_test.go b/coordinator/cdc_merge_test.go deleted file mode 100644 index 305931eb..00000000 --- a/coordinator/cdc_merge_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package coordinator - -import ( - "testing" - - "github.com/maxpert/marmot/common" -) - -// TestMergeCDCEntries_TableNameRequired is a CRITICAL test. -// It exists because commit 28cfab9 fixed handleCommit but forgot handleMutation. -// This test MUST fail if anyone removes TableName extraction from MergeCDCEntries. -func TestMergeCDCEntries_TableNameRequired(t *testing.T) { - entries := []common.CDCEntry{ - {Table: "users", IntentKey: []byte("1"), NewValues: map[string][]byte{"id": {1}}}, - } - result := MergeCDCEntries(entries) - - if result.TableName == "" { - t.Fatal("CRITICAL: TableName must be extracted from CDC entries. " + - "Without it, write_coordinator rejects CDC statements. " + - "See MergeCDCEntries() for details.") - } - if result.TableName != "users" { - t.Errorf("TableName = %q, want %q", result.TableName, "users") - } -} - -func TestMergeCDCEntries_MergesUpsertHooks(t *testing.T) { - // SQLite fires DELETE then INSERT for UPSERT on existing row - entries := []common.CDCEntry{ - {Table: "users", IntentKey: []byte("1"), OldValues: map[string][]byte{"name": []byte("old")}, NewValues: nil}, - {Table: "users", IntentKey: []byte("1"), OldValues: nil, NewValues: map[string][]byte{"name": []byte("new")}}, - } - result := MergeCDCEntries(entries) - - if result.TableName != "users" { - t.Errorf("TableName = %q, want %q", result.TableName, "users") - } - if string(result.IntentKey) != "1" { - t.Errorf("IntentKey = %q, want %q", result.IntentKey, "1") - } - if string(result.OldValues["name"]) != "old" { - t.Errorf("OldValues[name] = %q, want %q", result.OldValues["name"], "old") - } - if string(result.NewValues["name"]) != "new" { - t.Errorf("NewValues[name] = %q, want %q", result.NewValues["name"], "new") - } -} - -func TestMergeCDCEntries_EmptyEntries(t *testing.T) { - result := MergeCDCEntries(nil) - - if result.TableName != "" { - t.Errorf("TableName = %q, want empty", result.TableName) - } - if len(result.IntentKey) != 0 { - t.Errorf("IntentKey = %q, want empty", result.IntentKey) - } - if result.OldValues == nil { - t.Error("OldValues should be initialized to empty map, got nil") - } - if result.NewValues == nil { - t.Error("NewValues should be initialized to empty map, got nil") - } -} - -func TestMergeCDCEntries_SingleInsert(t *testing.T) { - entries := []common.CDCEntry{ - { - Table: "products", - IntentKey: []byte("42"), - OldValues: nil, - NewValues: map[string][]byte{ - "id": []byte("42"), - "name": []byte("Widget"), - "price": []byte("9.99"), - }, - }, - } - result := MergeCDCEntries(entries) - - if result.TableName != "products" { - t.Errorf("TableName = %q, want %q", result.TableName, "products") - } - if string(result.IntentKey) != "42" { - t.Errorf("IntentKey = %q, want %q", result.IntentKey, "42") - } - if len(result.OldValues) != 0 { - t.Errorf("OldValues should be empty for INSERT, got %d entries", len(result.OldValues)) - } - if len(result.NewValues) != 3 { - t.Errorf("NewValues should have 3 entries, got %d", len(result.NewValues)) - } -} - -func TestMergeCDCEntries_SingleDelete(t *testing.T) { - entries := []common.CDCEntry{ - { - Table: "products", - IntentKey: []byte("42"), - OldValues: map[string][]byte{ - "id": []byte("42"), - "name": []byte("Widget"), - }, - NewValues: nil, - }, - } - result := MergeCDCEntries(entries) - - if result.TableName != "products" { - t.Errorf("TableName = %q, want %q", result.TableName, "products") - } - if len(result.NewValues) != 0 { - t.Errorf("NewValues should be empty for DELETE, got %d entries", len(result.NewValues)) - } - if len(result.OldValues) != 2 { - t.Errorf("OldValues should have 2 entries, got %d", len(result.OldValues)) - } -} diff --git a/coordinator/handler.go b/coordinator/handler.go index 3ecf478e..03ccb5dd 100644 --- a/coordinator/handler.go +++ b/coordinator/handler.go @@ -62,7 +62,10 @@ type DatabaseManager interface { // ReplicatedDatabaseProvider provides access to replicated database operations type ReplicatedDatabaseProvider interface { - ExecuteLocalWithHooks(ctx context.Context, txnID uint64, requests []ExecutionRequest) (PendingExecution, error) + // ExecuteLocalWithHooks executes ONE autocommit statement with CDC capture. + // It takes a single request, not a slice: the insert id it reports is the + // statement's own, and a batch would silently report the first statement's. + ExecuteLocalWithHooks(ctx context.Context, txnID uint64, req ExecutionRequest) (PendingExecution, error) GetSchemaCache() interface{} // Returns *SchemaCache (using interface{} to avoid import cycle) // DescribeResultColumns reports the columns a query returns without running it. DescribeResultColumns(ctx context.Context, query string) ([]common.ResultColumn, error) @@ -80,49 +83,12 @@ type ExecutionRequest struct { Params []interface{} } -// CDCMergeResult holds merged CDC data from preupdate hooks. -type CDCMergeResult struct { - TableName string - IntentKey []byte - OldValues map[string][]byte - NewValues map[string][]byte -} - -// MergeCDCEntries merges CDC entries captured by preupdate hooks. -// For UPSERT (INSERT OR REPLACE), SQLite fires DELETE then INSERT hooks. -// This function combines them to get complete old/new values. -// -// CRITICAL CONTRACT: -// - TableName is ALWAYS extracted from entries (hooks capture it) -// - Never rely on parsed SQL for TableName with CDC data -// - See TestMergeCDCEntries_TableNameRequired for enforcement -func MergeCDCEntries(entries []common.CDCEntry) CDCMergeResult { - result := CDCMergeResult{ - OldValues: make(map[string][]byte), - NewValues: make(map[string][]byte), - } - - for _, e := range entries { - if len(result.IntentKey) == 0 { - result.IntentKey = e.IntentKey - } - if result.TableName == "" { - result.TableName = e.Table - } - for k, v := range e.OldValues { - result.OldValues[k] = v - } - for k, v := range e.NewValues { - result.NewValues[k] = v - } - } - - return result -} - // PendingExecution represents a locally executed transaction waiting for quorum type PendingExecution interface { GetTotalRowCount() int64 + // GetLastInsertId returns the OK packet's insert id: the FIRST + // AUTO_INCREMENT value the statement generated, 0 when it generated none. + // The name mirrors MySQL's OK-packet field, not SQLite's last-rowid. GetLastInsertId() int64 GetCDCEntries() []common.CDCEntry Commit() error @@ -330,12 +296,12 @@ func (h *CoordinatorHandler) HandleQuery(session *protocol.ConnectionSession, sq var schemaLookup protocol.SchemaLookupFunc if h.dbManager != nil && session.CurrentDatabase != "" { dbName := session.CurrentDatabase - schemaLookup = func(table string) string { - col, err := h.dbManager.GetAutoIncrementColumn(dbName, table) + schemaLookup = func(table string) *transform.SchemaInfo { + info, err := h.dbManager.GetTranspilerSchema(dbName, table) if err != nil { - return "" + return nil } - return col + return info } } schemaProvider := func(database, table string) *transform.SchemaInfo { @@ -390,7 +356,7 @@ func (h *CoordinatorHandler) HandleQuery(session *protocol.ConnectionSession, sq // statement Vitess only partially parsed) instead of forwarding the mangled // SQL into the read/2PC path, where it produces a confusing downstream error. if stmt.Type == protocol.StatementUnsupported && stmt.Error != "" { - return nil, protocol.NewMySQLError(protocol.ErrCodeParseError, protocol.SQLStateSyntax, stmt.Error) + return nil, protocol.UnsupportedStatementError(stmt) } // Handle SET commands: extract @@marmot_vec_* vars via Vitess AST; ignore others. @@ -663,7 +629,7 @@ func (h *CoordinatorHandler) handleMutation(stmt protocol.Statement, params []in // serialization order (see protocol.Statement.MergeExecParams). execParams := stmt.MergeExecParams(params) req := ExecutionRequest{SQL: stmt.SQL, Params: execParams} - pendingExec, err = replicatedDB.ExecuteLocalWithHooks(ctx, uint64(txnID), []ExecutionRequest{req}) + pendingExec, err = replicatedDB.ExecuteLocalWithHooks(ctx, uint64(txnID), req) if err != nil { cancel() // Only cancel on error diff --git a/db/database_manager.go b/db/database_manager.go index 9a0706b4..4e548fc3 100644 --- a/db/database_manager.go +++ b/db/database_manager.go @@ -1374,6 +1374,25 @@ func (dm *DatabaseManager) GetAutoIncrementColumn(database, table string) (strin return schema.GetAutoIncrementCol(), nil } +// columnOrdinal returns name's position in a table's column order, or -1 when +// name is empty or absent. TableSchema.Columns is declaration order with +// GENERATED columns removed, which is exactly the tuple a column-less +// "INSERT INTO t VALUES (...)" supplies - SQLite rejects a value for a +// generated column - so this index is the position of that column's value in +// such a statement. PRAGMA table_xinfo's cid is NOT usable here: it counts +// hidden columns too (see db/schema_cache.go loadSchema). +func columnOrdinal(columns []string, name string) int { + if name == "" { + return -1 + } + for i, col := range columns { + if strings.EqualFold(col, name) { + return i + } + } + return -1 +} + // GetTranspilerSchema returns schema information used by SQL transpilation rules. // Uses cached schema - does NOT query SQLite PRAGMA. func (dm *DatabaseManager) GetTranspilerSchema(database, table string) (*transform.SchemaInfo, error) { @@ -1387,8 +1406,10 @@ func (dm *DatabaseManager) GetTranspilerSchema(database, table string) (*transfo return nil, fmt.Errorf("schema not cached for table %s: %w", table, err) } + autoIncCol := schema.GetAutoIncrementCol() info := &transform.SchemaInfo{ - AutoIncrementColumn: schema.GetAutoIncrementCol(), + AutoIncrementColumn: autoIncCol, + AutoIncrementOrdinal: columnOrdinal(schema.Columns, autoIncCol), } // PrimaryKeys uses "rowid" sentinel when no explicit PRIMARY KEY is defined. diff --git a/db/database_manager_transpiler_schema_test.go b/db/database_manager_transpiler_schema_test.go index 19cb07ef..3482a09c 100644 --- a/db/database_manager_transpiler_schema_test.go +++ b/db/database_manager_transpiler_schema_test.go @@ -72,3 +72,65 @@ func TestGetTranspilerSchema_IgnoresRowIDSentinel(t *testing.T) { t.Fatalf("expected empty primary key, got %#v", info.PrimaryKey) } } + +// TestGetTranspilerSchema_AutoIncrementOrdinal pins the ordinal a column-less +// "INSERT INTO t VALUES (...)" indexes its VALUES tuple by. TableSchema.Columns +// is declaration order with GENERATED columns removed, which is exactly the +// tuple SQLite expects, so the ordinal must be the index into Columns and NOT +// PRAGMA table_xinfo's cid (ColumnPositions), which counts hidden columns too. +func TestGetTranspilerSchema_AutoIncrementOrdinal(t *testing.T) { + dm, _ := setupTestDatabaseManager(t) + defer dm.Close() + + if err := dm.CreateDatabase("testdb"); err != nil { + t.Fatalf("CreateDatabase failed: %v", err) + } + + mdb, err := dm.GetDatabase("testdb") + if err != nil { + t.Fatalf("GetDatabase failed: %v", err) + } + + cache, ok := mdb.GetSchemaCache().(*SchemaCache) + if !ok || cache == nil { + t.Fatal("schema cache is unavailable") + } + + // A table whose auto-increment column is neither first nor at its cid: + // "gen" is a generated column, so it is absent from Columns while it still + // occupies a cid, which is what makes ColumnPositions the wrong answer. + cache.Update("events", &TableSchema{ + Columns: []string{"tenant", "name", "event_id"}, + ColumnPositions: []int{0, 2, 3}, + PrimaryKeys: []string{"event_id"}, + AutoIncrementCol: "event_id", + }) + cache.Update("nokey", &TableSchema{ + Columns: []string{"a", "b"}, + PrimaryKeys: []string{"rowid"}, + }) + + // Mutation: return columnOrdinal(schema.ColumnPositions...) or a constant + // 0; this fires with 3 or 0 respectively. + info, err := dm.GetTranspilerSchema("testdb", "events") + if err != nil { + t.Fatalf("GetTranspilerSchema failed: %v", err) + } + if info.AutoIncrementOrdinal != 2 { + t.Fatalf("AutoIncrementOrdinal = %d, want 2", info.AutoIncrementOrdinal) + } + + // Mutation: drop the `name == ""` guard in columnOrdinal and let the loop + // return 0 for an absent column; a table with no auto-increment column + // reports ordinal 0 and the rule would inject into its first column. + info, err = dm.GetTranspilerSchema("testdb", "nokey") + if err != nil { + t.Fatalf("GetTranspilerSchema failed: %v", err) + } + if info.AutoIncrementColumn != "" { + t.Fatalf("AutoIncrementColumn = %q, want empty", info.AutoIncrementColumn) + } + if info.AutoIncrementOrdinal != -1 { + t.Fatalf("AutoIncrementOrdinal = %d, want -1", info.AutoIncrementOrdinal) + } +} diff --git a/db/db_integration.go b/db/db_integration.go index ee966f01..2cbe5916 100644 --- a/db/db_integration.go +++ b/db/db_integration.go @@ -15,9 +15,6 @@ import ( "github.com/rs/zerolog/log" ) -// Ensure PendingLocalExecution implements coordinator.PendingExecution -var _ coordinator.PendingExecution = (*PendingLocalExecution)(nil) - // ReplicatedDatabase wraps a SQL database with distributed transaction support // This is the main integration point between application layer and transactional storage // @@ -497,12 +494,6 @@ func (mdb *ReplicatedDatabase) ApplyCDCEntries(entries []*IntentEntry) error { return mdb.txnMgr.applyCDCEntries(0, mdb.clock.Now(), entries) } -// PendingLocalExecution represents a locally executed transaction waiting for quorum -// The SQLite transaction is held open until Commit or Rollback is called -type PendingLocalExecution struct { - session *EphemeralHookSession // Ephemeral session (owns its connection) -} - // CompletedLocalExecution represents a CDC capture that's already been rolled back. // Used by the new hookDB flow where we capture CDC then release the connection // BEFORE 2PC broadcast. Commit/Rollback are no-ops. @@ -562,73 +553,15 @@ func (c *CompletedLocalExecution) GetCDCEntries() []common.CDCEntry { return result } -// GetLastInsertId returns the last insert ID from the most recent insert +// GetLastInsertId returns the OK packet's insert id for this execution: the +// first AUTO_INCREMENT value the statement generated, or 0 when it generated +// none. The name mirrors MySQL's own OK-packet field and the coordinator +// interface; the value is deliberately the FIRST id, not the last, which is +// what SQLite's connection-wide last-rowid register would have given. func (c *CompletedLocalExecution) GetLastInsertId() int64 { return c.lastInsertId } -// GetTotalRowCount returns count from CDC entries. -func (p *PendingLocalExecution) GetTotalRowCount() int64 { - entries := p.GetCDCEntries() - return int64(len(entries)) -} - -// Commit finalizes the local transaction -func (p *PendingLocalExecution) Commit() error { - if p.session != nil { - return p.session.Commit() - } - return nil -} - -// Rollback aborts the local transaction -func (p *PendingLocalExecution) Rollback() error { - if p.session != nil { - return p.session.Rollback() - } - return nil -} - -// GetIntentEntries returns CDC entries from the system database -func (p *PendingLocalExecution) GetIntentEntries() ([]*IntentEntry, error) { - if p.session == nil { - return nil, nil - } - return p.session.GetIntentEntries() -} - -// GetCDCEntries returns CDC data captured by hooks for replication -func (p *PendingLocalExecution) GetCDCEntries() []common.CDCEntry { - if p.session == nil { - return nil - } - entries, err := p.session.GetIntentEntries() - if err != nil || len(entries) == 0 { - return nil - } - result := make([]common.CDCEntry, len(entries)) - for i, e := range entries { - result[i] = common.CDCEntry{ - Table: e.Table, - IntentKey: e.IntentKey, - Operation: e.Operation, - OldValues: e.OldValues, - NewValues: e.NewValues, - EncodedRow: e.EncodedRow, - EncodedCodec: e.EncodedCodec, - } - } - return result -} - -// GetLastInsertId returns the last insert ID from the most recent insert -func (p *PendingLocalExecution) GetLastInsertId() int64 { - if p.session == nil { - return 0 - } - return p.session.GetLastInsertId() -} - // ExecuteLocalWithHooks executes SQL locally with preupdate hooks capturing CDC data. // Returns a PendingExecution with captured CDC entries. The hookDB transaction is // ALREADY ROLLED BACK - no Commit/Rollback needed from caller. @@ -643,7 +576,7 @@ func (p *PendingLocalExecution) GetLastInsertId() int64 { // // This design avoids deadlock: hookDB is released before 2PC broadcast, // so incoming COMMIT from other coordinators can acquire writeDB. -func (mdb *ReplicatedDatabase) ExecuteLocalWithHooks(ctx context.Context, txnID uint64, requests []coordinator.ExecutionRequest) (coordinator.PendingExecution, error) { +func (mdb *ReplicatedDatabase) ExecuteLocalWithHooks(ctx context.Context, txnID uint64, req coordinator.ExecutionRequest) (coordinator.PendingExecution, error) { // Create ephemeral session with hookDB (NOT writeDB - avoids deadlock) // SchemaCache must be pre-populated via ReloadSchema() before calling this session, err := StartEphemeralSession(ctx, mdb.hookDB, mdb.metaStore, mdb.schemaCache, txnID) @@ -658,7 +591,7 @@ func (mdb *ReplicatedDatabase) ExecuteLocalWithHooks(ctx context.Context, txnID cdcEntries: nil, lastInsertId: 0, db: mdb, - rowCount: int64(len(requests)), + rowCount: 1, }, nil } return nil, fmt.Errorf("failed to start session: %w", err) @@ -670,19 +603,14 @@ func (mdb *ReplicatedDatabase) ExecuteLocalWithHooks(ctx context.Context, txnID return nil, fmt.Errorf("failed to begin transaction: %w", err) } - // Execute each statement - hooks capture raw CDC data to Pebble. + // Execute the statement - hooks capture raw CDC data to Pebble. // rows-affected is not used here: this autocommit path computes it from // the captured CDC entries via CompletedLocalExecution.GetTotalRowCount. - for _, req := range requests { - if _, err := session.ExecContext(ctx, req.SQL, req.Params...); err != nil { - _ = session.Rollback() - return nil, fmt.Errorf("failed to execute statement: %w", err) - } + if _, err := session.ExecContext(ctx, req.SQL, req.Params...); err != nil { + _ = session.Rollback() + return nil, fmt.Errorf("failed to execute statement: %w", err) } - // Get last insert ID BEFORE rollback (available immediately) - lastInsertId := session.GetLastInsertId() - // ROLLBACK hookDB - this also calls ProcessCapturedRows which converts // raw captured data to IntentEntries if err := session.Rollback(); err != nil { @@ -699,6 +627,12 @@ func (mdb *ReplicatedDatabase) ExecuteLocalWithHooks(ctx context.Context, txnID } session.cleanup() + // The insert id comes from this statement's own CDC entries, not from + // SQLite's connection-wide last-rowid register: hookDB is capped at one + // connection, so that register is shared by every client in turn. The + // signature takes one request, so cdcEntries cannot span two statements. + lastInsertId := statementInsertID(mdb.schemaCache, cdcEntries) + // Return completed execution with captured CDC data return &CompletedLocalExecution{ cdcEntries: cdcEntries, @@ -731,7 +665,7 @@ func (p *pinnedHookSession) ExecuteStatement(ctx context.Context, sql string, pa if err := p.session.captureAndLockNewRows(); err != nil { return 0, 0, err } - return rowsAffected, p.session.GetLastInsertId(), nil + return rowsAffected, p.session.StatementInsertID(), nil } func (p *pinnedHookSession) Query(ctx context.Context, sqlText string, params []interface{}) ([]string, []map[string]interface{}, error) { diff --git a/db/insert_or_replace_test.go b/db/insert_or_replace_test.go index b5ff143f..17e6aadf 100644 --- a/db/insert_or_replace_test.go +++ b/db/insert_or_replace_test.go @@ -66,7 +66,7 @@ func TestInsertOrReplaceWithHooks(t *testing.T) { SQL: "INSERT INTO test_upsert (id, value) VALUES ('key1', 'value1')", } - pending1, err := replicatedDB.ExecuteLocalWithHooks(ctx, 1001, []coordinator.ExecutionRequest{req1}) + pending1, err := replicatedDB.ExecuteLocalWithHooks(ctx, 1001, req1) if err != nil { t.Fatalf("Failed to execute INSERT: %v", err) } @@ -77,6 +77,12 @@ func TestInsertOrReplaceWithHooks(t *testing.T) { for _, e := range entries1 { t.Logf(" Entry: intentKey=%s oldVals=%d newVals=%d", e.IntentKey, len(e.OldValues), len(e.NewValues)) } + // A plain INSERT of a fresh key touches exactly one row, so it must emit + // exactly one CDC entry. Reddens if the hook double-captures the row or + // captures a spurious implicit delete for a key that never existed. + if len(entries1) != 1 { + t.Fatalf("plain INSERT of a fresh key must emit exactly 1 CDC entry, got %d", len(entries1)) + } if err := pending1.Commit(); err != nil { t.Fatalf("Failed to commit INSERT: %v", err) @@ -94,7 +100,7 @@ func TestInsertOrReplaceWithHooks(t *testing.T) { t.Log("About to call ExecuteLocalWithHooks for INSERT OR REPLACE...") start := time.Now() - pending2, err := replicatedDB.ExecuteLocalWithHooks(ctx2, 1002, []coordinator.ExecutionRequest{req2}) + pending2, err := replicatedDB.ExecuteLocalWithHooks(ctx2, 1002, req2) elapsed := time.Since(start) t.Logf("ExecuteLocalWithHooks returned after %v", elapsed) @@ -108,6 +114,15 @@ func TestInsertOrReplaceWithHooks(t *testing.T) { for _, e := range entries2 { t.Logf(" Entry: intentKey=%s oldVals=%d newVals=%d", e.IntentKey, len(e.OldValues), len(e.NewValues)) } + // INSERT OR REPLACE on an existing key is SQLite's implicit + // delete-then-insert: it must emit exactly 2 CDC entries (the delete of + // the old row plus the insert of the new one). This is what makes + // MySQL's affected_rows=2 for a REPLACE that actually replaced a row + // correct rather than accidental. Reddens if the hook collapses the pair + // into a single UPDATE-shaped entry, or captures only the insert half. + if len(entries2) != 2 { + t.Fatalf("INSERT OR REPLACE that replaces an existing row must emit exactly 2 CDC entries (implicit delete + insert), got %d", len(entries2)) + } if err := pending2.Commit(); err != nil { t.Fatalf("Failed to commit INSERT OR REPLACE: %v", err) @@ -172,7 +187,7 @@ func TestInsertOrReplaceNewRow(t *testing.T) { t.Log("About to call ExecuteLocalWithHooks...") start := time.Now() - pending, err := replicatedDB.ExecuteLocalWithHooks(ctx, 2001, []coordinator.ExecutionRequest{req}) + pending, err := replicatedDB.ExecuteLocalWithHooks(ctx, 2001, req) elapsed := time.Since(start) t.Logf("ExecuteLocalWithHooks returned after %v", elapsed) @@ -237,7 +252,7 @@ func TestLastInsertIdCapture(t *testing.T) { SQL: "INSERT INTO test_autoincrement (value) VALUES ('first')", } - pending1, err := replicatedDB.ExecuteLocalWithHooks(ctx1, 3001, []coordinator.ExecutionRequest{req1}) + pending1, err := replicatedDB.ExecuteLocalWithHooks(ctx1, 3001, req1) if err != nil { t.Fatalf("Failed to execute INSERT: %v", err) } @@ -261,7 +276,7 @@ func TestLastInsertIdCapture(t *testing.T) { SQL: "INSERT INTO test_autoincrement (value) VALUES ('second')", } - pending2, err := replicatedDB.ExecuteLocalWithHooks(ctx2, 3002, []coordinator.ExecutionRequest{req2}) + pending2, err := replicatedDB.ExecuteLocalWithHooks(ctx2, 3002, req2) if err != nil { t.Fatalf("Failed to execute second INSERT: %v", err) } @@ -285,15 +300,18 @@ func TestLastInsertIdCapture(t *testing.T) { SQL: "UPDATE test_autoincrement SET value = 'updated' WHERE id = 1", } - pending3, err := replicatedDB.ExecuteLocalWithHooks(ctx3, 3003, []coordinator.ExecutionRequest{req3}) + pending3, err := replicatedDB.ExecuteLocalWithHooks(ctx3, 3003, req3) if err != nil { t.Fatalf("Failed to execute UPDATE: %v", err) } lastInsertId3 := pending3.GetLastInsertId() t.Logf("UPDATE lastInsertId: %d", lastInsertId3) + // Reddens if EphemeralHookSession.ExecContext keeps reporting a stale + // sqlite3_last_insert_rowid() value from the earlier INSERT instead of 0 + // for a statement that only updated rows. if lastInsertId3 != 0 { - t.Logf("UPDATE returned lastInsertId=%d (expected 0, but may vary)", lastInsertId3) + t.Errorf("UPDATE must report lastInsertId=0, got %d", lastInsertId3) } if err := pending3.Commit(); err != nil { diff --git a/db/last_insert_id_test.go b/db/last_insert_id_test.go new file mode 100644 index 00000000..d30cf039 --- /dev/null +++ b/db/last_insert_id_test.go @@ -0,0 +1,350 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "context" + "fmt" + "testing" + + "github.com/maxpert/marmot/coordinator" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// This file pins the intended LAST_INSERT_ID / OK-packet insert-id behavior: +// the value of the table's AUTO-INCREMENT COLUMN in the FIRST row THIS +// STATEMENT inserted, read by column name from that statement's first insert +// CDC entry. It is 0 when the statement inserted no rows, and 0 when the +// target table has no auto-increment column. +// +// Setup reuses the shared helpers from rowid_sentinel_cdc_test.go: +// newRowidTestDatabase (temp dir + Pebble MetaStore + ReplicatedDatabase, +// with cleanup registered) and execAndReload (DDL exec + schema reload). + +// extractSortedInt64Column pulls an INTEGER column out of Query()'s +// []map[string]interface{} rows and returns it sorted ascending. Fails loudly +// (rather than silently returning a wrong/empty slice) if a row is missing +// the column or the value isn't an int64, since that means the test's own +// fixture is broken, not the code under test. +func extractSortedInt64Column(t *testing.T, rows []map[string]interface{}, col string) []int64 { + t.Helper() + ids := make([]int64, 0, len(rows)) + for i, row := range rows { + v, ok := row[col] + if !ok { + t.Fatalf("row %d missing column %q: %#v", i, col, row) + } + id, ok := v.(int64) + if !ok { + t.Fatalf("row %d column %q is %T (%v), want int64", i, col, v, v) + } + ids = append(ids, id) + } + for i := 1; i < len(ids); i++ { + if ids[i] < ids[i-1] { + t.Fatalf("extractSortedInt64Column: caller's query must ORDER BY %s, got unsorted %v", col, ids) + } + } + return ids +} + +// TestAutocommitMultiRowInsertReportsFirstID is OBSERVABLE 1: a single +// autocommit statement that inserts 3 rows must report the FIRST generated +// id, not SQLite's sqlite3_last_insert_rowid() (which is the LAST row). +func TestAutocommitMultiRowInsertReportsFirstID(t *testing.T) { + source := newRowidTestDatabase(t, 1) + require.NoError(t, execAndReload(source, `CREATE TABLE t1 (id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT)`)) + + ctx := context.Background() + req := coordinator.ExecutionRequest{SQL: "INSERT INTO t1(v) VALUES('a'),('b'),('c')"} + pending, err := source.ExecuteLocalWithHooks(ctx, 60001, req) + require.NoError(t, err) + + // Reddens if ExecContext keeps storing whatever + // result.LastInsertId()/sqlite3_last_insert_rowid() returns (the LAST row + // of the 3-row insert, id=3) instead of the FIRST id this statement + // generated. + assert.Equal(t, int64(1), pending.GetLastInsertId(), "3-row autocommit INSERT must report the FIRST id (1), got %d", pending.GetLastInsertId()) + + require.NoError(t, pending.Commit()) + + // Sanity check: confirm the "first" assertion above didn't pass by + // accident (e.g. because only one row actually got inserted). If this + // fails, the test's own fixture is wrong, not the fix. + rows, err := source.GetWriteDB().Query("SELECT id FROM t1 ORDER BY id") + require.NoError(t, err) + defer rows.Close() + var ids []int64 + for rows.Next() { + var id int64 + require.NoError(t, rows.Scan(&id)) + ids = append(ids, id) + } + require.NoError(t, rows.Err()) + require.Equal(t, []int64{1, 2, 3}, ids, "sanity: the 3-row insert must have actually produced ids 1,2,3") +} + +// TestPinnedSessionMultiRowInsertReportsFirstID is OBSERVABLE 2: the same +// 3-row INSERT run through the explicit-transaction path (BeginPinnedSession +// / ExecuteStatement) must also report the FIRST generated id. +func TestPinnedSessionMultiRowInsertReportsFirstID(t *testing.T) { + source := newRowidTestDatabase(t, 1) + require.NoError(t, execAndReload(source, `CREATE TABLE t1 (id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT)`)) + + ctx := context.Background() + session, err := source.BeginPinnedSession(ctx, 61001) + require.NoError(t, err) + defer session.Release() + + rowsAffected, lastInsertId, err := session.ExecuteStatement(ctx, "INSERT INTO t1(v) VALUES('a'),('b'),('c')", nil) + require.NoError(t, err) + require.Equal(t, int64(3), rowsAffected) + + // Reddens if pinnedHookSession.ExecuteStatement returns the raw shared + // register (session.GetLastInsertId() sourced from + // sqlite3_last_insert_rowid(), the LAST row id=3) instead of the FIRST id. + assert.Equal(t, int64(1), lastInsertId, "3-row pinned-session INSERT must report the FIRST id (1), got %d", lastInsertId) + + // Sanity check via the session's own uncommitted read (Release never + // commits, so this must be read before Release). + cols, rows, err := session.Query(ctx, "SELECT id FROM t1 ORDER BY id", nil) + require.NoError(t, err) + require.Contains(t, cols, "id") + ids := extractSortedInt64Column(t, rows, "id") + require.Equal(t, []int64{1, 2, 3}, ids, "sanity: the 3-row insert must have actually produced ids 1,2,3") +} + +// TestPinnedSessionPerStatementFirstIDBoundary is OBSERVABLE 3: within one +// pinned session, each statement must report ITS OWN first id - not another +// statement's table, and not "whichever table was touched last, forever". +func TestPinnedSessionPerStatementFirstIDBoundary(t *testing.T) { + source := newRowidTestDatabase(t, 1) + require.NoError(t, execAndReload(source, `CREATE TABLE tblA (id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT)`)) + require.NoError(t, execAndReload(source, `CREATE TABLE tblB (id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT)`)) + + // Pre-seed tblB so its generated ids start far away from tblA's (which + // will start at 1), so a wrong answer (e.g. "reports tblA's range") + // cannot coincide with the right one by chance. + _, err := source.GetWriteDB().Exec(`INSERT INTO tblB (id, v) VALUES (999, 'seed')`) + require.NoError(t, err) + + ctx := context.Background() + session, err := source.BeginPinnedSession(ctx, 62001) + require.NoError(t, err) + defer session.Release() + + // Statement 1: 3 rows into tblA. + rowsA1, firstA1, err := session.ExecuteStatement(ctx, "INSERT INTO tblA(v) VALUES('a1'),('a2'),('a3')", nil) + require.NoError(t, err) + require.Equal(t, int64(3), rowsA1) + _, rowsDataA1, err := session.Query(ctx, "SELECT id FROM tblA ORDER BY id", nil) + require.NoError(t, err) + idsA1 := extractSortedInt64Column(t, rowsDataA1, "id") + require.Equal(t, []int64{1, 2, 3}, idsA1, "sanity: tblA's fresh rows must be 1,2,3, or this test's own fixture is wrong") + // Reddens if statement 1 reports the LAST row of its own insert (3) + // instead of the FIRST (1) - same defect as observable 1, checked again + // here because it is the baseline the rest of this test's boundary + // assertions depend on. + assert.Equal(t, int64(1), firstA1, "statement 1 (tblA) must report its first inserted id 1, got %d", firstA1) + + // Statement 2: 2 rows into tblB (a different table). + rowsB1, firstB1, err := session.ExecuteStatement(ctx, "INSERT INTO tblB(v) VALUES('b1'),('b2')", nil) + require.NoError(t, err) + require.Equal(t, int64(2), rowsB1) + _, rowsDataB1, err := session.Query(ctx, "SELECT id FROM tblB WHERE id > 999 ORDER BY id", nil) + require.NoError(t, err) + idsB1 := extractSortedInt64Column(t, rowsDataB1, "id") + require.Equal(t, []int64{1000, 1001}, idsB1, "sanity: tblB's newly inserted rows must be 1000,1001 given the id=999 seed, or this test's own fixture is wrong") + // Reddens if statement 2 reports tblA's id (1 or 3) instead of its own + // table's first id (1000) - proves the report is scoped per statement, + // not "whatever the session's shared register/first table happened to + // hold". + assert.Equal(t, int64(1000), firstB1, "statement 2 (tblB) must report its own first inserted id 1000, not tblA's, got %d", firstB1) + + // Statement 3: back to tblA, 1 more row. + rowsA2, firstA2, err := session.ExecuteStatement(ctx, "INSERT INTO tblA(v) VALUES('a4')", nil) + require.NoError(t, err) + require.Equal(t, int64(1), rowsA2) + _, rowsDataA2, err := session.Query(ctx, "SELECT id FROM tblA WHERE id > 3 ORDER BY id", nil) + require.NoError(t, err) + idsA2 := extractSortedInt64Column(t, rowsDataA2, "id") + require.Equal(t, []int64{4}, idsA2, "sanity: tblA's new row must be 4, or this test's own fixture is wrong") + // Reddens if the implementation latches onto the last table it touched + // (tblB, ids ~1000) permanently instead of resolving per-statement, so + // statement 3 - back on tblA - never recovers tblA's own first id. + assert.Equal(t, int64(4), firstA2, "statement 3 (back to tblA) must report its own first inserted id 4, got %d", firstA2) +} + +// TestSharedRegisterNoLeakAcrossNoOpStatements is OBSERVABLE 4, the only test +// that proves the shared-register leak (db/db_integration.go hookDB capped +// at SetMaxOpenConns(1)) is fixed. It is written as a SAFETY assertion +// (checked repeatedly past the register's first successful, non-zero value), +// not a liveness one. +// +// It asserts 0 from the db layer, not "unchanged", because the session field +// above this layer drops a zero rather than storing it. That rule has exactly +// one home, protocol.ConnectionSession.RecordInsertId, and two call paths reach +// it: the coordinator's OK-packet paths (protocol/server.go) and a replica's +// forwarded responses (replica/handler.go applyForwardedSessionState). Returning +// 0 here is what preserves the client's own prior LAST_INSERT_ID() - a non-zero +// value, even the correct one from a DIFFERENT earlier statement, would +// overwrite it. +func TestSharedRegisterNoLeakAcrossNoOpStatements(t *testing.T) { + const iterations = 3 + + t.Run("autocommit", func(t *testing.T) { + source := newRowidTestDatabase(t, 1) + require.NoError(t, execAndReload(source, `CREATE TABLE t4a (id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT UNIQUE)`)) + + ctx := context.Background() + var txnID uint64 = 63000 + ignoreChecks, updateChecks := 0, 0 + + for i := 0; i < iterations; i++ { + v := fmt.Sprintf("shared-reg-%d", i) + + // (0) Prime the register with a real insert. + txnID++ + primeReq := coordinator.ExecutionRequest{SQL: fmt.Sprintf("INSERT INTO t4a(v) VALUES('%s')", v)} + primePending, err := source.ExecuteLocalWithHooks(ctx, txnID, primeReq) + require.NoError(t, err) + primeID := primePending.GetLastInsertId() + // Discriminator: without a genuinely non-zero primed id, a 0 from + // the no-op below would prove nothing (the register might never + // have held a real value in the first place). + require.NotZero(t, primeID, "iteration %d: priming insert reported id=0, so this iteration cannot discriminate a fixed register from a never-primed one", i) + require.NoError(t, primePending.Commit()) + + // (i) INSERT OR IGNORE on the just-primed duplicate key inserts + // nothing. + txnID++ + ignoreReq := coordinator.ExecutionRequest{SQL: fmt.Sprintf("INSERT OR IGNORE INTO t4a(v) VALUES('%s')", v)} + ignorePending, err := source.ExecuteLocalWithHooks(ctx, txnID, ignoreReq) + require.NoError(t, err) + ignoreID := ignorePending.GetLastInsertId() + // Reddens if ExecContext keeps reading the hookDB connection's + // single shared sqlite3_last_insert_rowid() register (still + // holding primeID here, from a completely different Go-level + // session/txnID that happened to reuse the same underlying + // SQLite connection) instead of recognizing zero rows were + // inserted by THIS statement. + assert.Equalf(t, int64(0), ignoreID, "iteration %d: INSERT OR IGNORE on duplicate key must report 0, got %d (primed id was %d)", i, ignoreID, primeID) + require.NoError(t, ignorePending.Commit()) + ignoreChecks++ + + // (ii) An upsert that updates the existing row rather than + // inserting. + txnID++ + upsertReq := coordinator.ExecutionRequest{SQL: fmt.Sprintf("INSERT INTO t4a(v) VALUES('%s') ON CONFLICT(v) DO UPDATE SET v = excluded.v", v)} + upsertPending, err := source.ExecuteLocalWithHooks(ctx, txnID, upsertReq) + require.NoError(t, err) + upsertID := upsertPending.GetLastInsertId() + assert.Equalf(t, int64(0), upsertID, "iteration %d: ON CONFLICT DO UPDATE must report 0 (it updated, not inserted), got %d", i, upsertID) + require.NoError(t, upsertPending.Commit()) + updateChecks++ + } + + // Guards against the loop silently degrading to fewer samples (e.g. + // a stray `continue`/early return) - the safety property must have + // actually been checked `iterations` times, not just once. + require.Equal(t, iterations, ignoreChecks, "autocommit: INSERT OR IGNORE no-op path must have been exercised %d times", iterations) + require.Equal(t, iterations, updateChecks, "autocommit: ON CONFLICT DO UPDATE no-op path must have been exercised %d times", iterations) + }) + + t.Run("pinned", func(t *testing.T) { + source := newRowidTestDatabase(t, 1) + require.NoError(t, execAndReload(source, `CREATE TABLE t4b (id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT UNIQUE)`)) + + ctx := context.Background() + session, err := source.BeginPinnedSession(ctx, 64001) + require.NoError(t, err) + defer session.Release() + + ignoreChecks, updateChecks := 0, 0 + + for i := 0; i < iterations; i++ { + v := fmt.Sprintf("pinned-shared-reg-%d", i) + + _, primeID, err := session.ExecuteStatement(ctx, fmt.Sprintf("INSERT INTO t4b(v) VALUES('%s')", v), nil) + require.NoError(t, err) + require.NotZero(t, primeID, "iteration %d: priming insert reported id=0, so this iteration cannot discriminate a fixed register from a never-primed one", i) + + // (i) no rows inserted. + _, ignoreID, err := session.ExecuteStatement(ctx, fmt.Sprintf("INSERT OR IGNORE INTO t4b(v) VALUES('%s')", v), nil) + require.NoError(t, err) + // Reddens for the same reason as the autocommit case above: + // pinnedHookSession.ExecuteStatement returns + // p.session.GetLastInsertId(), the raw shared register, never + // touching CDC entries (0 entries here) at all. + assert.Equalf(t, int64(0), ignoreID, "iteration %d: pinned INSERT OR IGNORE on duplicate key must report 0, got %d (primed id was %d)", i, ignoreID, primeID) + ignoreChecks++ + + // (ii) an existing row is updated, not inserted. + _, upsertID, err := session.ExecuteStatement(ctx, fmt.Sprintf("INSERT INTO t4b(v) VALUES('%s') ON CONFLICT(v) DO UPDATE SET v = excluded.v", v), nil) + require.NoError(t, err) + assert.Equalf(t, int64(0), upsertID, "iteration %d: pinned ON CONFLICT DO UPDATE must report 0, got %d", i, upsertID) + updateChecks++ + } + + require.Equal(t, iterations, ignoreChecks, "pinned: INSERT OR IGNORE no-op path must have been exercised %d times", iterations) + require.Equal(t, iterations, updateChecks, "pinned: ON CONFLICT DO UPDATE no-op path must have been exercised %d times", iterations) + }) +} + +// TestNoAutoIncrementColumnReportsZero is OBSERVABLE 5: the reported id must +// come from the table's auto-increment column, not "the pk" or SQLite's +// hidden rowid. db/schema_cache.go loadSchema only ever sets +// TableSchema.AutoIncrementCol inside `if len(pkColumns) == 1 { ... upperType +// == "INTEGER" || upperType == "BIGINT" ... }` (lines ~330-344): a single +// INTEGER/BIGINT PRIMARY KEY column. In that model, whenever +// AutoIncrementCol is set it is always equal to the table's sole PK column - +// this codebase's schema cache has no representation for "an auto-increment +// column that differs from the PK", so the strongest test the schema model +// allows is the negative case: a PK that is NOT eligible (a TEXT PRIMARY +// KEY), which must report 0, not SQLite's implicit hidden rowid (which +// exists and is non-zero for this table, since it is not WITHOUT ROWID). +func TestNoAutoIncrementColumnReportsZero(t *testing.T) { + t.Run("autocommit", func(t *testing.T) { + source := newRowidTestDatabase(t, 1) + require.NoError(t, execAndReload(source, `CREATE TABLE t5a (id TEXT PRIMARY KEY, v TEXT)`)) + + schema, err := source.GetCachedTableSchema("t5a") + require.NoError(t, err) + require.Empty(t, schema.AutoIncrementCol, "sanity: loadSchema must leave AutoIncrementCol empty for a TEXT PRIMARY KEY, or this test's own premise is wrong") + + ctx := context.Background() + req := coordinator.ExecutionRequest{SQL: "INSERT INTO t5a (id, v) VALUES ('abc', 'x')"} + pending, err := source.ExecuteLocalWithHooks(ctx, 65001, req) + require.NoError(t, err) + + // Reddens if ExecContext still surfaces SQLite's implicit hidden + // rowid (sqlite3_last_insert_rowid(), non-zero here even though the + // declared PK is TEXT) instead of looking up a named auto-increment + // column that does not exist on this table. + assert.Equal(t, int64(0), pending.GetLastInsertId(), "table with a TEXT PRIMARY KEY (no auto-increment column) must report insert id 0, got %d", pending.GetLastInsertId()) + require.NoError(t, pending.Commit()) + }) + + t.Run("pinned", func(t *testing.T) { + source := newRowidTestDatabase(t, 1) + require.NoError(t, execAndReload(source, `CREATE TABLE t5b (id TEXT PRIMARY KEY, v TEXT)`)) + + schema, err := source.GetCachedTableSchema("t5b") + require.NoError(t, err) + require.Empty(t, schema.AutoIncrementCol, "sanity: loadSchema must leave AutoIncrementCol empty for a TEXT PRIMARY KEY, or this test's own premise is wrong") + + ctx := context.Background() + session, err := source.BeginPinnedSession(ctx, 66001) + require.NoError(t, err) + defer session.Release() + + _, lastInsertId, err := session.ExecuteStatement(ctx, "INSERT INTO t5b (id, v) VALUES ('abc', 'x')", nil) + require.NoError(t, err) + // Same defect as the autocommit case, on the explicit-transaction + // path: pinnedHookSession.ExecuteStatement returns + // p.session.GetLastInsertId(), the raw hidden-rowid register. + assert.Equal(t, int64(0), lastInsertId, "pinned INSERT into a TEXT PRIMARY KEY table (no auto-increment column) must report insert id 0, got %d", lastInsertId) + }) +} diff --git a/db/preupdate_hook.go b/db/preupdate_hook.go index 419dfce0..8d0f2ad3 100644 --- a/db/preupdate_hook.go +++ b/db/preupdate_hook.go @@ -28,15 +28,14 @@ import ( // keep them in bounded session memory until SQLite rollback releases the writer; // oversized transactions spill directly to the CDC segment log. type EphemeralHookSession struct { - conn *sql.Conn // Dedicated user DB connection (closed on end) - tx *sql.Tx // Active transaction on user DB - metaStore MetaStore // MetaStore for intent entry storage - txnID uint64 // Transaction ID for intent entries - seq uint64 // Sequence counter for entries - schemaCache *SchemaCache // Shared schema cache - lastInsertId int64 // Last insert ID from most recent insert - mu sync.Mutex - captureMu sync.Mutex + conn *sql.Conn // Dedicated user DB connection (closed on end) + tx *sql.Tx // Active transaction on user DB + metaStore MetaStore // MetaStore for intent entry storage + txnID uint64 // Transaction ID for intent entries + seq uint64 // Sequence counter for entries + schemaCache *SchemaCache // Shared schema cache + mu sync.Mutex + captureMu sync.Mutex conflictError error // Set if conflict detected during hook capturedRows []capturedRow @@ -49,6 +48,12 @@ type EphemeralHookSession struct { lastProcessedSeq uint64 // high-water mark for captureAndLockNewRows eagerCaptureUsed bool // true once captureAndLockNewRows has run at least once + + // stmtInsertID is the insert id the LAST statement captured by + // captureAndLockNewRows reports to its client. It is recomputed per + // statement, so a transaction writing to two tables reports each + // statement's own id rather than the transaction's first. + stmtInsertID int64 } type capturedRow struct { @@ -161,10 +166,6 @@ func (s *EphemeralHookSession) ExecContext(ctx context.Context, query string, ar return 0, conflictErr } - if id, err := result.LastInsertId(); err == nil && id != 0 { - s.lastInsertId = id - } - rowsAffected, err := result.RowsAffected() if err != nil { return 0, err @@ -232,10 +233,17 @@ func (s *EphemeralHookSession) captureAndLockNewRows() error { } } + // newEntries holds exactly the rows THIS statement produced: the caller + // runs one statement then calls this, and lastProcessedSeq excludes every + // earlier statement's rows. s.intentEntries, by contrast, accumulates + // across the whole pinned transaction. + insertID := statementInsertID(s.schemaCache, newEntries) + s.mu.Lock() s.intentEntries = append(s.intentEntries, newEntries...) s.lastProcessedSeq = maxSeq s.eagerCaptureUsed = true + s.stmtInsertID = insertID s.mu.Unlock() return nil } @@ -436,9 +444,13 @@ func (s *EphemeralHookSession) GetTxnID() uint64 { return s.txnID } -// GetLastInsertId returns the last insert ID from the most recent insert -func (s *EphemeralHookSession) GetLastInsertId() int64 { - return s.lastInsertId +// StatementInsertID returns the insert id the most recently captured statement +// reports to its client - see statementInsertID. Valid only after +// captureAndLockNewRows has run for that statement. +func (s *EphemeralHookSession) StatementInsertID() int64 { + s.mu.Lock() + defer s.mu.Unlock() + return s.stmtInsertID } // GetConflictError returns any conflict error that occurred during CDC capture. diff --git a/db/preupdate_hook_stub.go b/db/preupdate_hook_stub.go index 90bd6e1f..e1871943 100644 --- a/db/preupdate_hook_stub.go +++ b/db/preupdate_hook_stub.go @@ -225,8 +225,8 @@ func (s *EphemeralHookSession) GetTxnID() uint64 { return 0 } -// GetLastInsertId returns 0 (stub) -func (s *EphemeralHookSession) GetLastInsertId() int64 { +// StatementInsertID returns 0 (stub) +func (s *EphemeralHookSession) StatementInsertID() int64 { return 0 } diff --git a/db/statement_insert_id.go b/db/statement_insert_id.go new file mode 100644 index 00000000..f28f1b59 --- /dev/null +++ b/db/statement_insert_id.go @@ -0,0 +1,48 @@ +package db + +// statementInsertID returns the insert id one statement reports to its client: +// the value of the target table's AUTO_INCREMENT column in the first row that +// statement inserted, read by column name from that statement's first insert +// CDC entry. +// +// It is deliberately NOT SQLite's sqlite3_last_insert_rowid(), which is the +// LAST row of a multi-row insert and lives on the connection rather than on the +// statement, and NOT "the row's primary key", which need not be the +// auto-increment column. +// +// It returns 0 when the statement inserted no rows and when the table has no +// auto-increment column. 0 is what leaves a client's LAST_INSERT_ID() unchanged, +// which is MySQL's documented behaviour when no rows are successfully inserted. +// +// That only holds because every writer of the session field drops a zero, and +// there are two call paths, not one: the coordinator's OK-packet paths in +// protocol/server.go and a replica's forwarded responses in +// replica/handler.go (applyForwardedSessionState). Both go through +// protocol.ConnectionSession.RecordInsertId, which is the single home of the +// rule; the replica path stored zeros unconditionally until that was fixed, so +// do not assume a new writer inherits the behaviour - route it through +// RecordInsertId. +func statementInsertID(schemaCache *SchemaCache, entries []*IntentEntry) int64 { + if schemaCache == nil { + return 0 + } + for _, entry := range entries { + if entry == nil || entry.Operation != uint8(OpTypeInsert) { + continue + } + schema, err := schemaCache.GetSchemaFor(entry.Table) + if err != nil || schema == nil { + return 0 + } + column := schema.GetAutoIncrementCol() + if column == "" { + return 0 + } + id, ok, err := decodeCDCInt64(entry.NewValues, column) + if err != nil || !ok { + return 0 + } + return id + } + return 0 +} diff --git a/grpc/common/marmot_common.pb.go b/grpc/common/marmot_common.pb.go index 56376b6a..24d2e7f3 100644 --- a/grpc/common/marmot_common.pb.go +++ b/grpc/common/marmot_common.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.10 -// protoc v6.33.4 +// protoc v6.33.6 // source: grpc/common/marmot_common.proto package common diff --git a/grpc/forward_error_code_test.go b/grpc/forward_error_code_test.go new file mode 100644 index 00000000..9024f714 --- /dev/null +++ b/grpc/forward_error_code_test.go @@ -0,0 +1,81 @@ +package grpc + +import ( + "errors" + "testing" + + "github.com/maxpert/marmot/protocol" + "github.com/maxpert/marmot/protocol/query/transform" +) + +// TestForwardFailureCarriesTheMySQLCode pins the leader half of the forwarded +// error path: the response carries the same MySQL error code the coordinator +// would have put in its own ERR packet, so a replica can reproduce it instead of +// reporting ER_UNKNOWN_ERROR for every failure. +// +// The code is taken from protocol.ConvertToMySQLError rather than re-derived +// here, so the two paths cannot drift apart. +func TestForwardFailureCarriesTheMySQLCode(t *testing.T) { + cases := []struct { + name string + err error + wantCode uint32 + wantSQLState string + wantMessage string + }{ + { + // The deliverable: a rule's rejection reaches the wire with 1235. + // Mutation: set ErrorCode to 0, or drop the field assignment. + name: "a rule rejection is carried as 1235", + err: transform.NewCodedError(transform.ErrCodeNotSupportedYet, "nope"), + wantCode: 1235, + wantSQLState: protocol.SQLStateSyntax, + wantMessage: "nope", + }, + { + // A typed MySQL error keeps its own code. + // Mutation: same as above. + name: "a typed MySQL error keeps its code", + err: protocol.NewMySQLError(protocol.ErrCodeDupEntry, protocol.SQLStateIntegrity, "Duplicate entry"), + wantCode: 1062, + wantSQLState: protocol.SQLStateIntegrity, + wantMessage: "Duplicate entry", + }, + { + // A plain error maps to ER_UNKNOWN_ERROR, which is what the + // coordinator's own ERR packet would carry for it. + // Mutation: hardcode a non-1105 default. + name: "a plain error becomes 1105", + err: errors.New("something broke"), + wantCode: 1105, + wantSQLState: protocol.SQLStateGeneral, + wantMessage: "something broke", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + resp := forwardFailure(tc.err) + + if resp.Success { + t.Error("forwardFailure produced Success = true") + } + if resp.ErrorCode != tc.wantCode { + t.Errorf("ErrorCode = %d, want %d", resp.ErrorCode, tc.wantCode) + } + // The SQLSTATE travels too, because the code does not determine it: + // 1105 pairs with both HY000 and 23000 on the leader. + // Mutation: drop the SqlState assignment in forwardFailure. + if resp.SqlState != tc.wantSQLState { + t.Errorf("SqlState = %q, want %q", resp.SqlState, tc.wantSQLState) + } + // The message must be the mapped message, not err.Error(): a + // *protocol.MySQLError formats its own code into Error(), and that + // string would otherwise be sent as the message text. + // Mutation: use err.Error() instead of mysqlErr.Message. + if resp.ErrorMessage != tc.wantMessage { + t.Errorf("ErrorMessage = %q, want %q", resp.ErrorMessage, tc.wantMessage) + } + }) + } +} diff --git a/grpc/forward_handler.go b/grpc/forward_handler.go index c275889b..5ab7786d 100644 --- a/grpc/forward_handler.go +++ b/grpc/forward_handler.go @@ -51,6 +51,28 @@ func NewForwardHandler(nodeID uint64, clock *hlc.Clock, sessionMgr *ForwardSessi } } +// forwardFailure builds the response for an error the leader's own handler +// returned, carrying the MySQL error code the coordinator would have put in its +// ERR packet so the replica can reproduce it rather than flattening every +// failure to ER_UNKNOWN_ERROR. +// +// It goes through protocol.ConvertToMySQLError - the same mapping the protocol +// server uses on its own ERR path - so neither the code nor the SQLSTATE is +// ever re-derived. Both travel, because the code alone does not determine the +// SQLSTATE: 1105 pairs with HY000 and with 23000. The +// message is the mapped message, not err.Error(), because a *protocol.MySQLError +// formats its own code into Error() and that string would then be sent as the +// message text. +func forwardFailure(err error) *ForwardQueryResponse { + mysqlErr := protocol.ConvertToMySQLError(err) + return &ForwardQueryResponse{ + Success: false, + ErrorMessage: mysqlErr.Message, + ErrorCode: uint32(mysqlErr.Code), + SqlState: mysqlErr.SQLState, + } +} + // HandleForwardQuery handles a forwarded query from a read-only replica func (h *ForwardHandler) HandleForwardQuery(ctx context.Context, req *ForwardQueryRequest) (*ForwardQueryResponse, error) { if req.ReplicaNodeId == 0 || req.SessionId == 0 { @@ -176,10 +198,7 @@ func (h *ForwardHandler) HandleForwardLoadData(ctx context.Context, req *Forward return h.executeIdempotentLoad(ctx, session, req, func(connSession *protocol.ConnectionSession) *ForwardQueryResponse { rs, err := h.coordHandler.HandleLoadData(connSession, req.Sql, req.Data) if err != nil { - return &ForwardQueryResponse{ - Success: false, - ErrorMessage: err.Error(), - } + return forwardFailure(err) } resp := &ForwardQueryResponse{Success: true} @@ -226,10 +245,7 @@ func (h *ForwardHandler) executeIdempotentRequest( return execFn(connSession), nil }) if execErr != nil { - resp = &ForwardQueryResponse{ - Success: false, - ErrorMessage: execErr.Error(), - } + resp = forwardFailure(execErr) } if resp == nil { resp = &ForwardQueryResponse{Success: true} @@ -252,10 +268,7 @@ func (h *ForwardHandler) handleDatabaseOp(connSession *protocol.ConnectionSessio rs, err := h.coordHandler.HandleQuery(connSession, req.Sql, nil) if err != nil { - return &ForwardQueryResponse{ - Success: false, - ErrorMessage: err.Error(), - } + return forwardFailure(err) } var rowsAffected int64 @@ -300,10 +313,7 @@ func (h *ForwardHandler) handleTxnControl(connSession *protocol.ConnectionSessio rs, err := h.coordHandler.HandleQuery(connSession, sql, nil) connSession.TranspilationEnabled = prevTranspilation if err != nil { - return &ForwardQueryResponse{ - Success: false, - ErrorMessage: err.Error(), - } + return forwardFailure(err) } resp := &ForwardQueryResponse{Success: true} @@ -342,10 +352,7 @@ func (h *ForwardHandler) handleStatement(connSession *protocol.ConnectionSession Uint64("request_id", req.RequestId). Str("sql", req.Sql). Msg("Statement execution failed") - return &ForwardQueryResponse{ - Success: false, - ErrorMessage: execErr.Error(), - } + return forwardFailure(execErr) } var rowsAffected int64 diff --git a/grpc/marmot.pb.go b/grpc/marmot.pb.go index e3090b34..b4022074 100644 --- a/grpc/marmot.pb.go +++ b/grpc/marmot.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.10 -// protoc v6.33.4 +// protoc v6.33.6 // source: grpc/marmot.proto package grpc @@ -3046,8 +3046,19 @@ type ForwardQueryResponse struct { LastInsertId int64 `protobuf:"varint,4,opt,name=last_insert_id,json=lastInsertId,proto3" json:"last_insert_id,omitempty"` CommittedTxnId uint64 `protobuf:"varint,5,opt,name=committed_txn_id,json=committedTxnId,proto3" json:"committed_txn_id,omitempty"` // For read-your-writes InTransaction bool `protobuf:"varint,6,opt,name=in_transaction,json=inTransaction,proto3" json:"in_transaction,omitempty"` // True if session has active transaction on leader - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // MySQL server error code for a failed query, so a replica can reproduce the + // leader's ERR packet instead of flattening every failure to ER_UNKNOWN_ERROR. + // 0 means "not supplied": an older leader, or a rejection raised before the + // statement reached the coordinator. A replica reports 1105 for 0. + ErrorCode uint32 `protobuf:"varint,7,opt,name=error_code,json=errorCode,proto3" json:"error_code,omitempty"` + // SQLSTATE for the same failure. It is sent rather than derived from the code + // because the mapping is not injective: the leader pairs code 1105 with both + // HY000 and 23000 (an unclassified error vs an unclassified CONSTRAINT + // failure), so a replica rebuilding it from the number alone would have to + // guess. Empty means "not supplied", same as error_code 0. + SqlState string `protobuf:"bytes,8,opt,name=sql_state,json=sqlState,proto3" json:"sql_state,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ForwardQueryResponse) Reset() { @@ -3122,6 +3133,20 @@ func (x *ForwardQueryResponse) GetInTransaction() bool { return false } +func (x *ForwardQueryResponse) GetErrorCode() uint32 { + if x != nil { + return x.ErrorCode + } + return 0 +} + +func (x *ForwardQueryResponse) GetSqlState() string { + if x != nil { + return x.SqlState + } + return "" +} + type ForwardLoadDataRequest struct { state protoimpl.MessageState `protogen:"open.v1"` ReplicaNodeId uint64 `protobuf:"varint,1,opt,name=replica_node_id,json=replicaNodeId,proto3" json:"replica_node_id,omitempty"` @@ -3583,14 +3608,17 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x14wait_for_replication\x18\x06 \x01(\bR\x12waitForReplication\x12\x16\n" + "\x06params\x18\a \x03(\fR\x06params\x12\x1d\n" + "\n" + - "timeout_ms\x18\b \x01(\rR\ttimeoutMs\"\xf1\x01\n" + + "timeout_ms\x18\b \x01(\rR\ttimeoutMs\"\xad\x02\n" + "\x14ForwardQueryResponse\x12\x18\n" + "\asuccess\x18\x01 \x01(\bR\asuccess\x12#\n" + "\rerror_message\x18\x02 \x01(\tR\ferrorMessage\x12#\n" + "\rrows_affected\x18\x03 \x01(\x03R\frowsAffected\x12$\n" + "\x0elast_insert_id\x18\x04 \x01(\x03R\flastInsertId\x12(\n" + "\x10committed_txn_id\x18\x05 \x01(\x04R\x0ecommittedTxnId\x12%\n" + - "\x0ein_transaction\x18\x06 \x01(\bR\rinTransaction\"\x91\x02\n" + + "\x0ein_transaction\x18\x06 \x01(\bR\rinTransaction\x12\x1d\n" + + "\n" + + "error_code\x18\a \x01(\rR\terrorCode\x12\x1b\n" + + "\tsql_state\x18\b \x01(\tR\bsqlState\"\x91\x02\n" + "\x16ForwardLoadDataRequest\x12&\n" + "\x0freplica_node_id\x18\x01 \x01(\x04R\rreplicaNodeId\x12\x1d\n" + "\n" + diff --git a/grpc/marmot.proto b/grpc/marmot.proto index d05b4f9f..1e3fde02 100644 --- a/grpc/marmot.proto +++ b/grpc/marmot.proto @@ -423,6 +423,17 @@ message ForwardQueryResponse { int64 last_insert_id = 4; uint64 committed_txn_id = 5; // For read-your-writes bool in_transaction = 6; // True if session has active transaction on leader + // MySQL server error code for a failed query, so a replica can reproduce the + // leader's ERR packet instead of flattening every failure to ER_UNKNOWN_ERROR. + // 0 means "not supplied": an older leader, or a rejection raised before the + // statement reached the coordinator. A replica reports 1105 for 0. + uint32 error_code = 7; + // SQLSTATE for the same failure. It is sent rather than derived from the code + // because the mapping is not injective: the leader pairs code 1105 with both + // HY000 and 23000 (an unclassified error vs an unclassified CONSTRAINT + // failure), so a replica rebuilding it from the number alone would have to + // guess. Empty means "not supplied", same as error_code 0. + string sql_state = 8; } message ForwardLoadDataRequest { diff --git a/grpc/marmot_grpc.pb.go b/grpc/marmot_grpc.pb.go index e072fcbd..4c01c85b 100644 --- a/grpc/marmot_grpc.pb.go +++ b/grpc/marmot_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.5.1 -// - protoc v6.33.4 +// - protoc v6.33.6 // source: grpc/marmot.proto package grpc diff --git a/grpc/replication_handler.go b/grpc/replication_handler.go index e8b3de19..20eb9267 100644 --- a/grpc/replication_handler.go +++ b/grpc/replication_handler.go @@ -116,10 +116,17 @@ func (rh *ReplicationHandler) handlePrepare(ctx context.Context, req *Transactio // Convert proto statements to internal format statements := make([]protocol.Statement, 0, len(req.Statements)) for _, stmt := range req.Statements { - internalStmt := protocolStatementFromProto(stmt) + internalStmt, err := protocolStatementFromProto(stmt) + if err != nil { + log.Error().Err(err).Uint64("txn_id", req.TxnId).Str("database", req.Database).Msg("Rejecting PREPARE: unrecognized statement on wire") + return &TransactionResponse{ + Success: false, + ErrorMessage: fmt.Sprintf("unrecognized statement: %v", err), + }, nil + } + // protocolStatementFromProto already copied SQL and the inline payload + // from this arm; only the out-of-band fetch below is still needed. if loadData := stmt.GetLoadDataChange(); loadData != nil { - internalStmt.SQL = loadData.Sql - internalStmt.LoadDataPayload = loadData.Data if len(internalStmt.LoadDataPayload) == 0 && loadData.LoadId != "" { payload, err := rh.pullLoadDataPayload(ctx, req.SourceNodeId, loadData.LoadId, loadData.DataSize, loadData.ChunkBytes) if err != nil { @@ -183,7 +190,15 @@ func (rh *ReplicationHandler) handleCommit(ctx context.Context, req *Transaction // durable from PREPARE; COMMIT may carry only DML intent metadata. statements := make([]protocol.Statement, 0, len(req.Statements)) for _, stmt := range req.Statements { - statements = append(statements, protocolStatementFromProto(stmt)) + internalStmt, err := protocolStatementFromProto(stmt) + if err != nil { + log.Error().Err(err).Uint64("txn_id", req.TxnId).Str("database", req.Database).Msg("Rejecting COMMIT: unrecognized statement on wire") + return &TransactionResponse{ + Success: false, + ErrorMessage: fmt.Sprintf("unrecognized statement: %v", err), + }, nil + } + statements = append(statements, internalStmt) } engineReq := &db.CommitRequest{ diff --git a/grpc/vector_statement.go b/grpc/vector_statement.go index 75c697b3..03269d1c 100644 --- a/grpc/vector_statement.go +++ b/grpc/vector_statement.go @@ -1,6 +1,8 @@ package grpc import ( + "fmt" + appcommon "github.com/maxpert/marmot/common" "github.com/maxpert/marmot/protocol" ) @@ -98,16 +100,31 @@ func vectorActionFromProto(action VectorIndexAction) appcommon.VectorIndexAction } } -func protocolStatementFromProto(stmt *Statement) protocol.Statement { +// protocolStatementFromProto converts a wire Statement into the internal +// protocol.Statement representation. It returns an error - rather than +// panicking - when the wire carries a StatementType or a oneof payload case +// this binary does not recognise, so that a newer peer talking to an older +// node fails the transaction instead of crashing the process or silently +// dropping the change. +func protocolStatementFromProto(stmt *Statement) (protocol.Statement, error) { + code, ok := appcommon.FromWireType(stmt.Type) + if !ok { + return protocol.Statement{}, fmt.Errorf("unknown wire StatementType %d", stmt.Type) + } + internalStmt := protocol.Statement{ SQL: stmt.GetSQL(), - Type: appcommon.MustFromWireType(stmt.Type), + Type: code, TableName: stmt.TableName, Database: stmt.Database, IntentKey: stmt.GetIntentKey(), } - if change := stmt.GetVectorIndexChange(); change != nil { - vectorChange := vectorChangeFromProto(change) + + switch payload := stmt.Payload.(type) { + case nil: + // No payload set - accepted as-is (e.g. tests, control-only statements). + case *Statement_VectorIndexChange: + vectorChange := vectorChangeFromProto(payload.VectorIndexChange) internalStmt.Type = statementTypeForVectorAction(vectorChange.Action) internalStmt.TableName = vectorChange.TableName internalStmt.Database = vectorChange.Database @@ -120,23 +137,27 @@ func protocolStatementFromProto(stmt *Statement) protocol.Statement { internalStmt.VectorNprobe = vectorChange.Nprobe internalStmt.VectorMaxNorm = vectorChange.MaxNorm internalStmt.VectorIndexChange = &vectorChange - return internalStmt - } - if rowChange := stmt.GetRowChange(); rowChange != nil { - internalStmt.EncodedRow = rowChange.EncodedRow - internalStmt.EncodedCodec = rowChange.EncodedRowCodec + return internalStmt, nil + case *Statement_RowChange: + internalStmt.EncodedRow = payload.RowChange.EncodedRow + internalStmt.EncodedCodec = payload.RowChange.EncodedRowCodec if row, err := decodeRowChange(stmt); err == nil && row != nil { internalStmt.IntentKey = row.IntentKey internalStmt.Operation = row.Op internalStmt.OldValues = row.OldValues internalStmt.NewValues = row.NewValues } + case *Statement_LoadDataChange: + internalStmt.SQL = payload.LoadDataChange.Sql + internalStmt.LoadDataPayload = payload.LoadDataChange.Data + case *Statement_DdlChange: + // Already consumed above via stmt.GetSQL(). + case *Statement_DmlIntent: + // Already consumed above via stmt.GetIntentKey(). + default: + return protocol.Statement{}, fmt.Errorf("unknown statement payload type %T", payload) } - if loadData := stmt.GetLoadDataChange(); loadData != nil { - internalStmt.SQL = loadData.Sql - internalStmt.LoadDataPayload = loadData.Data - } - return internalStmt + return internalStmt, nil } func statementTypeForVectorAction(action appcommon.VectorIndexAction) protocol.StatementCode { diff --git a/grpc/vector_statement_test.go b/grpc/vector_statement_test.go index 574daab7..0d347160 100644 --- a/grpc/vector_statement_test.go +++ b/grpc/vector_statement_test.go @@ -68,7 +68,8 @@ func TestConvertStatementsToProtoUsesVectorControlPayload(t *testing.T) { require.NotNil(t, stmts[0].GetVectorIndexChange()) require.Empty(t, stmts[0].GetSQL()) - internal := protocolStatementFromProto(stmts[0]) + internal, err := protocolStatementFromProto(stmts[0]) + require.NoError(t, err) require.Equal(t, protocol.StatementCreateVectorIndex, internal.Type) require.Equal(t, "docs_embed_idx", internal.VectorIndexName) require.NotNil(t, internal.VectorIndexChange) diff --git a/grpc/vector_statement_wiretype_test.go b/grpc/vector_statement_wiretype_test.go new file mode 100644 index 00000000..deea4508 --- /dev/null +++ b/grpc/vector_statement_wiretype_test.go @@ -0,0 +1,197 @@ +package grpc + +import ( + "testing" + + "github.com/maxpert/marmot/db" + pb "github.com/maxpert/marmot/grpc/common" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// callProtocolStatementFromProto invokes protocolStatementFromProto and proves +// the call never panics, regardless of what the result turns out to be. This +// is what makes TestProtocolStatementFromProto_UnknownWireType red if +// MustFromWireType (which panics instead of returning an error) is ever +// restored on this path: the recover fires and t.Fatalf runs before the +// normal error assertions are even reached. +func callProtocolStatementFromProto(t *testing.T, stmt *Statement) (protocol.Statement, error) { + t.Helper() + defer func() { + if r := recover(); r != nil { + t.Fatalf("protocolStatementFromProto panicked: %v", r) + } + }() + return protocolStatementFromProto(stmt) +} + +// TestProtocolStatementFromProto_UnknownWireType asserts that an unknown wire +// StatementType produces a non-nil, descriptive error instead of a panic. +// Mutation caught: reverting appcommon.FromWireType back to +// appcommon.MustFromWireType makes the recover() in +// callProtocolStatementFromProto fire and fail the test at that Fatalf, +// before the require.Error assertion below is even reached. +func TestProtocolStatementFromProto_UnknownWireType(t *testing.T) { + t.Parallel() + + stmt := &Statement{Type: pb.StatementType(999), TableName: "docs", Database: "app"} + _, err := callProtocolStatementFromProto(t, stmt) + + // Mutation caught: dropping the `if !ok { return ..., err }` guard (i.e. + // ignoring the FromWireType ok value) makes err nil and reddens this line. + require.Error(t, err) + require.Contains(t, err.Error(), "999") +} + +// unknownStatementPayload is a oneof implementation this binary does not +// know about - standing in for a payload case added by a newer peer. It is +// defined in this package (not imported) because isStatement_Payload() is +// unexported and can only be implemented from within package grpc. +type unknownStatementPayload struct{} + +func (unknownStatementPayload) isStatement_Payload() {} + +// TestProtocolStatementFromProto_UnknownPayload asserts that a recognised +// payload case is required. Mutation caught: removing the `default:` arm +// from the switch in protocolStatementFromProto (or turning it into a no-op) +// makes err nil and reddens the require.Error assertion below - the +// statement would otherwise silently ACK with no SQL, no intent key and no +// row image (the original silent-data-loss defect). +func TestProtocolStatementFromProto_UnknownPayload(t *testing.T) { + t.Parallel() + + stmt := &Statement{ + Type: pb.StatementType_INSERT, + TableName: "docs", + Database: "app", + Payload: unknownStatementPayload{}, + } + _, err := callProtocolStatementFromProto(t, stmt) + + require.Error(t, err) + require.Contains(t, err.Error(), "unknownStatementPayload") +} + +// TestProtocolStatementFromProto_KnownPayloads is a regression guard: every +// payload kind the wire format knows about today must keep converting with a +// nil error and the same field values it produces today. +func TestProtocolStatementFromProto_KnownPayloads(t *testing.T) { + t.Parallel() + + vectorChange := &VectorIndexChange{ + Action: VectorIndexAction_VECTOR_INDEX_ACTION_CREATE, + Database: "app", + IndexName: "docs_embed_idx", + TableName: "docs", + Metric: "cosine", + Dim: 128, + } + + tests := []struct { + name string + stmt *Statement + check func(t *testing.T, got protocol.Statement) + }{ + { + name: "row_change", + stmt: &Statement{ + Type: pb.StatementType_INSERT, + TableName: "docs", + Database: "app", + Payload: &Statement_RowChange{RowChange: testInsertRowChange("docs", []byte("pk:1"), map[string][]byte{"id": {1}})}, + }, + check: func(t *testing.T, got protocol.Statement) { + t.Helper() + // Mutation caught: dropping the *Statement_RowChange case (or its + // decodeRowChange call) leaves IntentKey/NewValues zero, reddening + // these two assertions. + require.Equal(t, []byte("pk:1"), got.IntentKey) + require.Equal(t, map[string][]byte{"id": {1}}, got.NewValues) + // The encoded row image is what a peer actually applies, and + // nothing else in this file asserted it - a mutation arm that + // deleted these two assignments went green. + // Mutation caught: dropping either assignment in the + // *Statement_RowChange case. + require.NotEmpty(t, got.EncodedRow) + require.Equal(t, db.EncodedCapturedRowCodecMsgpack(), got.EncodedCodec) + }, + }, + { + name: "ddl_change", + stmt: &Statement{ + Type: pb.StatementType_DDL, + TableName: "docs", + Database: "app", + Payload: &Statement_DdlChange{DdlChange: &DDLChange{Sql: "CREATE TABLE docs (id INT)"}}, + }, + check: func(t *testing.T, got protocol.Statement) { + t.Helper() + // Mutation caught: making the ddl_change case fall into `default` + // (returning an error) turns this into a test failure via err below; + // mutating stmt.GetSQL() away from the top-level assignment reddens + // this SQL assertion specifically. + require.Equal(t, "CREATE TABLE docs (id INT)", got.SQL) + }, + }, + { + name: "load_data_change", + stmt: &Statement{ + Type: pb.StatementType_LOAD_DATA, + TableName: "docs", + Database: "app", + Payload: &Statement_LoadDataChange{LoadDataChange: &LoadDataChange{Sql: "LOAD DATA ...", Data: []byte("payload")}}, + }, + check: func(t *testing.T, got protocol.Statement) { + t.Helper() + // Mutation caught: dropping the *Statement_LoadDataChange case + // leaves SQL/LoadDataPayload unset, reddening these two. + require.Equal(t, "LOAD DATA ...", got.SQL) + require.Equal(t, []byte("payload"), got.LoadDataPayload) + }, + }, + { + name: "vector_index_change", + stmt: &Statement{ + Type: pb.StatementType_VECTOR_INDEX, + TableName: "docs", + Database: "app", + Payload: &Statement_VectorIndexChange{VectorIndexChange: vectorChange}, + }, + check: func(t *testing.T, got protocol.Statement) { + t.Helper() + // Mutation caught: dropping the *Statement_VectorIndexChange case + // leaves Type as StatementInsert (from the wire type) instead of + // StatementCreateVectorIndex, and VectorIndexName empty. + require.Equal(t, protocol.StatementCreateVectorIndex, got.Type) + require.Equal(t, "docs_embed_idx", got.VectorIndexName) + require.NotNil(t, got.VectorIndexChange) + }, + }, + { + name: "dml_intent", + stmt: &Statement{ + Type: pb.StatementType_UPDATE, + TableName: "docs", + Database: "app", + Payload: &Statement_DmlIntent{DmlIntent: &DMLIntent{IntentKey: []byte("pk:2")}}, + }, + check: func(t *testing.T, got protocol.Statement) { + t.Helper() + // Mutation caught: making the dml_intent case fall into `default` + // turns this into an error via err below; mutating + // stmt.GetIntentKey() away from the top-level assignment reddens + // this IntentKey assertion specifically. + require.Equal(t, []byte("pk:2"), got.IntentKey) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, err := protocolStatementFromProto(tt.stmt) + require.NoError(t, err) + tt.check(t, got) + }) + } +} diff --git a/protocol/error_mapper.go b/protocol/error_mapper.go index 767cf68b..600f5ca4 100644 --- a/protocol/error_mapper.go +++ b/protocol/error_mapper.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/mattn/go-sqlite3" + "github.com/maxpert/marmot/protocol/query/transform" ) // ConvertToMySQLError converts any error to *MySQLError with appropriate MySQL error codes @@ -18,6 +19,12 @@ func ConvertToMySQLError(err error) *MySQLError { return mysqlErr } + // A transformation rule refused the statement and chose the code itself. + var coded *transform.CodedError + if errors.As(err, &coded) { + return NewMySQLError(coded.Code, sqlStateForCode(coded.Code), coded.Message) + } + // Try to extract sqlite3.Error var sqliteErr sqlite3.Error if errors.As(err, &sqliteErr) { @@ -28,6 +35,55 @@ func ConvertToMySQLError(err error) *MySQLError { return mapByMessage(err.Error()) } +// UnsupportedStatementError builds the error a client sees for a statement the +// query pipeline refused. A transformation rule that rejected the statement +// chose its own MySQL error code; a rule that failed for any other reason is +// ER_UNKNOWN_ERROR, because a rule failing internally is not a syntax error; +// anything else reached StatementUnsupported from the parser and is one. +func UnsupportedStatementError(stmt Statement) *MySQLError { + if stmt.TranspileErr != nil { + var coded *transform.CodedError + if errors.As(stmt.TranspileErr, &coded) { + return NewMySQLError(coded.Code, sqlStateForCode(coded.Code), coded.Message) + } + return NewMySQLError(ErrCodeUnknown, SQLStateGeneral, stmt.Error) + } + return NewMySQLError(ErrCodeParseError, SQLStateSyntax, stmt.Error) +} + +// sqlStateForCode returns the SQLSTATE MySQL pairs with a server error code. +// +// Its one caller is the rule-rejection path: a transformation rule raised below +// the protocol layer names only a code, because the SQLSTATE constants are not +// visible from there. It is NOT a general inverse of the constructors in this +// file and must not be used as one - the pairing is not injective, since 1105 +// occurs with both HY000 and SQLStateIntegrity. Anything that needs a leader's +// exact SQLSTATE has to carry it, not rebuild it. +func sqlStateForCode(code uint16) string { + switch code { + case ErrCodeDupEntry, ErrCodeBadNull, ErrCodeNoReferencedRow, ErrCodeCheckConstraint: + return SQLStateIntegrity + case ErrCodeDeadlock: + return SQLStateDeadlock + case ErrCodeNoSuchTable: + return SQLStateNoSuchTable + case ErrCodeTableExists: + return SQLStateTableExists + case ErrCodeBadField: + return SQLStateNoSuchCol + case ErrCodeDupFieldName: + return SQLStateDupColumn + case ErrCodeParseError, transform.ErrCodeNotSupportedYet: + return SQLStateSyntax + case ErrCodeNoDB: + return SQLStateNoDB + case ErrCodeServerShutdown: + return "08S01" + default: + return SQLStateGeneral + } +} + func mapSQLiteError(e sqlite3.Error, msg string) *MySQLError { // Check extended codes first (more specific) switch e.ExtendedCode { diff --git a/protocol/mysql_errors.go b/protocol/mysql_errors.go index e96fbfd9..6b29d763 100644 --- a/protocol/mysql_errors.go +++ b/protocol/mysql_errors.go @@ -13,6 +13,7 @@ const ( ErrCodeParseError uint16 = 1064 ErrCodeTooBigRowsize uint16 = 1118 ErrCodeNoSuchTable uint16 = 1146 + ErrCodeNoDB uint16 = 1046 ErrCodeLockTimeout uint16 = 1205 ErrCodeDeadlock uint16 = 1213 ErrCodeReadOnly uint16 = 1290 @@ -31,6 +32,7 @@ const ( SQLStateNoSuchTable = "42S02" SQLStateNoSuchCol = "42S22" SQLStateDupColumn = "42S21" + SQLStateNoDB = "3D000" ) // MySQLError represents a MySQL protocol error with error code and SQLSTATE diff --git a/protocol/parser.go b/protocol/parser.go index 5d7c5857..5768da82 100644 --- a/protocol/parser.go +++ b/protocol/parser.go @@ -56,51 +56,12 @@ var ( // Load XML pattern (MySQL specific) loadXMLPattern = regexp.MustCompile(`(?i)^\s*LOAD\s+XML\s+`) - // DDL patterns for MySQL-specific objects not in Vitess - createTriggerPattern = regexp.MustCompile(`(?i)^\s*CREATE\s+TRIGGER\s+`) - dropTriggerPattern = regexp.MustCompile(`(?i)^\s*DROP\s+TRIGGER\s+`) - createProcedurePattern = regexp.MustCompile(`(?i)^\s*CREATE\s+PROCEDURE\s+`) - dropProcedurePattern = regexp.MustCompile(`(?i)^\s*DROP\s+PROCEDURE\s+`) - alterProcedurePattern = regexp.MustCompile(`(?i)^\s*ALTER\s+PROCEDURE\s+`) - createFunctionPattern = regexp.MustCompile(`(?i)^\s*CREATE\s+FUNCTION\s+`) - dropFunctionPattern = regexp.MustCompile(`(?i)^\s*DROP\s+FUNCTION\s+`) - alterFunctionPattern = regexp.MustCompile(`(?i)^\s*ALTER\s+FUNCTION\s+`) - createEventPattern = regexp.MustCompile(`(?i)^\s*CREATE\s+EVENT\s+`) - dropEventPattern = regexp.MustCompile(`(?i)^\s*DROP\s+EVENT\s+`) - alterEventPattern = regexp.MustCompile(`(?i)^\s*ALTER\s+EVENT\s+`) - createTablespacePattern = regexp.MustCompile(`(?i)^\s*CREATE\s+TABLESPACE\s+`) - dropTablespacePattern = regexp.MustCompile(`(?i)^\s*DROP\s+TABLESPACE\s+`) - alterTablespacePattern = regexp.MustCompile(`(?i)^\s*ALTER\s+TABLESPACE\s+`) - createLogfileGroupPattern = regexp.MustCompile(`(?i)^\s*CREATE\s+LOGFILE\s+GROUP\s+`) - dropLogfileGroupPattern = regexp.MustCompile(`(?i)^\s*DROP\s+LOGFILE\s+GROUP\s+`) - alterLogfileGroupPattern = regexp.MustCompile(`(?i)^\s*ALTER\s+LOGFILE\s+GROUP\s+`) - createServerPattern = regexp.MustCompile(`(?i)^\s*CREATE\s+SERVER\s+`) - dropServerPattern = regexp.MustCompile(`(?i)^\s*DROP\s+SERVER\s+`) - alterServerPattern = regexp.MustCompile(`(?i)^\s*ALTER\s+SERVER\s+`) - createSpatialRefPattern = regexp.MustCompile(`(?i)^\s*CREATE\s+SPATIAL\s+REFERENCE\s+SYSTEM\s+`) - dropSpatialRefPattern = regexp.MustCompile(`(?i)^\s*DROP\s+SPATIAL\s+REFERENCE\s+SYSTEM\s+`) - createResourceGroupPattern = regexp.MustCompile(`(?i)^\s*CREATE\s+RESOURCE\s+GROUP\s+`) - dropResourceGroupPattern = regexp.MustCompile(`(?i)^\s*DROP\s+RESOURCE\s+GROUP\s+`) - alterResourceGroupPattern = regexp.MustCompile(`(?i)^\s*ALTER\s+RESOURCE\s+GROUP\s+`) - dropIndexPattern = regexp.MustCompile(`(?i)^\s*DROP\s+INDEX\s+`) - renameTablePattern = regexp.MustCompile(`(?i)^\s*RENAME\s+TABLE\s+`) + // DDL pattern for an object Vitess does not parse + dropIndexPattern = regexp.MustCompile(`(?i)^\s*DROP\s+INDEX\s+`) // Vector index DDL patterns createVectorIndexPattern = regexp.MustCompile(`(?i)^\s*CREATE\s+VECTOR\s+INDEX\s+`) dropVectorIndexPattern = regexp.MustCompile(`(?i)^\s*DROP\s+VECTOR\s+INDEX\s+`) - - // DCL patterns - createUserPattern = regexp.MustCompile(`(?i)^\s*CREATE\s+USER\s+`) - dropUserPattern = regexp.MustCompile(`(?i)^\s*DROP\s+USER\s+`) - alterUserPattern = regexp.MustCompile(`(?i)^\s*ALTER\s+USER\s+`) - renameUserPattern = regexp.MustCompile(`(?i)^\s*RENAME\s+USER\s+`) - setPasswordPattern = regexp.MustCompile(`(?i)^\s*SET\s+PASSWORD\s+`) - grantPattern = regexp.MustCompile(`(?i)^\s*GRANT\s+`) - revokePattern = regexp.MustCompile(`(?i)^\s*REVOKE\s+`) - createRolePattern = regexp.MustCompile(`(?i)^\s*CREATE\s+ROLE\s+`) - dropRolePattern = regexp.MustCompile(`(?i)^\s*DROP\s+ROLE\s+`) - setRolePattern = regexp.MustCompile(`(?i)^\s*SET\s+ROLE\s+`) - setDefaultRolePattern = regexp.MustCompile(`(?i)^\s*SET\s+DEFAULT\s+ROLE\s+`) ) var globalPipeline *query.Pipeline @@ -113,8 +74,9 @@ func InitializePipeline(cacheSize int, idGen id.Generator) error { return err } -// SchemaLookupFunc returns the auto-increment column name for a table, or empty string if none. -type SchemaLookupFunc func(table string) string +// SchemaLookupFunc returns the schema facts an INSERT needs for auto-increment +// id injection, or nil if the table is unknown. +type SchemaLookupFunc func(table string) *transform.SchemaInfo // ParseOptions holds options for parsing SQL statements. type ParseOptions struct { @@ -146,9 +108,10 @@ func ParseStatementWithOptions(sql string, opts ParseOptions) Statement { Bool("skip_transpilation", opts.SkipTranspilation). Msg("PARSE: Pipeline processing failed") return Statement{ - SQL: sql, - Type: StatementUnsupported, - Error: err.Error(), + SQL: sql, + Type: StatementUnsupported, + Error: err.Error(), + TranspileErr: ctx.Output.TranspileErr, } } @@ -195,9 +158,10 @@ func ParseStatementWithSchema(sql string, schemaLookup SchemaLookupFunc) Stateme Str("sql_prefix", truncateSQLForLog(sql, 80)). Msg("PARSE: Pipeline processing failed") return Statement{ - SQL: sql, - Type: StatementUnsupported, - Error: err.Error(), + SQL: sql, + Type: StatementUnsupported, + Error: err.Error(), + TranspileErr: ctx.Output.TranspileErr, } } @@ -244,9 +208,10 @@ func ParseStatementsWithSchema(sql string, schemaLookup SchemaLookupFunc) []Stat Str("sql_prefix", truncateSQLForLog(sql, 80)). Msg("PARSE: Pipeline processing failed") return []Statement{{ - SQL: sql, - Type: StatementUnsupported, - Error: err.Error(), + SQL: sql, + Type: StatementUnsupported, + Error: err.Error(), + TranspileErr: ctx.Output.TranspileErr, }} } @@ -319,16 +284,6 @@ func NormalizeSQLForSQLite(sql string) string { return sql } -// extractTableName extracts table name using a regex pattern -func extractTableName(sql, pattern string) string { - re := regexp.MustCompile(pattern) - matches := re.FindStringSubmatch(sql) - if len(matches) > 1 { - return matches[1] - } - return "" -} - // ExtractConsistencyHint extracts consistency hint from SQL comment // Example: /*+ CONSISTENCY(QUORUM) */ SELECT * FROM users func ExtractConsistencyHint(sql string) (ConsistencyLevel, bool) { diff --git a/protocol/parser_vitess.go b/protocol/parser_vitess.go deleted file mode 100644 index d9ec66e3..00000000 --- a/protocol/parser_vitess.go +++ /dev/null @@ -1,887 +0,0 @@ -package protocol - -import ( - "database/sql" - "fmt" - "regexp" - "strconv" - "strings" - "sync" - - _ "github.com/mattn/go-sqlite3" - "vitess.io/vitess/go/vt/sqlparser" -) - -// SQLiteDriverName is the custom SQLite driver name with REGEXP support. -// This must match the driver name registered in db/sqlite_driver.go. -// The driver will be registered when the db package is imported elsewhere in the application. -const SQLiteDriverName = "sqlite3_marmot" - -// Global parser instance (reused for efficiency) -var vitessParser *sqlparser.Parser - -// SQLite syntax validator (in-memory, persistent) -var ( - sqliteSyntaxChecker *sql.DB - sqliteCheckerMu sync.Mutex -) - -func init() { - var err error - vitessParser, err = sqlparser.New(sqlparser.Options{}) - if err != nil { - panic("failed to initialize Vitess parser: " + err.Error()) - } - - // Initialize in-memory SQLite for syntax validation - // Try custom driver first (with REGEXP support), fall back to sqlite3 - sqliteSyntaxChecker, err = sql.Open(SQLiteDriverName, ":memory:") - if err != nil { - // Fallback to standard sqlite3 driver if custom driver not registered - sqliteSyntaxChecker, err = sql.Open("sqlite3", ":memory:") - if err != nil { - panic("failed to initialize SQLite syntax checker: " + err.Error()) - } - } -} - -// transpileMySQLToSQLite converts MySQL-specific syntax to SQLite-compatible syntax -// This handles all known syntax differences between MySQL DML and SQLite -func transpileMySQLToSQLite(sql string) string { - // === INSERT statements === - - // INSERT IGNORE INTO -> INSERT OR IGNORE INTO - re := regexp.MustCompile(`(?i)\bINSERT\s+IGNORE\s+INTO\b`) - sql = re.ReplaceAllString(sql, "INSERT OR IGNORE INTO") - - // REPLACE INTO -> INSERT OR REPLACE INTO - // Only match REPLACE at start of statement (not "INSERT OR REPLACE") - re = regexp.MustCompile(`(?i)^\s*REPLACE\s+INTO\b`) - sql = re.ReplaceAllString(sql, "INSERT OR REPLACE INTO") - - // === Index hints (all statement types) === - - // FORCE INDEX(name) or FORCE INDEX(`name`) - re = regexp.MustCompile(`(?i)\s+FORCE\s+INDEX\s*\([^)]+\)`) - sql = re.ReplaceAllString(sql, "") - - // USE INDEX(name) - re = regexp.MustCompile(`(?i)\s+USE\s+INDEX\s*\([^)]+\)`) - sql = re.ReplaceAllString(sql, "") - - // IGNORE INDEX(name) - re = regexp.MustCompile(`(?i)\s+IGNORE\s+INDEX\s*\([^)]+\)`) - sql = re.ReplaceAllString(sql, "") - - // === SELECT hints and modifiers === - - // STRAIGHT_JOIN - re = regexp.MustCompile(`(?i)\bSTRAIGHT_JOIN\b`) - sql = re.ReplaceAllString(sql, "JOIN") - - // SQL_CALC_FOUND_ROWS - re = regexp.MustCompile(`(?i)\bSQL_CALC_FOUND_ROWS\b`) - sql = re.ReplaceAllString(sql, "") - - // SQL_CACHE / SQL_NO_CACHE - re = regexp.MustCompile(`(?i)\bSQL_(NO_)?CACHE\b`) - sql = re.ReplaceAllString(sql, "") - - // SQL_SMALL_RESULT / SQL_BIG_RESULT / SQL_BUFFER_RESULT - re = regexp.MustCompile(`(?i)\bSQL_(SMALL|BIG|BUFFER)_RESULT\b`) - sql = re.ReplaceAllString(sql, "") - - // HIGH_PRIORITY / LOW_PRIORITY / DELAYED - re = regexp.MustCompile(`(?i)\b(HIGH|LOW)_PRIORITY\b`) - sql = re.ReplaceAllString(sql, "") - re = regexp.MustCompile(`(?i)\bDELAYED\b`) - sql = re.ReplaceAllString(sql, "") - - // === Locking clauses === - - // FOR UPDATE - re = regexp.MustCompile(`(?i)\s+FOR\s+UPDATE(\s+|$)`) - sql = re.ReplaceAllString(sql, " ") - - // LOCK IN SHARE MODE - re = regexp.MustCompile(`(?i)\s+LOCK\s+IN\s+SHARE\s+MODE(\s+|$)`) - sql = re.ReplaceAllString(sql, " ") - - // === LIMIT clause === - - // MySQL: LIMIT offset, count → SQLite: LIMIT count OFFSET offset - re = regexp.MustCompile(`(?i)\bLIMIT\s+(\d+)\s*,\s*(\d+)`) - sql = re.ReplaceAllStringFunc(sql, func(match string) string { - parts := regexp.MustCompile(`(?i)\bLIMIT\s+(\d+)\s*,\s*(\d+)`).FindStringSubmatch(match) - if len(parts) == 3 { - return fmt.Sprintf("LIMIT %s OFFSET %s", parts[2], parts[1]) - } - return match - }) - - // Clean up multiple spaces left by removals - re = regexp.MustCompile(`\s+`) - sql = re.ReplaceAllString(sql, " ") - - // Clean up space before closing parenthesis or comma - sql = strings.TrimSpace(sql) - - return sql -} - -// ValidateSQLiteSyntax checks if a query is valid SQLite syntax -// Returns nil if valid, error if invalid (only for true syntax errors) -func ValidateSQLiteSyntax(query string) error { - sqliteCheckerMu.Lock() - defer sqliteCheckerMu.Unlock() - - // Translate MySQL-specific syntax to SQLite for validation - queryToValidate := transpileMySQLToSQLite(query) - - stmt, err := sqliteSyntaxChecker.Prepare(queryToValidate) - if err != nil { - errStr := err.Error() - // Ignore "no such table/column" errors - those are schema errors, not syntax errors - // We only want to catch true syntax errors - if strings.Contains(errStr, "no such table") || - strings.Contains(errStr, "no such column") || - strings.Contains(errStr, "no such index") || - strings.Contains(errStr, "no such view") || - strings.Contains(errStr, "no such trigger") { - return nil // Schema error, not syntax error - allow it - } - return err - } - stmt.Close() - return nil -} - -// shouldValidateWithSQLite returns true if we should validate this SQL against SQLite -// Only validate DML statements - these are what get replicated and executed on SQLite -func shouldValidateWithSQLite(sql string) bool { - // Strip leading comments - upper := strings.ToUpper(strings.TrimSpace(sql)) - for strings.HasPrefix(upper, "/*") { - if idx := strings.Index(upper, "*/"); idx >= 0 { - upper = strings.TrimSpace(upper[idx+2:]) - } else { - break - } - } - for strings.HasPrefix(upper, "--") || strings.HasPrefix(upper, "#") { - if idx := strings.Index(upper, "\n"); idx >= 0 { - upper = strings.TrimSpace(upper[idx+1:]) - } else { - return false - } - } - - // Only validate core DML that gets replicated - return strings.HasPrefix(upper, "INSERT ") || - strings.HasPrefix(upper, "UPDATE ") || - strings.HasPrefix(upper, "DELETE ") || - strings.HasPrefix(upper, "SELECT ") || - strings.HasPrefix(upper, "REPLACE ") -} - -// ParseStatementVitess uses Vitess sqlparser for accurate MySQL parsing -// First validates syntax against SQLite, then extracts metadata via Vitess -func ParseStatementVitess(sql string) Statement { - sql = strings.TrimSpace(sql) - - stmt := Statement{SQL: sql} - - // Step 1: Validate DML syntax against SQLite first - // Only validate INSERT/UPDATE/DELETE/SELECT - these are the replicated statements - // Skip DDL and other statement types as they have MySQL/SQLite syntax differences - if shouldValidateWithSQLite(sql) { - if err := ValidateSQLiteSyntax(sql); err != nil { - // Invalid SQLite syntax - reject early with clear error - stmt.Type = StatementUnsupported - stmt.Error = "invalid SQLite syntax: " + err.Error() - return stmt - } - } - - // Step 2: Continue with Vitess parsing to extract metadata (table names, type, etc.) - - // Check for savepoint statements first (Vitess doesn't parse these properly) - if savepointPattern.MatchString(sql) || releaseSavepointPattern.MatchString(sql) { - stmt.Type = StatementSavepoint - return stmt - } - - // Check for SET TRANSACTION (should be treated as transaction control) - if setTransactionPattern.MatchString(sql) { - stmt.Type = StatementBegin - return stmt - } - - // Check for XA transactions (Vitess doesn't parse these) - if xaStartPattern.MatchString(sql) || xaEndPattern.MatchString(sql) || - xaPreparePattern.MatchString(sql) || xaCommitPattern.MatchString(sql) || - xaRollbackPattern.MatchString(sql) || xaRecoverPattern.MatchString(sql) { - stmt.Type = StatementXA - return stmt - } - - // Check for lock instance statements (Vitess doesn't parse these) - if lockInstancePattern.MatchString(sql) || unlockInstancePattern.MatchString(sql) { - stmt.Type = StatementLock - return stmt - } - - // Check for administrative statements (Vitess doesn't parse these) - if installPluginPattern.MatchString(sql) || uninstallPluginPattern.MatchString(sql) || - installComponentPattern.MatchString(sql) || uninstallComponentPattern.MatchString(sql) { - stmt.Type = StatementAdmin - return stmt - } - - // Check for LOAD XML (Vitess might not handle this) - if loadXMLPattern.MatchString(sql) { - stmt.Type = StatementLoadData - return stmt - } - - // Check for INSERT variants that Vitess might not parse correctly - insertDelayedPattern := regexp.MustCompile(`(?i)^\s*INSERT\s+(DELAYED|LOW_PRIORITY|HIGH_PRIORITY)\s+`) - if insertDelayedPattern.MatchString(sql) { - stmt.Type = StatementInsert - // Try to extract table name - tablePattern := regexp.MustCompile(`(?i)INSERT\s+(DELAYED|LOW_PRIORITY|HIGH_PRIORITY)\s+INTO\s+(\w+)`) - if matches := tablePattern.FindStringSubmatch(sql); len(matches) > 2 { - stmt.TableName = matches[2] - } - return stmt - } - - // Check for SQLite-specific INSERT syntax (from transpiled MySQL queries) - // INSERT OR IGNORE INTO ... or INSERT OR REPLACE INTO ... - insertOrPattern := regexp.MustCompile(`(?i)^\s*INSERT\s+OR\s+(IGNORE|REPLACE)\s+INTO\s+(\w+)`) - if matches := insertOrPattern.FindStringSubmatch(sql); len(matches) > 0 { - if len(matches) > 1 && strings.ToUpper(matches[1]) == "REPLACE" { - stmt.Type = StatementReplace - } else { - stmt.Type = StatementInsert - } - // Extract table name - if len(matches) > 2 { - stmt.TableName = matches[2] - } - return stmt - } - - // Check for DDL statements that Vitess might not handle properly - // Triggers - if createTriggerPattern.MatchString(sql) || dropTriggerPattern.MatchString(sql) { - stmt.Type = StatementDDL - return stmt - } - - // Procedures - if createProcedurePattern.MatchString(sql) || dropProcedurePattern.MatchString(sql) || - alterProcedurePattern.MatchString(sql) { - stmt.Type = StatementDDL - return stmt - } - - // Functions - if createFunctionPattern.MatchString(sql) || dropFunctionPattern.MatchString(sql) || - alterFunctionPattern.MatchString(sql) { - stmt.Type = StatementDDL - return stmt - } - - // Events - if createEventPattern.MatchString(sql) || dropEventPattern.MatchString(sql) || - alterEventPattern.MatchString(sql) { - stmt.Type = StatementDDL - return stmt - } - - // Tablespaces - if createTablespacePattern.MatchString(sql) || dropTablespacePattern.MatchString(sql) || - alterTablespacePattern.MatchString(sql) { - stmt.Type = StatementDDL - return stmt - } - - // Logfile Groups - if createLogfileGroupPattern.MatchString(sql) || dropLogfileGroupPattern.MatchString(sql) || - alterLogfileGroupPattern.MatchString(sql) { - stmt.Type = StatementDDL - return stmt - } - - // Servers - if createServerPattern.MatchString(sql) || dropServerPattern.MatchString(sql) || - alterServerPattern.MatchString(sql) { - stmt.Type = StatementDDL - return stmt - } - - // Spatial Reference Systems - if createSpatialRefPattern.MatchString(sql) || dropSpatialRefPattern.MatchString(sql) { - stmt.Type = StatementDDL - return stmt - } - - // Resource Groups - if createResourceGroupPattern.MatchString(sql) || dropResourceGroupPattern.MatchString(sql) || - alterResourceGroupPattern.MatchString(sql) { - stmt.Type = StatementDDL - return stmt - } - - // Drop Index (Vitess might not handle this) - if dropIndexPattern.MatchString(sql) { - stmt.Type = StatementDDL - return stmt - } - - // Vector Index DDL (sqlite-vec extension syntax not parsed by Vitess) - if createVectorIndexPattern.MatchString(sql) { - stmt.Type = StatementCreateVectorIndex - // Capture: (1) index name, (2) table name, (3) column name - vecIdxPattern := regexp.MustCompile(`(?i)CREATE\s+VECTOR\s+INDEX\s+(?:IF\s+NOT\s+EXISTS\s+)?(\w+)\s+ON\s+(\w+)\s*\(\s*(\w+)\s*\)`) - if m := vecIdxPattern.FindStringSubmatch(sql); len(m) > 3 { - stmt.VectorIndexName = m[1] - stmt.TableName = m[2] - stmt.VectorColumnName = m[3] - } - // Parse DIM and METRIC from the main syntax (before optional WITH clause). - if m := regexp.MustCompile(`(?i)\bDIM\s+(\d+)`).FindStringSubmatch(sql); len(m) > 1 { - stmt.VectorDim = parseDecimalInt(m[1]) - } - if m := regexp.MustCompile(`(?i)\bMETRIC\s+(\w+)`).FindStringSubmatch(sql); len(m) > 1 { - stmt.VectorMetric = strings.ToLower(m[1]) - } - // WITH clause overrides main-syntax values for all keys it mentions. - withPattern := regexp.MustCompile(`(?i)WITH\s*\(([^)]+)\)`) - if wm := withPattern.FindStringSubmatch(sql); len(wm) > 1 { - parseVectorWithClause(&stmt, wm[1]) - } - return stmt - } - if dropVectorIndexPattern.MatchString(sql) { - stmt.Type = StatementDropVectorIndex - // ON is optional — index names are unique within a database. - vecDropPattern := regexp.MustCompile(`(?i)DROP\s+VECTOR\s+INDEX\s+(?:IF\s+EXISTS\s+)?(\w+)(?:\s+ON\s+(\w+))?`) - if m := vecDropPattern.FindStringSubmatch(sql); len(m) > 1 { - stmt.VectorIndexName = m[1] - if len(m) > 2 { - stmt.TableName = m[2] - } - } - return stmt - } - - // REINDEX VECTOR — SQLite-specific DDL (design §8.3). Vitess cannot - // parse this, so we tokenize the statement ourselves rather than fall - // through to Vitess. Tokenization preserves the hard "no regex for DDL - // parsing" rule: we walk whitespace-separated tokens and validate shape - // explicitly. `REINDEX TABLE ` and plain `REINDEX` fall through - // unchanged — only the VECTOR variant is intercepted here. - if name, ok, rvErr := parseReindexVectorDDL(sql); ok { - if rvErr != nil { - stmt.Type = StatementUnsupported - stmt.Error = rvErr.Error() - return stmt - } - stmt.Type = StatementReindexVectorIndex - stmt.VectorIndexName = name - return stmt - } - - // Check for DCL statements (Vitess doesn't parse these) - if createUserPattern.MatchString(sql) || dropUserPattern.MatchString(sql) || - alterUserPattern.MatchString(sql) || renameUserPattern.MatchString(sql) || - setPasswordPattern.MatchString(sql) { - stmt.Type = StatementDCL - return stmt - } - if grantPattern.MatchString(sql) || revokePattern.MatchString(sql) { - stmt.Type = StatementDCL - return stmt - } - if createRolePattern.MatchString(sql) || dropRolePattern.MatchString(sql) || - setRolePattern.MatchString(sql) || setDefaultRolePattern.MatchString(sql) { - stmt.Type = StatementDCL - return stmt - } - - // Parse with Vitess - parsed, err := vitessParser.Parse(sql) - if err != nil { - // Fallback: check regex patterns for statements Vitess can't parse - fallback := checkRegexFallback(sql) - if fallback.Type != StatementSelect { - return fallback - } - // Unknown statement - validate against SQLite to check if it's valid SQL - if validateErr := ValidateSQLiteSyntax(sql); validateErr != nil { - stmt.Type = StatementUnsupported - stmt.Error = "invalid SQL syntax: " + validateErr.Error() - return stmt - } - // Valid SQL but unknown type - treat as SELECT (safe default) - stmt.Type = StatementSelect - return stmt - } - - // Map Vitess AST to StatementCode and extract metadata FIRST - switch parsed := parsed.(type) { - case *sqlparser.Insert: - if parsed.Action == sqlparser.ReplaceAct { - stmt.Type = StatementReplace - } else { - stmt.Type = StatementInsert - } - // Extract table name from AliasedTableExpr - if parsed.Table != nil { - if tn, ok := parsed.Table.Expr.(sqlparser.TableName); ok { - stmt.TableName = tn.Name.String() - if tn.Qualifier.NotEmpty() { - stmt.Database = tn.Qualifier.String() - } - } - } - - case *sqlparser.Update: - stmt.Type = StatementUpdate - if len(parsed.TableExprs) > 0 { - if aliased, ok := parsed.TableExprs[0].(*sqlparser.AliasedTableExpr); ok { - if tn, ok := aliased.Expr.(sqlparser.TableName); ok { - stmt.TableName = tn.Name.String() - if tn.Qualifier.NotEmpty() { - stmt.Database = tn.Qualifier.String() - } - } - } - } - - case *sqlparser.Delete: - stmt.Type = StatementDelete - if len(parsed.TableExprs) > 0 { - if aliased, ok := parsed.TableExprs[0].(*sqlparser.AliasedTableExpr); ok { - if tn, ok := aliased.Expr.(sqlparser.TableName); ok { - stmt.TableName = tn.Name.String() - if tn.Qualifier.NotEmpty() { - stmt.Database = tn.Qualifier.String() - } - } - } - } - - case *sqlparser.Select: - stmt.Type = StatementSelect - // Check if this is an INFORMATION_SCHEMA query - if isInformationSchemaQuery(parsed) { - stmt.Type = StatementInformationSchema - } - // Extract database from CTEs first (WITH clause) - if parsed.With != nil { - for _, cte := range parsed.With.CTEs { - if cteSelect, ok := cte.Subquery.(*sqlparser.Select); ok { - if len(cteSelect.From) > 0 { - db, table := extractDatabaseFromTableExpr(cteSelect.From[0]) - if db != "" { - stmt.Database = db - if table != "" { - stmt.TableName = table - } - break // Use first database found - } - } - } - } - } - // If no database found in CTE, extract from main FROM clause - if stmt.Database == "" && len(parsed.From) > 0 { - db, table := extractDatabaseFromTableExpr(parsed.From[0]) - if db != "" { - stmt.Database = db - } - if table != "" { - stmt.TableName = table - } - } - - // DDL statements - now using specific types instead of generic DDL - case *sqlparser.CreateTable: - stmt.Type = StatementDDL - stmt.TableName = parsed.Table.Name.String() - if parsed.Table.Qualifier.NotEmpty() { - stmt.Database = parsed.Table.Qualifier.String() - } - - case *sqlparser.AlterTable: - stmt.Type = StatementDDL - stmt.TableName = parsed.Table.Name.String() - if parsed.Table.Qualifier.NotEmpty() { - stmt.Database = parsed.Table.Qualifier.String() - } - - case *sqlparser.DropTable: - stmt.Type = StatementDDL - if len(parsed.FromTables) > 0 { - stmt.TableName = parsed.FromTables[0].Name.String() - if parsed.FromTables[0].Qualifier.NotEmpty() { - stmt.Database = parsed.FromTables[0].Qualifier.String() - } - } - - case *sqlparser.CreateDatabase: - stmt.Type = StatementCreateDatabase - stmt.Database = parsed.DBName.String() - - case *sqlparser.DropDatabase: - stmt.Type = StatementDropDatabase - stmt.Database = parsed.DBName.String() - - case *sqlparser.AlterDatabase: - stmt.Type = StatementDDL - stmt.Database = parsed.DBName.String() - - // Handle RENAME TABLE specifically - case *sqlparser.RenameTable: - stmt.Type = StatementDDL - if len(parsed.TablePairs) > 0 { - stmt.TableName = parsed.TablePairs[0].FromTable.Name.String() - if parsed.TablePairs[0].FromTable.Qualifier.NotEmpty() { - stmt.Database = parsed.TablePairs[0].FromTable.Qualifier.String() - } - } - - // Handle other DDL types that might still use the DDLStatement interface - case sqlparser.DDLStatement: - // Generic DDL handling for types we don't explicitly handle above - stmt.Type = StatementDDL - table := parsed.GetTable() - if !table.IsEmpty() { - stmt.TableName = table.Name.String() - if table.Qualifier.NotEmpty() { - stmt.Database = table.Qualifier.String() - } - } - - case *sqlparser.Show: - // Show.Internal contains the actual show type - if showBasic, ok := parsed.Internal.(*sqlparser.ShowBasic); ok { - // Check the command type - switch showBasic.Command { - case sqlparser.Database: - stmt.Type = StatementShowDatabases - case sqlparser.Table: - stmt.Type = StatementShowTables - // Extract database name if specified - if showBasic.DbName.NotEmpty() { - stmt.Database = showBasic.DbName.String() - } - // Extract LIKE filter if specified - if showBasic.Filter != nil && showBasic.Filter.Like != "" { - stmt.ShowFilter = showBasic.Filter.Like - } - case sqlparser.Column: - stmt.Type = StatementShowColumns - // Extract table name - if !showBasic.Tbl.IsEmpty() { - stmt.TableName = showBasic.Tbl.Name.String() - if showBasic.Tbl.Qualifier.NotEmpty() { - stmt.Database = showBasic.Tbl.Qualifier.String() - } - } - case sqlparser.Index: - stmt.Type = StatementShowIndexes - // Extract table name - if !showBasic.Tbl.IsEmpty() { - stmt.TableName = showBasic.Tbl.Name.String() - if showBasic.Tbl.Qualifier.NotEmpty() { - stmt.Database = showBasic.Tbl.Qualifier.String() - } - } - case sqlparser.TableStatus: - stmt.Type = StatementShowTableStatus - // Extract database name and filter - if showBasic.DbName.NotEmpty() { - stmt.Database = showBasic.DbName.String() - } - // Extract table name from LIKE filter if present - if showBasic.Filter != nil && showBasic.Filter.Like != "" { - stmt.TableName = showBasic.Filter.Like - } - default: - // Other SHOW commands treated as SELECT - stmt.Type = StatementSelect - } - } else if showCreate, ok := parsed.Internal.(*sqlparser.ShowCreate); ok { - // SHOW CREATE TABLE - stmt.Type = StatementShowCreateTable - if !showCreate.Op.IsEmpty() { - stmt.TableName = showCreate.Op.Name.String() - if showCreate.Op.Qualifier.NotEmpty() { - stmt.Database = showCreate.Op.Qualifier.String() - } - } - } else { - // Other SHOW variants treated as SELECT - stmt.Type = StatementSelect - } - - case *sqlparser.Use: - stmt.Type = StatementUseDatabase - stmt.Database = parsed.DBName.String() - - case *sqlparser.Begin: - stmt.Type = StatementBegin - - case *sqlparser.Commit: - stmt.Type = StatementCommit - - case *sqlparser.Rollback: - stmt.Type = StatementRollback - - case *sqlparser.Savepoint: - stmt.Type = StatementSavepoint - - case *sqlparser.Set: - stmt.Type = StatementSet - - case *sqlparser.Load: - stmt.Type = StatementLoadData - - case *sqlparser.OtherAdmin: - stmt.Type = StatementAdmin - - case *sqlparser.LockTables: - stmt.Type = StatementLock - - case *sqlparser.UnlockTables: - stmt.Type = StatementLock - - default: - // Unknown type - safe default - stmt.Type = StatementSelect - } - - // Strip database qualifiers from AST if any database was extracted - // This ensures that "SELECT * FROM db.table" becomes "SELECT * FROM table" - // since we route queries to the correct database file using stmt.Database - if stmt.Database != "" { - stripDatabaseQualifiers(parsed) - stmt.SQL = sqlparser.String(parsed) - } - - // Transpile MySQL-specific syntax to SQLite-compatible syntax - // This must be done AFTER sqlparser.String() to fix the rendered SQL - stmt.SQL = transpileMySQLToSQLite(stmt.SQL) - - return stmt -} - -// Vitess token constants used by parseReindexVectorDDL. These match the -// yacc-generated values in vitess.io/vitess/go/vt/sqlparser/sql.go. -// Declared here to avoid exporting internals; the values are stable across -// Vitess minor versions since they come from the generated grammar. -const ( - vitessTokenID = 57440 // unquoted or backtick-quoted identifier - vitessTokenVECTOR = 57730 // the VECTOR keyword -) - -// parseReindexVectorDDL detects and parses `REINDEX VECTOR ` statements -// using the Vitess tokenizer so that SQL comments, quoted identifiers, and -// string literals are handled correctly. No regex, no string splitting. -// -// Returns: -// -// (name, true, nil) — valid `REINDEX VECTOR ` (optional trailing ';') -// ("", true, err) — starts with `REINDEX VECTOR` but shape is invalid -// (missing name, extra tokens, etc.) — a typed error -// so the caller can surface it as Unsupported. -// ("", false, nil) — not a REINDEX VECTOR statement (e.g. plain REINDEX, -// REINDEX TABLE, or anything else). Caller falls -// through to the normal parse path. -// -// The index name must also pass identifier shape validation at manager level; -// this helper only enforces syntactic shape — the two keywords followed by -// exactly one identifier token. -func parseReindexVectorDDL(sql string) (name string, matched bool, err error) { - tkn := vitessParser.NewStringTokenizer(sql) - - // Consume non-significant tokens (comments) until the first keyword. - tok1, val1 := scanSkipComments(tkn) - if tok1 == 0 { - return "", false, nil - } - // REINDEX is not a MySQL keyword — Vitess lexes it as a plain ID. - if tok1 != vitessTokenID || !strings.EqualFold(val1, "REINDEX") { - return "", false, nil - } - - tok2, _ := scanSkipComments(tkn) - if tok2 != vitessTokenVECTOR { - // `REINDEX TABLE`, `REINDEX idx`, or just `REINDEX` — not ours. - return "", false, nil - } - - // From here we own the statement — shape errors surface as Unsupported. - tok3, val3 := scanSkipComments(tkn) - if tok3 == 0 || tok3 == ';' { - return "", true, fmt.Errorf("MARMOT-VEC-015: REINDEX VECTOR requires an index name") - } - if tok3 != vitessTokenID { - return "", true, fmt.Errorf("MARMOT-VEC-015: REINDEX VECTOR index name must be an identifier, got token %d", tok3) - } - - // Expect end-of-input or a trailing semicolon — nothing else. - tok4, _ := scanSkipComments(tkn) - if tok4 != 0 && tok4 != ';' { - return "", true, fmt.Errorf("MARMOT-VEC-015: REINDEX VECTOR takes exactly one index name; unexpected trailing token") - } - return val3, true, nil -} - -// scanSkipComments calls tkn.Scan() in a loop, skipping comment tokens -// (Vitess emits them as token type 57449 / COMMENT). Returns (0, "") at -// end-of-input. -func scanSkipComments(tkn *sqlparser.Tokenizer) (int, string) { - for { - tok, val := tkn.Scan() - if tok == 0 { - return 0, "" - } - // Vitess comment token = 57449; skip silently. - if tok == 57449 { - continue - } - return tok, val - } -} - -// parseVectorWithClause parses key=value pairs from a WITH(...) clause into Statement vector fields. -// Recognized keys: metric, dim, col/column, nlist, nprobe, max_norm. -func parseVectorWithClause(stmt *Statement, clause string) { - kvPattern := regexp.MustCompile(`(?i)(\w+)\s*=\s*'?([^',)]+)'?`) - for _, m := range kvPattern.FindAllStringSubmatch(clause, -1) { - if len(m) < 3 { - continue - } - key := strings.ToLower(strings.TrimSpace(m[1])) - val := strings.TrimSpace(m[2]) - switch key { - case "metric": - stmt.VectorMetric = strings.ToLower(val) - case "dim": - stmt.VectorDim = parseDecimalInt(val) - case "col", "column": - stmt.VectorColumnName = val - case "nlist": - stmt.VectorNlist = parseDecimalInt(val) - case "nprobe": - stmt.VectorNprobe = parseDecimalInt(val) - case "max_norm": - if f, err := strconv.ParseFloat(val, 32); err == nil { - stmt.VectorMaxNorm = float32(f) - } - } - } -} - -// parseDecimalInt converts a decimal digit string to int, returning 0 on any error. -func parseDecimalInt(s string) int { - n, _ := strconv.Atoi(strings.TrimSpace(s)) - return n -} - -// isInformationSchemaQuery checks if a SELECT statement queries INFORMATION_SCHEMA -func isInformationSchemaQuery(sel *sqlparser.Select) bool { - // Check all table expressions in FROM clause - for _, tableExpr := range sel.From { - if aliased, ok := tableExpr.(*sqlparser.AliasedTableExpr); ok { - if tableName, ok := aliased.Expr.(sqlparser.TableName); ok { - // Check if database qualifier is INFORMATION_SCHEMA - if tableName.Qualifier.NotEmpty() { - qualifier := strings.ToUpper(tableName.Qualifier.String()) - if qualifier == "INFORMATION_SCHEMA" { - return true - } - } - } - } - } - return false -} - -// checkRegexFallback uses regex patterns to detect statement types when Vitess can't parse -func checkRegexFallback(sql string) Statement { - stmt := Statement{SQL: sql} - - // Check for RENAME TABLE - if renameTablePattern.MatchString(sql) { - stmt.Type = StatementDDL - stmt.TableName = extractTableName(sql, `(?i)RENAME\s+TABLE\s+(\w+)`) - return stmt - } - - // This fallback should only be called for statements Vitess can't parse - // Default to SELECT - stmt.Type = StatementSelect - return stmt -} - -// extractDatabaseFromTableExpr recursively extracts database and table names from any TableExpr -// This handles simple tables, JOINs, subqueries, CTEs, etc. -// Returns the first database qualifier found (for routing purposes) -func extractDatabaseFromTableExpr(expr sqlparser.TableExpr) (database string, table string) { - switch e := expr.(type) { - case *sqlparser.AliasedTableExpr: - // Simple table or subquery - if tn, ok := e.Expr.(sqlparser.TableName); ok { - table = tn.Name.String() - if tn.Qualifier.NotEmpty() { - database = tn.Qualifier.String() - } - } - case *sqlparser.JoinTableExpr: - // JOIN expression - extract from left side (primary table) - database, table = extractDatabaseFromTableExpr(e.LeftExpr) - // If no database found on left, try right side - if database == "" { - rightDB, rightTable := extractDatabaseFromTableExpr(e.RightExpr) - if rightDB != "" { - database = rightDB - if table == "" { - table = rightTable - } - } - } - case *sqlparser.ParenTableExpr: - // Parenthesized table expression - for _, t := range e.Exprs { - db, tbl := extractDatabaseFromTableExpr(t) - if db != "" { - return db, tbl - } - } - } - return database, table -} - -// stripDatabaseQualifiers removes database qualifiers from all table references in the AST -// This modifies the AST in-place to convert "db.table" to "table" -func stripDatabaseQualifiers(stmt sqlparser.Statement) { - // Use Vitess's Rewrite function to traverse and modify AST - sqlparser.Rewrite(stmt, func(cursor *sqlparser.Cursor) bool { - switch n := cursor.Node().(type) { - case sqlparser.TableName: - // Clear the qualifier (database name) by creating a new TableName - newTableName := sqlparser.TableName{ - Name: n.Name, - Qualifier: sqlparser.NewIdentifierCS(""), - } - cursor.Replace(newTableName) - } - return true - }, nil) -} diff --git a/protocol/parser_vitess_test.go b/protocol/parser_vitess_test.go index df77343d..35823c92 100644 --- a/protocol/parser_vitess_test.go +++ b/protocol/parser_vitess_test.go @@ -6,39 +6,6 @@ import ( "testing" ) -// TestParseReindexVectorDDL_Positive covers the happy path plus common -// acceptable formatting (trailing semicolon, extra whitespace, mixed case). -// Locks the contract described in design §8.3: `REINDEX VECTOR ` is -// a SQLite-specific DDL routed to VectorIndexManager.ReindexIndex. -func TestParseReindexVectorDDL_Positive(t *testing.T) { - t.Parallel() - - cases := []struct { - sql string - want string - }{ - {"REINDEX VECTOR embeddings", "embeddings"}, - {"reindex vector embeddings", "embeddings"}, - {" REINDEX VECTOR my_idx ", "my_idx"}, - {"REINDEX VECTOR embeddings;", "embeddings"}, - {"REINDEX VECTOR embeddings ;", "embeddings"}, - {"/* rebuild */ REINDEX VECTOR embeddings", "embeddings"}, - {"REINDEX VECTOR `my_idx`", "my_idx"}, - } - for _, tc := range cases { - tc := tc - t.Run(tc.sql, func(t *testing.T) { - stmt := ParseStatementVitess(tc.sql) - if stmt.Type != StatementReindexVectorIndex { - t.Fatalf("Type = %v, want StatementReindexVectorIndex (err=%q)", stmt.Type, stmt.Error) - } - if stmt.VectorIndexName != tc.want { - t.Fatalf("VectorIndexName = %q, want %q", stmt.VectorIndexName, tc.want) - } - }) - } -} - func TestParseVectorIndexDDL_PipelineMetadata(t *testing.T) { t.Parallel() @@ -99,57 +66,6 @@ func TestParseVectorQueryDoesNotExtractPlannerLiterals(t *testing.T) { } } -// TestParseReindexVectorDDL_Negative covers rejection paths and passthrough. -// `REINDEX TABLE foo` and plain `REINDEX` must NOT be claimed by the -// VectorReindex dispatcher — they are legitimate SQLite statements that -// belong to the standard execution path. -func TestParseReindexVectorDDL_Negative(t *testing.T) { - t.Parallel() - - t.Run("missing index name", func(t *testing.T) { - stmt := ParseStatementVitess("REINDEX VECTOR") - if stmt.Type != StatementUnsupported { - t.Fatalf("Type = %v, want StatementUnsupported", stmt.Type) - } - if !strings.Contains(stmt.Error, "MARMOT-VEC-015") { - t.Fatalf("Error = %q, want MARMOT-VEC-015 prefix", stmt.Error) - } - }) - - t.Run("extra tokens", func(t *testing.T) { - stmt := ParseStatementVitess("REINDEX VECTOR foo bar") - if stmt.Type != StatementUnsupported { - t.Fatalf("Type = %v, want StatementUnsupported", stmt.Type) - } - if !strings.Contains(stmt.Error, "MARMOT-VEC-015") { - t.Fatalf("Error = %q, want MARMOT-VEC-015 prefix", stmt.Error) - } - }) - - t.Run("REINDEX TABLE is not claimed", func(t *testing.T) { - // `REINDEX TABLE foo` is a legitimate SQLite REINDEX — must NOT be - // intercepted as vector reindex. - stmt := ParseStatementVitess("REINDEX TABLE foo") - if stmt.Type == StatementReindexVectorIndex { - t.Fatalf("REINDEX TABLE foo must not be classified as StatementReindexVectorIndex") - } - }) - - t.Run("plain REINDEX is not claimed", func(t *testing.T) { - stmt := ParseStatementVitess("REINDEX") - if stmt.Type == StatementReindexVectorIndex { - t.Fatalf("plain REINDEX must not be classified as StatementReindexVectorIndex") - } - }) - - t.Run("unrelated statement is not claimed", func(t *testing.T) { - stmt := ParseStatementVitess("SELECT 1") - if stmt.Type == StatementReindexVectorIndex { - t.Fatalf("SELECT must not be classified as StatementReindexVectorIndex") - } - }) -} - // TestReindexVectorIndex_IsMutation guards the routing contract: REINDEX // must flow through the same mutation path as other vector DDL so it's // rejected during shutdown drain and replicated as vector-control metadata. diff --git a/protocol/query/context.go b/protocol/query/context.go index 87f3ab97..63897fbb 100644 --- a/protocol/query/context.go +++ b/protocol/query/context.go @@ -106,6 +106,13 @@ type QueryOutput struct { Database string IsValid bool ValidationErr error + + // TranspileErr holds the error transpilation failed with, typed, when the + // failure came from a transformation rule rather than from parsing. The + // protocol layer needs the type to tell a rule's rejection (which carries + // its own MySQL error code) from a Vitess syntax error, without matching + // on message text. nil when parsing failed or nothing failed. + TranspileErr error } // TranspiledStatement represents a single transpiled SQL statement with parameters. @@ -159,7 +166,7 @@ type QueryContext struct { Input QueryInput Output QueryOutput MySQLState *MySQLParseState // nil for SQLite dialect - SchemaLookup func(table string) string + SchemaLookup func(table string) *transform.SchemaInfo SchemaProvider transform.SchemaProvider // SkipTranspilation bypasses MySQL→SQLite transpilation when true. diff --git a/protocol/query/pipeline.go b/protocol/query/pipeline.go index abc10f54..0aac5abb 100644 --- a/protocol/query/pipeline.go +++ b/protocol/query/pipeline.go @@ -84,6 +84,9 @@ func (p *Pipeline) Process(ctx *QueryContext) error { // Transpile (handles database qualifier stripping and literal extraction) if err := p.transpiler.Transpile(ctx); err != nil { log.Debug().Err(err).Str("sql", ctx.Input.SQL).Msg("Transpile failed") + // Record it typed and separately from a parse failure: only a + // transpilation error can carry a rule's own MySQL error code. + ctx.Output.TranspileErr = err return err } diff --git a/protocol/query/pipeline_autoincrement_test.go b/protocol/query/pipeline_autoincrement_test.go index 422f0308..209d935d 100644 --- a/protocol/query/pipeline_autoincrement_test.go +++ b/protocol/query/pipeline_autoincrement_test.go @@ -4,6 +4,8 @@ import ( "strings" "sync/atomic" "testing" + + "github.com/maxpert/marmot/protocol/query/transform" ) // mockIDGenerator is a test ID generator that returns predictable IDs @@ -15,10 +17,16 @@ func (m *mockIDGenerator) NextID() uint64 { return m.counter.Add(1) } -// mockSchemaLookup returns a SchemaLookup function that knows about specific tables -func mockSchemaLookup(tableColumns map[string]string) func(string) string { - return func(table string) string { - return tableColumns[table] +// mockSchemaLookup returns a schema lookup that knows about specific tables. +// The ordinal is the auto-increment column's position in the table's own column +// order, which is what a column-less INSERT's VALUES tuple is indexed by. +func mockSchemaLookup(tables map[string]transform.SchemaInfo) func(string) *transform.SchemaInfo { + return func(table string) *transform.SchemaInfo { + info, ok := tables[table] + if !ok { + return nil + } + return &info } } @@ -52,8 +60,8 @@ func TestPipelineAutoIncrementIDInjection(t *testing.T) { } // Set up schema lookup for sbtest1 table - schemaLookup := mockSchemaLookup(map[string]string{ - "sbtest1": "id", + schemaLookup := mockSchemaLookup(map[string]transform.SchemaInfo{ + "sbtest1": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 0}, }) // Now test INSERT with 0 value @@ -104,8 +112,8 @@ func TestPipelineAutoIncrementIDInjection_NullValue(t *testing.T) { } // Set up schema lookup for users table - schemaLookup := mockSchemaLookup(map[string]string{ - "users": "id", + schemaLookup := mockSchemaLookup(map[string]transform.SchemaInfo{ + "users": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 0}, }) // Test INSERT with NULL value @@ -158,8 +166,8 @@ func TestPipelineAutoIncrementIDInjection_ExplicitID(t *testing.T) { } // Set up schema lookup for users table - schemaLookup := mockSchemaLookup(map[string]string{ - "users": "id", + schemaLookup := mockSchemaLookup(map[string]transform.SchemaInfo{ + "users": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 0}, }) // When user provides explicit non-zero ID, it should not be modified @@ -188,8 +196,8 @@ func TestPipelineAutoIncrementIDInjection_MultiRow(t *testing.T) { } // Set up schema lookup for users table - schemaLookup := mockSchemaLookup(map[string]string{ - "users": "id", + schemaLookup := mockSchemaLookup(map[string]transform.SchemaInfo{ + "users": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 0}, }) sql := "INSERT INTO users (id, name) VALUES (0, 'alice'), (0, 'bob'), (100, 'charlie')" @@ -248,8 +256,8 @@ func TestPipelineAutoIncrementIDInjection_CacheBypassForIDInjection(t *testing.T } // Set up schema lookup for users table - schemaLookup := mockSchemaLookup(map[string]string{ - "users": "id", + schemaLookup := mockSchemaLookup(map[string]transform.SchemaInfo{ + "users": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 0}, }) // Process the same INSERT twice - each should get unique ID @@ -295,8 +303,8 @@ func TestPipelineAutoIncrementIDInjection_InsertIgnoreWithPatternTransform(t *te } // Set up schema lookup for users table - schemaLookup := mockSchemaLookup(map[string]string{ - "users": "id", + schemaLookup := mockSchemaLookup(map[string]transform.SchemaInfo{ + "users": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 0}, }) // Test INSERT IGNORE with 0 value - should apply both ID injection AND pattern transform diff --git a/protocol/query/rules/autoincrement_id.go b/protocol/query/rules/autoincrement_id.go index fe3dc614..9f7c2f01 100644 --- a/protocol/query/rules/autoincrement_id.go +++ b/protocol/query/rules/autoincrement_id.go @@ -5,19 +5,23 @@ import ( "strings" "github.com/maxpert/marmot/id" + "github.com/maxpert/marmot/protocol/query/transform" "vitess.io/vitess/go/vt/sqlparser" ) -// SchemaLookup returns the auto-increment column name for a table. -// Returns empty string if the table has no auto-increment column. -type SchemaLookup func(table string) string +// SchemaLookup returns the schema facts an INSERT needs for id injection. +// It returns nil when the table is unknown. +type SchemaLookup func(table string) *transform.SchemaInfo -// AutoIncrementIDRule injects distributed HLC-based IDs for auto-increment columns. -// Detection is based on SQLite schema: single BIGINT PRIMARY KEY column. +// AutoIncrementIDRule injects distributed IDs for auto-increment columns. +// Detection is based on the SQLite schema: a single INTEGER/BIGINT PRIMARY KEY. // -// The rule handles two cases: -// 1. Column missing from INSERT → add column with generated ID -// 2. Column present with 0 or NULL → replace with generated ID +// The governing principle is that SQLite must never assign a primary key, +// because two nodes assigning rowids independently both mint 1, 2, 3 and CDC +// replays the collision as INSERT OR REPLACE. The rule therefore substitutes a +// generated id wherever the id's position in the statement is knowable, and +// refuses the statement only where safe substitution is structurally +// impossible. type AutoIncrementIDRule struct { generator id.Generator } @@ -32,116 +36,155 @@ func NewAutoIncrementIDRule(gen id.Generator) *AutoIncrementIDRule { func (r *AutoIncrementIDRule) Name() string { return "AutoIncrementID" } func (r *AutoIncrementIDRule) Priority() int { return 5 } -// NeedsIDInjection checks if an INSERT statement needs ID injection. -// This is used to bypass cache for statements that need unique IDs. -func (r *AutoIncrementIDRule) NeedsIDInjection(stmt sqlparser.Statement, schemaLookup SchemaLookup) bool { - if r.generator == nil || schemaLookup == nil { - return false - } +// insertPlan is what analyzeInsert decided ApplyAST must do to an INSERT. +// Exactly one of appendColumn and valueIdx is meaningful: appendColumn names a +// column to add to an explicit column list (with one generated value per row), +// while valueIdx is the position in each VALUES tuple whose placeholder value +// is replaced by a generated id. +type insertPlan struct { + insert *sqlparser.Insert + rows sqlparser.Values + appendColumn string + valueIdx int +} +// analyzeInsert decides what the rule must do with a statement, without +// mutating it. It returns (nil, nil) when there is nothing to do, a plan when +// the statement can be rewritten, and an error when the statement must be +// refused. NeedsIDInjection and ApplyAST both go through it so the transpiler's +// gate and the rewrite can never disagree. +func analyzeInsert(stmt sqlparser.Statement, schemaLookup SchemaLookup) (*insertPlan, error) { insert, ok := stmt.(*sqlparser.Insert) if !ok { - return false + return nil, nil } tableName := extractInsertTableName(insert) if tableName == "" { - return false + return nil, nil } - autoIncCol := schemaLookup(tableName) - if autoIncCol == "" { - return false + info := schemaLookup(tableName) + if !info.HasAutoIncrement() { + return nil, nil } - - // Check if the auto-inc column is missing or has 0/NULL value + autoIncCol := info.AutoIncrementColumn colIdx := findColumnIndex(insert.Columns, autoIncCol) - if colIdx < 0 { - // Column missing from INSERT - // If no column list specified, we can't safely inject without schema - if len(insert.Columns) == 0 { - return false + + rows, isValues := insert.Rows.(sqlparser.Values) + if !isValues { + // The rows come from a query, not from literals this rule can rewrite: + // sqlparser.InsertRows is only Values, *Select, *Union or + // *ValuesStatement, and the last three all supply their values at + // execution time. When the column list names the auto-increment + // column the query supplies its values and the statement is left + // alone; otherwise SQLite would assign the rowid itself, so the + // statement is refused. + // A table value constructor (VALUES ROW(...)) is refused in BOTH forms. + // Its literals live inside a statement node this rule cannot index + // positionally, so the column-listed form would otherwise pass the gate + // with a NULL id and reach SQLite, which cannot parse VALUES ROW at all. + if _, isRowConstructor := insert.Rows.(*sqlparser.ValuesStatement); isRowConstructor { + return nil, transform.NewCodedError(transform.ErrCodeNotSupportedYet, + "This version of MySQL doesn't yet support 'INSERT ... VALUES ROW(...) into table `%s`, which has AUTO_INCREMENT column `%s`'", + tableName, autoIncCol) } - // Column missing from explicit column list - needs injection - return true + if colIdx >= 0 { + return nil, nil + } + return nil, transform.NewCodedError(transform.ErrCodeNotSupportedYet, + "This version of MySQL doesn't yet support 'INSERT ... SELECT into table `%s` without a projected value for AUTO_INCREMENT column `%s`'", + tableName, autoIncCol) } - // Column present - check if any row has 0/NULL value - rows, ok := insert.Rows.(sqlparser.Values) - if !ok { - return false + if colIdx >= 0 { + // The column list names the auto-increment column: substitute wherever + // a row left the value for the server to fill in. + if rowsNeedID(rows, colIdx) { + return &insertPlan{insert: insert, rows: rows, valueIdx: colIdx}, nil + } + return nil, nil + } + + if len(insert.Columns) > 0 { + // Explicit column list that omits the auto-increment column: add it, + // with one generated id per row. + return &insertPlan{insert: insert, rows: rows, appendColumn: autoIncCol, valueIdx: -1}, nil + } + + // Column-less INSERT. Every column gets a value in the table's own order, + // so the id sits at the auto-increment column's ordinal. A row of the + // wrong length is SQLite's error to report, exactly as it is today, so + // rows that cannot hold the ordinal are left untouched. + ordinal := info.AutoIncrementOrdinal + if ordinal < 0 { + return nil, nil } + if rowsNeedID(rows, ordinal) { + return &insertPlan{insert: insert, rows: rows, valueIdx: ordinal}, nil + } + return nil, nil +} +// rowsNeedID reports whether any row leaves position idx for the server to +// fill in. +func rowsNeedID(rows sqlparser.Values, idx int) bool { for _, row := range rows { - if colIdx < len(row) && needsIDInjection(row[colIdx]) { + if idx < len(row) && needsIDInjection(row[idx]) { return true } } - return false } -func (r *AutoIncrementIDRule) ApplyAST(stmt sqlparser.Statement, schemaLookup SchemaLookup) (sqlparser.Statement, bool, error) { +// NeedsIDInjection reports whether the statement needs this rule to run. The +// transpiler uses it both to decide whether to call ApplyAST and to bypass its +// SQL cache, so a statement this rule would refuse must answer true as well. +func (r *AutoIncrementIDRule) NeedsIDInjection(stmt sqlparser.Statement, schemaLookup SchemaLookup) bool { if r.generator == nil || schemaLookup == nil { - return stmt, false, nil - } - - insert, ok := stmt.(*sqlparser.Insert) - if !ok { - return stmt, false, nil + return false } + plan, err := analyzeInsert(stmt, schemaLookup) + return err != nil || plan != nil +} - tableName := extractInsertTableName(insert) - if tableName == "" { +// ApplyAST rewrites the statement in place, or returns an error when the +// statement must be refused. The error is returned to the caller verbatim; a +// *transform.CodedError carries the MySQL error code the client must see. +func (r *AutoIncrementIDRule) ApplyAST(stmt sqlparser.Statement, schemaLookup SchemaLookup) (sqlparser.Statement, bool, error) { + if r.generator == nil || schemaLookup == nil { return stmt, false, nil } - autoIncCol := schemaLookup(tableName) - if autoIncCol == "" { - return stmt, false, nil + plan, err := analyzeInsert(stmt, schemaLookup) + if err != nil { + return stmt, false, err } - - rows, ok := insert.Rows.(sqlparser.Values) - if !ok { + if plan == nil { return stmt, false, nil } - modified := false - - // Check if the auto-inc column is in the INSERT columns list - colIdx := findColumnIndex(insert.Columns, autoIncCol) - - if colIdx < 0 { - // Column missing from INSERT - // If no column list specified, we can't safely add a column without knowing schema - if len(insert.Columns) == 0 { - return stmt, false, nil - } - // Add column to explicit column list - insert.Columns = append(insert.Columns, sqlparser.NewIdentifierCI(autoIncCol)) - - // Add generated ID to each row (appended to match new column) - for i := range rows { - newID := r.generator.NextID() - rows[i] = append(rows[i], sqlparser.NewIntLiteral(strconv.FormatUint(newID, 10))) - } - modified = true - } else { - // Column present - replace 0/NULL values with generated IDs - for _, row := range rows { - if colIdx < len(row) && needsIDInjection(row[colIdx]) { - newID := r.generator.NextID() - row[colIdx] = sqlparser.NewIntLiteral(strconv.FormatUint(newID, 10)) - modified = true - } + if plan.appendColumn != "" { + plan.insert.Columns = append(plan.insert.Columns, sqlparser.NewIdentifierCI(plan.appendColumn)) + for i := range plan.rows { + plan.rows[i] = append(plan.rows[i], r.newIDLiteral()) } + return plan.insert, true, nil } - if !modified { - return stmt, false, nil + // analyzeInsert only returns a substitution plan when at least one row + // needs one, so this loop always replaces something. + for _, row := range plan.rows { + if plan.valueIdx < len(row) && needsIDInjection(row[plan.valueIdx]) { + row[plan.valueIdx] = r.newIDLiteral() + } } + return plan.insert, true, nil +} - return insert, true, nil +// newIDLiteral mints one id and renders it as an integer literal. +func (r *AutoIncrementIDRule) newIDLiteral() *sqlparser.Literal { + return sqlparser.NewIntLiteral(strconv.FormatUint(r.generator.NextID(), 10)) } // extractInsertTableName extracts the table name from an INSERT statement. @@ -166,18 +209,18 @@ func findColumnIndex(columns sqlparser.Columns, name string) int { return -1 } -// needsIDInjection checks if an expression needs an auto-increment ID injection. -// Returns true for: -// - NULL values -// - Integer literal 0 +// needsIDInjection reports whether an expression in the auto-increment +// column's position asks the server to supply the value. MySQL treats NULL, +// literal 0 and DEFAULT identically here. func needsIDInjection(expr sqlparser.Expr) bool { switch v := expr.(type) { case *sqlparser.NullVal: return true + case *sqlparser.Default: + return true case *sqlparser.Literal: if v.Type == sqlparser.IntVal { - val := strings.TrimSpace(string(v.Val)) - return val == "0" + return strings.TrimSpace(string(v.Val)) == "0" } } return false diff --git a/protocol/query/rules/autoincrement_id_test.go b/protocol/query/rules/autoincrement_id_test.go new file mode 100644 index 00000000..73065aec --- /dev/null +++ b/protocol/query/rules/autoincrement_id_test.go @@ -0,0 +1,304 @@ +package rules + +import ( + "errors" + "strings" + "sync/atomic" + "testing" + + "github.com/maxpert/marmot/protocol/query/transform" + "vitess.io/vitess/go/vt/sqlparser" +) + +// seqGenerator hands out 1000, 1001, ... so an injected id is recognisable in +// the serialized statement and the order of injection is observable. +type seqGenerator struct{ n atomic.Uint64 } + +func (g *seqGenerator) NextID() uint64 { return 1000 + g.n.Add(1) - 1 } + +// lookupFor builds the schema lookup the rule receives. +func lookupFor(tables map[string]transform.SchemaInfo) SchemaLookup { + return func(table string) *transform.SchemaInfo { + info, ok := tables[table] + if !ok { + return nil + } + return &info + } +} + +func parseOne(t *testing.T, sql string) sqlparser.Statement { + t.Helper() + p, err := sqlparser.New(sqlparser.Options{}) + if err != nil { + t.Fatalf("sqlparser.New: %v", err) + } + stmt, err := p.Parse(sql) + if err != nil { + t.Fatalf("parse %q: %v", sql, err) + } + return stmt +} + +// idFirst is a table whose auto-increment column is its first column; idThird +// is one where it is the third. The ordinal is the whole point of the +// column-less arms: a rule that assumed position 0 passes idFirst and corrupts +// idThird. +var injectionSchemas = map[string]transform.SchemaInfo{ + "idfirst": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 0}, + "idthird": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 2}, + "plain": {AutoIncrementColumn: "", AutoIncrementOrdinal: -1}, +} + +func TestAutoIncrementIDRuleInjectionShapes(t *testing.T) { + cases := []struct { + name string + sql string + // want is the serialized statement after the rule ran. Empty means + // "must be left exactly as parsed". + want string + wantApplied bool + }{ + { + // Mutation: restore the `len(insert.Columns) == 0 -> return` early + // exit in analyzeInsert; no id is injected and want no longer matches. + name: "column-less single row at ordinal 0", + sql: "insert into idfirst values (null, 'a')", + want: "insert into idfirst values (1000, 'a')", + wantApplied: true, + }, + { + // Mutation: inject once instead of per row (hoist newIDLiteral out + // of the loop); the three ids collapse to one value. + name: "column-less multi row mixes NULL, 0 and DEFAULT", + sql: "insert into idfirst values (null, 'a'), (0, 'b'), (default, 'c')", + want: "insert into idfirst values (1000, 'a'), (1001, 'b'), (1002, 'c')", + wantApplied: true, + }, + { + // Mutation: use 0 instead of info.AutoIncrementOrdinal for the + // column-less case; the id lands in the first column. + name: "column-less substitutes at a non-zero ordinal", + sql: "insert into idthird values ('x', 'y', null)", + want: "insert into idthird values ('x', 'y', 1000)", + wantApplied: true, + }, + { + // Mutation: drop the *sqlparser.Default arm from needsIDInjection; + // DEFAULT reaches SQLite and is a syntax error there. + name: "DEFAULT in a listed auto-increment column", + sql: "insert into idfirst (id, v) values (default, 'a')", + want: "insert into idfirst(id, v) values (1000, 'a')", + wantApplied: true, + }, + { + // Mutation: delete the `len(insert.Columns) > 0` append branch; + // the column is never added. + name: "explicit column list omitting the auto-increment column", + sql: "insert into idfirst (v) values ('a'), ('b')", + want: "insert into idfirst(v, id) values ('a', 1000), ('b', 1001)", + wantApplied: true, + }, + { + // Mutation: make needsIDInjection return true for any Literal; + // mysqldump's explicit ids get overwritten and the dump no longer + // restores the ids it recorded. + name: "mysqldump shape: column-less with an explicit id is untouched", + sql: "insert into idfirst values (7, 'a'), (8, 'b')", + want: "", + wantApplied: false, + }, + { + // Mutation: same as above, for the listed-column form. + name: "explicit non-zero id in a listed column is untouched", + sql: "insert into idfirst (id, v) values (7, 'a')", + want: "", + wantApplied: false, + }, + { + // Mutation: drop the HasAutoIncrement guard; the rule injects into + // a table that has no auto-increment column at all. + name: "table without an auto-increment column is untouched", + sql: "insert into plain values (1, 'a')", + want: "", + wantApplied: false, + }, + { + // Mutation: drop the `colIdx >= 0 -> return nil, nil` arm on the + // non-Values path; a projecting INSERT ... SELECT starts erroring. + name: "INSERT ... SELECT projecting the auto-increment column is untouched", + sql: "insert into idfirst (id, v) select id, v from other", + want: "", + wantApplied: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rule := NewAutoIncrementIDRule(&seqGenerator{}) + lookup := lookupFor(injectionSchemas) + stmt := parseOne(t, tc.sql) + before := sqlparser.String(stmt) + + needs := rule.NeedsIDInjection(stmt, lookup) + got, applied, err := rule.ApplyAST(stmt, lookup) + if err != nil { + t.Fatalf("ApplyAST returned an error for %q: %v", tc.sql, err) + } + if applied != tc.wantApplied { + t.Errorf("applied = %v, want %v, for %q", applied, tc.wantApplied, tc.sql) + } + // The transpiler calls ApplyAST only when NeedsIDInjection says so + // and bypasses its SQL cache on the same answer, so a gate that + // disagrees with the rewrite makes the rewrite unreachable. + // Mutation: restore the independent column-less/Values checks in + // NeedsIDInjection and this fires on every column-less arm. + if needs != tc.wantApplied { + t.Errorf("NeedsIDInjection = %v but ApplyAST applied = %v, for %q", needs, tc.wantApplied, tc.sql) + } + + want := tc.want + if want == "" { + want = before + } + if out := sqlparser.String(got); out != want { + t.Errorf("statement after rule:\n got: %s\nwant: %s", out, want) + } + }) + } +} + +func TestAutoIncrementIDRuleRejectsUnprojectedInsertSelect(t *testing.T) { + rule := NewAutoIncrementIDRule(&seqGenerator{}) + lookup := lookupFor(injectionSchemas) + + for _, sql := range []string{ + "insert into idfirst select id, v from other", + "insert into idfirst (v) select v from other", + } { + stmt := parseOne(t, sql) + + // Mutation: restore the `insert.Rows.(sqlparser.Values)` early return + // in analyzeInsert. The statement is then accepted, SQLite assigns the + // rowid, and two nodes both mint 1, 2, 3. + if !rule.NeedsIDInjection(stmt, lookup) { + t.Fatalf("NeedsIDInjection = false for %q: the transpiler would cache it and never call ApplyAST", sql) + } + + _, applied, err := rule.ApplyAST(stmt, lookup) + if err == nil { + t.Fatalf("ApplyAST accepted %q; want a rejection", sql) + } + if applied { + t.Errorf("ApplyAST reported applied = true alongside an error for %q", sql) + } + + // Mutation: return a plain fmt.Errorf instead of a CodedError; the + // client then sees ER_UNKNOWN_ERROR (1105) instead of 1235. + var coded *transform.CodedError + if !errors.As(err, &coded) { + t.Fatalf("error for %q is %T, want *transform.CodedError", sql, err) + } + if coded.Code != transform.ErrCodeNotSupportedYet { + t.Errorf("error code = %d, want %d (ER_NOT_SUPPORTED_YET) for %q", coded.Code, transform.ErrCodeNotSupportedYet, sql) + } + // 1235's MySQL message template is "This version of MySQL doesn't yet + // support '%s'". A client that matches on it must still recognise ours. + // Mutation: reword the message without the template. + if !strings.HasPrefix(coded.Message, "This version of MySQL doesn't yet support '") { + t.Errorf("message %q does not follow MySQL's ER_NOT_SUPPORTED_YET template", coded.Message) + } + // The message must name the column the client has to add, or the + // error is unactionable. + // Mutation: drop the column name from the message. + if !strings.Contains(coded.Message, "`id`") { + t.Errorf("error message %q does not name the auto-increment column", coded.Message) + } + } +} + +// TestAutoIncrementIDRuleRejectsRowConstructor covers MySQL 8.0.19+'s table +// value constructor. BOTH forms must be refused with 1235: the column-less one +// because the rule cannot index the tuple positionally, and the column-listed +// one because it otherwise passes the gate carrying a literal NULL id and +// reaches SQLite, which cannot parse VALUES ROW(...) at all. Relying on that +// parse error is an admission-shaped guard - it happens to fail closed today +// for a reason unrelated to id assignment. +func TestAutoIncrementIDRuleRejectsRowConstructor(t *testing.T) { + rule := NewAutoIncrementIDRule(&seqGenerator{}) + lookup := lookupFor(injectionSchemas) + + for _, sql := range []string{ + "insert into idfirst values row(null, 'a')", + "insert into idfirst (id, v) values row(null, 'a')", + } { + stmt := parseOne(t, sql) + + // Mutation: drop the *sqlparser.ValuesStatement arm in analyzeInsert. + // The column-listed form then returns nothing-to-do and this fires. + if !rule.NeedsIDInjection(stmt, lookup) { + t.Fatalf("NeedsIDInjection = false for %q: the transpiler would cache it and never call ApplyAST", sql) + } + _, applied, err := rule.ApplyAST(stmt, lookup) + if err == nil { + t.Fatalf("ApplyAST accepted %q; want a 1235 rejection", sql) + } + if applied { + t.Errorf("ApplyAST reported applied = true alongside an error for %q", sql) + } + var coded *transform.CodedError + if !errors.As(err, &coded) { + t.Fatalf("error for %q is %T, want *transform.CodedError", sql, err) + } + if coded.Code != transform.ErrCodeNotSupportedYet { + t.Errorf("error code = %d, want %d for %q", coded.Code, transform.ErrCodeNotSupportedYet, sql) + } + // Mutation: reuse the INSERT ... SELECT wording here; it names a + // projection that a row constructor does not have. + if !strings.Contains(coded.Message, "VALUES ROW") { + t.Errorf("message %q does not name the row-constructor shape, for %q", coded.Message, sql) + } + if !strings.HasPrefix(coded.Message, "This version of MySQL doesn't yet support '") { + t.Errorf("message %q does not follow MySQL's ER_NOT_SUPPORTED_YET template", coded.Message) + } + } +} + +func TestAutoIncrementIDRuleNoLookupOrGenerator(t *testing.T) { + stmt := parseOne(t, "insert into idfirst values (null, 'a')") + + // Mutation: drop either nil guard in ApplyAST/NeedsIDInjection; the rule + // dereferences a nil generator or a nil lookup and panics. + rule := NewAutoIncrementIDRule(nil) + if rule.NeedsIDInjection(stmt, lookupFor(injectionSchemas)) { + t.Error("NeedsIDInjection = true with a nil generator") + } + if _, applied, err := rule.ApplyAST(stmt, lookupFor(injectionSchemas)); applied || err != nil { + t.Errorf("ApplyAST with a nil generator: applied=%v err=%v", applied, err) + } + + rule = NewAutoIncrementIDRule(&seqGenerator{}) + if rule.NeedsIDInjection(stmt, nil) { + t.Error("NeedsIDInjection = true with a nil schema lookup") + } + if _, applied, err := rule.ApplyAST(stmt, nil); applied || err != nil { + t.Errorf("ApplyAST with a nil schema lookup: applied=%v err=%v", applied, err) + } +} + +// TestAutoIncrementIDRuleUnknownTable pins that a lookup returning nil - the +// table is not in the schema cache - is not a nil dereference. +// Mutation: read info.AutoIncrementColumn directly instead of going through +// info.HasAutoIncrement(), which is the nil-safe accessor. +func TestAutoIncrementIDRuleUnknownTable(t *testing.T) { + rule := NewAutoIncrementIDRule(&seqGenerator{}) + lookup := lookupFor(injectionSchemas) + stmt := parseOne(t, "insert into nosuchtable values (null, 'a')") + + if rule.NeedsIDInjection(stmt, lookup) { + t.Error("NeedsIDInjection = true for a table the lookup does not know") + } + if _, applied, err := rule.ApplyAST(stmt, lookup); applied || err != nil { + t.Errorf("ApplyAST on an unknown table: applied=%v err=%v", applied, err) + } +} diff --git a/protocol/query/rules/int_to_bigint.go b/protocol/query/rules/int_to_bigint.go deleted file mode 100644 index 9b2b9173..00000000 --- a/protocol/query/rules/int_to_bigint.go +++ /dev/null @@ -1,63 +0,0 @@ -package rules - -import ( - "strings" - - "vitess.io/vitess/go/vt/sqlparser" -) - -// IntToBigintRule transforms INT/INTEGER AUTO_INCREMENT columns to BIGINT -// in CREATE TABLE statements. This is needed for distributed auto-increment -// support because HLC-based IDs are 64-bit and require BIGINT storage. -// -// Transformation: -// - INT AUTO_INCREMENT → BIGINT -// - INTEGER AUTO_INCREMENT → BIGINT -// -// This rule must run BEFORE CreateTableRule which removes AUTO_INCREMENT. -type IntToBigintRule struct{} - -func (r *IntToBigintRule) Name() string { return "IntToBigint" } -func (r *IntToBigintRule) Priority() int { return 4 } // Run before CreateTableRule (priority 5) - -func (r *IntToBigintRule) ApplyAST(stmt sqlparser.Statement) (sqlparser.Statement, bool, error) { - create, ok := stmt.(*sqlparser.CreateTable) - if !ok { - return stmt, false, nil - } - - if create.TableSpec == nil { - return stmt, false, nil - } - - modified := false - - for _, col := range create.TableSpec.Columns { - if col.Type == nil || col.Type.Options == nil { - continue - } - - // Only transform columns with AUTO_INCREMENT - if !col.Type.Options.Autoincrement { - continue - } - - // Check if type is INT or INTEGER (case-insensitive) - upperType := strings.ToUpper(col.Type.Type) - if upperType == "INT" || upperType == "INTEGER" { - col.Type.Type = "bigint" - modified = true - } - } - - if !modified { - return stmt, false, nil - } - - return create, true, nil -} - -func (r *IntToBigintRule) ApplyPattern(sql string) (string, bool, error) { - // No pattern-based transformations needed - return sql, false, nil -} diff --git a/protocol/query/rules/int_to_bigint_test.go b/protocol/query/rules/int_to_bigint_test.go deleted file mode 100644 index 8b761c47..00000000 --- a/protocol/query/rules/int_to_bigint_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package rules - -import ( - "strings" - "testing" - - "vitess.io/vitess/go/vt/sqlparser" -) - -// parseSQL is a helper to parse SQL for tests - Vitess uses NewTestParser() -func parseSQL(sql string) (sqlparser.Statement, error) { - return sqlparser.NewTestParser().Parse(sql) -} - -func TestIntToBigintRule(t *testing.T) { - tests := []struct { - name string - input string - wantSQL string - modified bool - }{ - { - name: "INT AUTO_INCREMENT to BIGINT", - input: "CREATE TABLE users (id INT AUTO_INCREMENT PRIMARY KEY, name VARCHAR(100))", - wantSQL: "bigint", - modified: true, - }, - { - name: "INTEGER AUTO_INCREMENT to BIGINT", - input: "CREATE TABLE items (id INTEGER AUTO_INCREMENT PRIMARY KEY, data TEXT)", - wantSQL: "bigint", - modified: true, - }, - { - name: "int lowercase AUTO_INCREMENT to BIGINT", - input: "CREATE TABLE logs (id int AUTO_INCREMENT, msg TEXT)", - wantSQL: "bigint", - modified: true, - }, - { - name: "INT without AUTO_INCREMENT unchanged", - input: "CREATE TABLE orders (id INT PRIMARY KEY, total REAL)", - wantSQL: "int", - modified: false, - }, - { - name: "BIGINT AUTO_INCREMENT unchanged", - input: "CREATE TABLE big (id BIGINT AUTO_INCREMENT PRIMARY KEY, data TEXT)", - wantSQL: "bigint", - modified: false, - }, - { - name: "VARCHAR column unchanged", - input: "CREATE TABLE names (id VARCHAR(50) PRIMARY KEY, value TEXT)", - wantSQL: "varchar", - modified: false, - }, - { - name: "Non-CREATE statement unchanged", - input: "INSERT INTO users (id, name) VALUES (1, 'test')", - wantSQL: "", - modified: false, - }, - } - - rule := &IntToBigintRule{} - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - stmt, err := parseSQL(tt.input) - if err != nil { - if tt.wantSQL == "" { - // Expected to not be a CREATE TABLE - return - } - t.Fatalf("failed to parse SQL: %v", err) - } - - newStmt, modified, err := rule.ApplyAST(stmt) - if err != nil { - t.Fatalf("ApplyAST failed: %v", err) - } - - if modified != tt.modified { - t.Errorf("modified = %v, want %v", modified, tt.modified) - } - - if tt.wantSQL != "" { - output := sqlparser.String(newStmt) - if !strings.Contains(strings.ToLower(output), tt.wantSQL) { - t.Errorf("output SQL doesn't contain %q:\n%s", tt.wantSQL, output) - } - } - }) - } -} - -func TestIntToBigintRule_Priority(t *testing.T) { - rule := &IntToBigintRule{} - - // IntToBigintRule should run before CreateTableRule (priority 5) - if rule.Priority() >= 5 { - t.Errorf("Priority = %d, want < 5 (must run before CreateTableRule)", rule.Priority()) - } -} - -func TestIntToBigintRule_Name(t *testing.T) { - rule := &IntToBigintRule{} - if rule.Name() != "IntToBigint" { - t.Errorf("Name = %q, want %q", rule.Name(), "IntToBigint") - } -} - -func TestIntToBigintRule_MultipleColumns(t *testing.T) { - rule := &IntToBigintRule{} - - input := "CREATE TABLE test (id INT AUTO_INCREMENT PRIMARY KEY, ref_id INT, counter INT AUTO_INCREMENT)" - stmt, err := parseSQL(input) - if err != nil { - t.Fatalf("failed to parse SQL: %v", err) - } - - newStmt, modified, err := rule.ApplyAST(stmt) - if err != nil { - t.Fatalf("ApplyAST failed: %v", err) - } - - if !modified { - t.Error("expected statement to be modified") - } - - output := sqlparser.String(newStmt) - - // Both AUTO_INCREMENT columns should be BIGINT now - // ref_id without AUTO_INCREMENT should remain INT - lowerOutput := strings.ToLower(output) - - // Count occurrences of "bigint" - bigintCount := strings.Count(lowerOutput, "bigint") - if bigintCount != 2 { - t.Errorf("expected 2 bigint columns, got %d in: %s", bigintCount, output) - } -} diff --git a/protocol/query/transform/errors.go b/protocol/query/transform/errors.go new file mode 100644 index 00000000..d5882dc8 --- /dev/null +++ b/protocol/query/transform/errors.go @@ -0,0 +1,37 @@ +package transform + +import "fmt" + +// MySQL server error codes raised by transformation rules. +// +// These live here, rather than beside the protocol package's error codes, +// because transform is a leaf package: protocol imports it and never the +// reverse, so a rule cannot name protocol.ErrCode*. Only codes a rule actually +// raises belong here, and none of them is declared by protocol as well. +const ( + // ErrCodeNotSupportedYet is MySQL's ER_NOT_SUPPORTED_YET (1235): the + // server understands the statement but does not implement this form of it. + ErrCodeNotSupportedYet uint16 = 1235 +) + +// CodedError is the error a transformation rule returns when it refuses a +// statement it cannot safely rewrite. Code is the MySQL server error code the +// client must see; protocol.ConvertToMySQLError pairs it with its SQLSTATE and +// turns it into the error packet. +// +// A rule that fails for any other reason returns a plain error, which the +// protocol layer reports as ER_UNKNOWN_ERROR (1105). +type CodedError struct { + Code uint16 + Message string +} + +func (e *CodedError) Error() string { + return fmt.Sprintf("ERROR %d: %s", e.Code, e.Message) +} + +// NewCodedError builds a rule rejection carrying the MySQL error code the +// client must see. +func NewCodedError(code uint16, format string, args ...interface{}) *CodedError { + return &CodedError{Code: code, Message: fmt.Sprintf(format, args...)} +} diff --git a/protocol/query/transform/rule.go b/protocol/query/transform/rule.go index 4aa225c4..916d3501 100644 --- a/protocol/query/transform/rule.go +++ b/protocol/query/transform/rule.go @@ -48,11 +48,14 @@ type SchemaInfo struct { // Empty string if no auto-increment column exists. // In SQLite, this is typically INTEGER PRIMARY KEY. AutoIncrementColumn string -} -// HasPrimaryKey returns true if table has a PRIMARY KEY defined -func (s *SchemaInfo) HasPrimaryKey() bool { - return s != nil && len(s.PrimaryKey) > 0 + // AutoIncrementOrdinal is AutoIncrementColumn's position in the table's + // own column order, counting only columns a column-less + // "INSERT INTO t VALUES (...)" supplies a value for - that is, excluding + // GENERATED columns, which SQLite refuses to accept a value for. It is the + // index into such a statement's VALUES tuple that holds the id. + // -1 when AutoIncrementColumn is empty. + AutoIncrementOrdinal int } // HasAutoIncrement returns true if table has an auto-increment column diff --git a/protocol/query/transpiler.go b/protocol/query/transpiler.go index 71ebb342..141ec16a 100644 --- a/protocol/query/transpiler.go +++ b/protocol/query/transpiler.go @@ -107,7 +107,13 @@ func (t *Transpiler) Transpile(ctx *QueryContext) error { // Apply ID injection FIRST if needed if needsIDInjection { newAST, applied, err := t.autoIncRule.ApplyAST(ast, ctx.SchemaLookup) - if err == nil && applied { + if err != nil { + // A rule that refuses a statement must reach the client. Returning + // the error unchanged keeps whatever MySQL error code the rule + // chose (see transform.CodedError). + return err + } + if applied { ast = newAST transformations = append(transformations, Transformation{ Rule: t.autoIncRule.Name(), diff --git a/protocol/query/transpiler_rule_error_test.go b/protocol/query/transpiler_rule_error_test.go new file mode 100644 index 00000000..5efd6daa --- /dev/null +++ b/protocol/query/transpiler_rule_error_test.go @@ -0,0 +1,95 @@ +package query + +import ( + "testing" + + "github.com/maxpert/marmot/protocol/query/transform" +) + +// ruleErrorSchemas gives the target table an auto-increment column, which is +// what makes an INSERT ... SELECT that does not project it a rejection. +var ruleErrorSchemas = map[string]transform.SchemaInfo{ + "orders": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 0}, +} + +// TestTranspileReturnsRuleErrorVerbatim is the regression guard on the +// swallow that used to sit in Transpile: `if err == nil && applied`, which +// dropped every error a rule returned and let the statement proceed. +func TestTranspileReturnsRuleErrorVerbatim(t *testing.T) { + pipeline, err := NewPipeline(1000, &mockIDGenerator{}) + if err != nil { + t.Fatalf("NewPipeline: %v", err) + } + + ctx := NewContext("INSERT INTO orders SELECT id, total FROM staging", nil) + ctx.SchemaLookup = mockSchemaLookup(ruleErrorSchemas) + + if err := pipeline.parser.Parse(ctx); err != nil { + t.Fatalf("parse: %v", err) + } + + gotErr := pipeline.transpiler.Transpile(ctx) + + // Mutation: restore `if err == nil && applied` in Transpile. gotErr is nil + // and this fires - the statement would have gone on to SQLite, which + // assigns the rowid itself. + if gotErr == nil { + t.Fatal("Transpile returned nil; the rule's rejection was swallowed") + } + + // Verbatim: a direct type assertion, not errors.As, so wrapping the error + // on the way out also fails this. + // Mutation: wrap the error in Transpile with fmt.Errorf("%w"). + coded, ok := gotErr.(*transform.CodedError) + if !ok { + t.Fatalf("Transpile returned %T (%v); want the rule's own *transform.CodedError", gotErr, gotErr) + } + if coded.Code != transform.ErrCodeNotSupportedYet { + t.Errorf("error code = %d, want %d", coded.Code, transform.ErrCodeNotSupportedYet) + } +} + +// TestProcessRecordsTranspileErrorTyped pins that the pipeline hands the typed +// error up to the protocol layer. Without it the protocol layer cannot tell a +// rule's rejection from a Vitess syntax error and reports 1064 for both. +func TestProcessRecordsTranspileErrorTyped(t *testing.T) { + pipeline, err := NewPipeline(1000, &mockIDGenerator{}) + if err != nil { + t.Fatalf("NewPipeline: %v", err) + } + + ctx := NewContext("INSERT INTO orders SELECT id, total FROM staging", nil) + ctx.SchemaLookup = mockSchemaLookup(ruleErrorSchemas) + + if err := pipeline.Process(ctx); err == nil { + t.Fatal("Process returned nil for a statement the rule refuses") + } + + // Mutation: delete the `ctx.Output.TranspileErr = err` assignment in + // Pipeline.Process. + if ctx.Output.TranspileErr == nil { + t.Fatal("Process did not record TranspileErr") + } + if _, ok := ctx.Output.TranspileErr.(*transform.CodedError); !ok { + t.Errorf("TranspileErr is %T, want *transform.CodedError", ctx.Output.TranspileErr) + } +} + +// TestProcessLeavesTranspileErrNilOnParseFailure is the converse guard: a +// syntax error must NOT be reported as a rule rejection, or every parse error +// changes error code. +// Mutation: set ctx.Output.TranspileErr in the parser branch of Process too. +func TestProcessLeavesTranspileErrNilOnParseFailure(t *testing.T) { + pipeline, err := NewPipeline(1000, &mockIDGenerator{}) + if err != nil { + t.Fatalf("NewPipeline: %v", err) + } + + ctx := NewContext("SELECT FROM WHERE ORDER BY", nil) + if err := pipeline.Process(ctx); err == nil { + t.Fatal("Process accepted a syntactically invalid statement") + } + if ctx.Output.TranspileErr != nil { + t.Errorf("TranspileErr = %v for a parse failure; want nil", ctx.Output.TranspileErr) + } +} diff --git a/protocol/server.go b/protocol/server.go index 19459c25..f4405b4f 100644 --- a/protocol/server.go +++ b/protocol/server.go @@ -94,6 +94,7 @@ type ConnectionSession struct { // LastInsertId stores the value returned by LAST_INSERT_ID(). // Updated after INSERT statements that generate auto-increment IDs. + // Write it only through RecordInsertId, which owns the zero rule. LastInsertId atomic.Int64 // TranspilationEnabled controls MySQL→SQLite transpilation for this session. @@ -568,9 +569,7 @@ func (s *MySQLServer) processQuery(conn net.Conn, session *ConnectionSession, qu if rs != nil { rowsAffected = rs.RowsAffected lastInsertId = rs.LastInsertId - if lastInsertId != 0 { - session.LastInsertId.Store(lastInsertId) - } + session.RecordInsertId(lastInsertId) } _ = s.writeOK(conn, 1, session, rowsAffected, lastInsertId) return @@ -697,6 +696,24 @@ func (s *MySQLServer) writeErrorWithState(w io.Writer, seq byte, code uint16, sq return s.writePacket(w, seq, buf.Bytes()) } +// RecordInsertId stores a statement's insert id as this session's +// LAST_INSERT_ID(). It is the ONLY place the zero rule lives, and every path +// that receives an insert id must go through it: the coordinator's own OK-packet +// paths here, and a replica's forwarded responses in replica/handler.go. +// +// A zero id means the statement generated no AUTO_INCREMENT value - an INSERT +// that inserted nothing, an upsert that updated an existing row, a table with no +// auto-increment column. MySQL leaves LAST_INSERT_ID() unchanged in that case +// ("The value of LAST_INSERT_ID() remains unchanged if no rows are successfully +// inserted"), so a zero is dropped rather than stored. Storing it would reset a +// value the client is entitled to keep reading. +func (s *ConnectionSession) RecordInsertId(id int64) { + if id == 0 { + return + } + s.LastInsertId.Store(id) +} + // writeMySQLErr writes a MySQLError to the connection, extracting code/state from the error func (s *MySQLServer) writeMySQLErr(w io.Writer, seq byte, err error) error { mysqlErr := ConvertToMySQLError(err) @@ -1426,9 +1443,7 @@ func (s *MySQLServer) handleStmtExecute(conn net.Conn, session *ConnectionSessio if rs != nil { rowsAffected = rs.RowsAffected lastInsertId = rs.LastInsertId - if lastInsertId != 0 { - session.LastInsertId.Store(lastInsertId) - } + session.RecordInsertId(lastInsertId) } _ = s.writeOK(conn, 1, session, rowsAffected, lastInsertId) } diff --git a/protocol/transaction.go b/protocol/transaction.go index a26882b2..d53d22b3 100644 --- a/protocol/transaction.go +++ b/protocol/transaction.go @@ -128,6 +128,12 @@ type Statement struct { IntentKey []byte `msgpack:"IntentKey"` // Intent key for MVCC conflict detection (binary format) Error string `msgpack:"Error"` // Error message if Type is StatementUnsupported + // TranspileErr is the typed error transpilation failed with, set only when + // a transformation rule refused the statement (never when parsing failed). + // It carries the MySQL error code the client must see; Error holds the same + // failure as text. Never replicated: a peer re-parses from SQL. + TranspileErr error `msgpack:"-"` + // CDC: Row-level change data (for DML operations) // Decoded local apply state. Replication sends EncodedRow, not raw SQL. OldValues map[string][]byte `msgpack:"OldValues"` // Before image (for UPDATE/DELETE) diff --git a/protocol/transpile_test.go b/protocol/transpile_test.go deleted file mode 100644 index 1afc2b93..00000000 --- a/protocol/transpile_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package protocol - -import ( - "testing" -) - -func TestTranspileMySQLToSQLite(t *testing.T) { - tests := []struct { - name string - input string - expected string - }{ - // INSERT statements - { - name: "INSERT IGNORE", - input: "INSERT IGNORE INTO users (id, name) VALUES (1, 'test')", - expected: "INSERT OR IGNORE INTO users (id, name) VALUES (1, 'test')", - }, - { - name: "REPLACE INTO", - input: "REPLACE INTO users (id, name) VALUES (1, 'test')", - expected: "INSERT OR REPLACE INTO users (id, name) VALUES (1, 'test')", - }, - - // Index hints - { - name: "FORCE INDEX", - input: "SELECT * FROM users FORCE INDEX(PRIMARY) WHERE id = 1", - expected: "SELECT * FROM users WHERE id = 1", - }, - { - name: "USE INDEX", - input: "SELECT * FROM users USE INDEX(idx_name) WHERE name = 'test'", - expected: "SELECT * FROM users WHERE name = 'test'", - }, - { - name: "IGNORE INDEX", - input: "SELECT * FROM users IGNORE INDEX(idx_name) WHERE id > 10", - expected: "SELECT * FROM users WHERE id > 10", - }, - { - name: "FORCE INDEX with backticks", - input: "SELECT * FROM users FORCE INDEX(`PRIMARY`) WHERE id = 1", - expected: "SELECT * FROM users WHERE id = 1", - }, - - // SELECT modifiers - { - name: "STRAIGHT_JOIN", - input: "SELECT STRAIGHT_JOIN * FROM users u JOIN orders o ON u.id = o.user_id", - expected: "SELECT JOIN * FROM users u JOIN orders o ON u.id = o.user_id", - }, - { - name: "SQL_CALC_FOUND_ROWS", - input: "SELECT SQL_CALC_FOUND_ROWS * FROM users LIMIT 10", - expected: "SELECT * FROM users LIMIT 10", - }, - { - name: "SQL_CACHE", - input: "SELECT SQL_CACHE * FROM users WHERE id = 1", - expected: "SELECT * FROM users WHERE id = 1", - }, - { - name: "SQL_NO_CACHE", - input: "SELECT SQL_NO_CACHE * FROM users WHERE id = 1", - expected: "SELECT * FROM users WHERE id = 1", - }, - { - name: "SQL_SMALL_RESULT", - input: "SELECT SQL_SMALL_RESULT * FROM users GROUP BY status", - expected: "SELECT * FROM users GROUP BY status", - }, - { - name: "HIGH_PRIORITY", - input: "SELECT HIGH_PRIORITY * FROM users WHERE id = 1", - expected: "SELECT * FROM users WHERE id = 1", - }, - - // Locking clauses - { - name: "FOR UPDATE", - input: "SELECT * FROM users WHERE id = 1 FOR UPDATE", - expected: "SELECT * FROM users WHERE id = 1", - }, - { - name: "LOCK IN SHARE MODE", - input: "SELECT * FROM users WHERE id = 1 LOCK IN SHARE MODE", - expected: "SELECT * FROM users WHERE id = 1", - }, - - // LIMIT clause - { - name: "LIMIT offset, count", - input: "SELECT * FROM users LIMIT 10, 20", - expected: "SELECT * FROM users LIMIT 20 OFFSET 10", - }, - { - name: "LIMIT with OFFSET (already correct)", - input: "SELECT * FROM users LIMIT 20 OFFSET 10", - expected: "SELECT * FROM users LIMIT 20 OFFSET 10", - }, - - // Complex combinations - { - name: "Multiple hints and modifiers", - input: "SELECT SQL_CALC_FOUND_ROWS * FROM users FORCE INDEX(PRIMARY) WHERE id > 10 FOR UPDATE LIMIT 5, 10", - expected: "SELECT * FROM users WHERE id > 10 LIMIT 10 OFFSET 5", - }, - { - name: "UPDATE with index hint", - input: "UPDATE users FORCE INDEX(PRIMARY) SET name = 'test' WHERE id = 1", - expected: "UPDATE users SET name = 'test' WHERE id = 1", - }, - { - name: "DELETE with index hint", - input: "DELETE FROM users FORCE INDEX(PRIMARY) WHERE id = 1", - expected: "DELETE FROM users WHERE id = 1", - }, - - // Case insensitivity - { - name: "Mixed case INSERT IGNORE", - input: "insert ignore into users (id) values (1)", - expected: "INSERT OR IGNORE INTO users (id) values (1)", - }, - { - name: "Mixed case REPLACE", - input: "Replace Into users (id) values (1)", - expected: "INSERT OR REPLACE INTO users (id) values (1)", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := transpileMySQLToSQLite(tt.input) - // Normalize whitespace for comparison - if result != tt.expected { - t.Errorf("transpileMySQLToSQLite() = %q, want %q", result, tt.expected) - } - }) - } -} diff --git a/protocol/unsupported_statement_error_test.go b/protocol/unsupported_statement_error_test.go new file mode 100644 index 00000000..70b1d186 --- /dev/null +++ b/protocol/unsupported_statement_error_test.go @@ -0,0 +1,92 @@ +package protocol + +import ( + "errors" + "testing" + + "github.com/maxpert/marmot/protocol/query/transform" +) + +// TestUnsupportedStatementError pins the MySQL error code a client sees for +// each way a statement can reach StatementUnsupported. Before the transpiler +// propagated rule errors, every one of them was 1064. +func TestUnsupportedStatementError(t *testing.T) { + cases := []struct { + name string + stmt Statement + wantCode uint16 + wantSQLState string + wantMessage string + }{ + { + // Mutation: drop the errors.As branch in + // UnsupportedStatementError; the rule's 1235 becomes 1105. + name: "rule rejection keeps its own code", + stmt: Statement{ + Type: StatementUnsupported, + Error: "flattened text", + TranspileErr: transform.NewCodedError(transform.ErrCodeNotSupportedYet, "cannot rewrite %s", "this"), + }, + wantCode: 1235, + wantSQLState: SQLStateSyntax, + wantMessage: "cannot rewrite this", + }, + { + // Mutation: fall through to ErrCodeParseError for a non-coded + // transpile error; a rule failing internally is reported as a + // client syntax error. + name: "other transpile failure is ER_UNKNOWN_ERROR", + stmt: Statement{ + Type: StatementUnsupported, + Error: "rule blew up", + TranspileErr: errors.New("rule blew up"), + }, + wantCode: ErrCodeUnknown, + wantSQLState: SQLStateGeneral, + wantMessage: "rule blew up", + }, + { + // Mutation: route the nil-TranspileErr case through + // ConvertToMySQLError; parse errors stop being 1064. + name: "parse failure stays ER_PARSE_ERROR", + stmt: Statement{ + Type: StatementUnsupported, + Error: "syntax error at position 12", + }, + wantCode: ErrCodeParseError, + wantSQLState: SQLStateSyntax, + wantMessage: "syntax error at position 12", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := UnsupportedStatementError(tc.stmt) + if got.Code != tc.wantCode { + t.Errorf("Code = %d, want %d", got.Code, tc.wantCode) + } + if got.SQLState != tc.wantSQLState { + t.Errorf("SQLState = %q, want %q", got.SQLState, tc.wantSQLState) + } + if got.Message != tc.wantMessage { + t.Errorf("Message = %q, want %q", got.Message, tc.wantMessage) + } + }) + } +} + +// TestConvertToMySQLErrorCodedError pins that a rule's rejection surviving +// anywhere else in the error path - a wrapped return, a forwarded query - +// still reaches the client with its own code. +// Mutation: remove the *transform.CodedError branch from ConvertToMySQLError; +// the message-based fallback returns 1105. +func TestConvertToMySQLErrorCodedError(t *testing.T) { + err := transform.NewCodedError(transform.ErrCodeNotSupportedYet, "nope") + got := ConvertToMySQLError(err) + if got.Code != transform.ErrCodeNotSupportedYet { + t.Errorf("Code = %d, want %d", got.Code, transform.ErrCodeNotSupportedYet) + } + if got.SQLState != SQLStateSyntax { + t.Errorf("SQLState = %q, want %q", got.SQLState, SQLStateSyntax) + } +} diff --git a/protocol/vitess_parser_instance.go b/protocol/vitess_parser_instance.go new file mode 100644 index 00000000..4406a81d --- /dev/null +++ b/protocol/vitess_parser_instance.go @@ -0,0 +1,21 @@ +package protocol + +import ( + "vitess.io/vitess/go/vt/sqlparser" +) + +// SQLiteDriverName is the custom SQLite driver name with REGEXP support. +// This must match the driver name registered in db/sqlite_driver.go. +// The driver will be registered when the db package is imported elsewhere in the application. +const SQLiteDriverName = "sqlite3_marmot" + +// Global parser instance (reused for efficiency) +var vitessParser *sqlparser.Parser + +func init() { + var err error + vitessParser, err = sqlparser.New(sqlparser.Options{}) + if err != nil { + panic("failed to initialize Vitess parser: " + err.Error()) + } +} diff --git a/replica/handler.go b/replica/handler.go index 4ffbc460..4415db08 100644 --- a/replica/handler.go +++ b/replica/handler.go @@ -197,7 +197,7 @@ func (h *ReadOnlyHandler) HandleQuery(session *protocol.ConnectionSession, sql s func (h *ReadOnlyHandler) HandleLoadData(session *protocol.ConnectionSession, sql string, data []byte) (*protocol.ResultSet, error) { stmt := protocol.ParseStatement(sql) if stmt.Type != protocol.StatementLoadData { - return nil, fmt.Errorf("ERROR 1105 (HY000): statement is not LOAD DATA") + return nil, protocol.NewMySQLError(protocol.ErrCodeUnknown, protocol.SQLStateGeneral, "statement is not LOAD DATA") } if !h.forwardWrites { return nil, protocol.ErrReadOnly() @@ -205,7 +205,7 @@ func (h *ReadOnlyHandler) HandleLoadData(session *protocol.ConnectionSession, sq client := h.replica.streamClient.GetClient() if client == nil { - return nil, fmt.Errorf("ERROR 2003 (HY000): Not connected to leader") + return nil, protocol.NewMySQLError(protocol.ErrCodeUnknown, protocol.SQLStateGeneral, "Not connected to leader") } requestID := session.NextForwardRequestID() @@ -220,14 +220,13 @@ func (h *ReadOnlyHandler) HandleLoadData(session *protocol.ConnectionSession, sq TimeoutMs: uint32(h.forwardTimeout.Milliseconds()), }) if err != nil { - return nil, fmt.Errorf("ERROR 2013 (HY000): Lost connection to leader during query: %v", err) + return nil, protocol.NewMySQLError(protocol.ErrCodeUnknown, protocol.SQLStateGeneral, fmt.Sprintf("Lost connection to leader during query: %v", err)) } if !resp.Success { - return nil, fmt.Errorf("ERROR 1105 (HY000): %s", resp.ErrorMessage) + return nil, forwardedError(resp.ErrorCode, resp.SqlState, resp.ErrorMessage) } - session.LastInsertId.Store(resp.LastInsertId) - session.ForwardedTxnActive = resp.InTransaction + h.applyForwardedSessionState(session, resp) if session.WaitForReplication && resp.CommittedTxnId > 0 { if err := h.waitForReplication(session.CurrentDatabase, resp.CommittedTxnId); err != nil { @@ -242,6 +241,55 @@ func (h *ReadOnlyHandler) HandleLoadData(session *protocol.ConnectionSession, sq }, nil } +// forwardedError turns a leader's failure message into the error this replica +// returns to its client. +// +// It is typed rather than a formatted "ERROR 1105 (HY000): ..." string for two +// reasons. A formatted code is re-parsed downstream by +// protocol.ConvertToMySQLError's message matching, so a leader message that +// merely contains "syntax error" came back to the client as 1064; and the +// formatted prefix was duplicated into the ERR packet's message. +// +// forwardedError rebuilds, for this replica's client, the error the leader +// reported. The leader sends both its MySQL error code and its SQLSTATE, so a +// rule rejection's 1235/42000 arrives intact. +// +// Both travel because the code does not determine the SQLSTATE: the leader +// pairs 1105 with HY000 for an unclassified error and with 23000 for an +// unclassified CONSTRAINT failure, so a replica deriving the SQLSTATE from the +// number alone reported HY000 for a genuine integrity violation, and a client +// branching on the SQLSTATE class saw a different answer from a replica than +// from a coordinator for the same failure. +// +// A zero code means "not supplied": an older leader on a partially upgraded +// cluster, or a rejection raised before the statement reached the coordinator. +// That keeps the previous behaviour of reporting ER_UNKNOWN_ERROR. A non-zero +// code with an empty SQLSTATE cannot come from a current leader - forwardFailure +// always fills both - and falls back to HY000 rather than sending an empty +// SQLSTATE to the client. +func forwardedError(code uint32, sqlState string, message string) error { + if code == 0 { + return protocol.NewMySQLError(protocol.ErrCodeUnknown, protocol.SQLStateGeneral, message) + } + if sqlState == "" { + sqlState = protocol.SQLStateGeneral + } + return protocol.NewMySQLError(uint16(code), sqlState, message) +} + +// applyForwardedSessionState copies the session-visible state a leader's +// forwarded response carries back into this session. +// +// The insert id goes through ConnectionSession.RecordInsertId rather than +// storing it directly, because a forwarded zero means the leader's statement +// generated no AUTO_INCREMENT value and MySQL leaves LAST_INSERT_ID() unchanged +// in that case. Storing it unconditionally reset the client's value on every +// INSERT that inserted nothing and on every upsert that updated. +func (h *ReadOnlyHandler) applyForwardedSessionState(session *protocol.ConnectionSession, resp *marmotgrpc.ForwardQueryResponse) { + session.RecordInsertId(resp.LastInsertId) + session.ForwardedTxnActive = resp.InTransaction +} + // handleSystemQuery handles MySQL system variable queries func (h *ReadOnlyHandler) handleSystemQuery(session *protocol.ConnectionSession, stmt protocol.Statement) (*protocol.ResultSet, error) { config := handlers.SystemVarConfig{ @@ -258,7 +306,7 @@ func (h *ReadOnlyHandler) handleSystemQuery(session *protocol.ConnectionSession, // executeLocalRead executes a read query locally func (h *ReadOnlyHandler) executeLocalRead(stmt protocol.Statement, params []interface{}) (*protocol.ResultSet, error) { if stmt.Database == "" { - return nil, fmt.Errorf("ERROR 1046 (3D000): No database selected") + return nil, protocol.NewMySQLError(protocol.ErrCodeNoDB, protocol.SQLStateNoDB, "No database selected") } sqlDB, err := h.dbManager.GetDatabaseConnection(stmt.Database) @@ -365,13 +413,13 @@ func (h *ReadOnlyHandler) handleMarmotCommand(session *protocol.ConnectionSessio func (h *ReadOnlyHandler) forwardMutation(session *protocol.ConnectionSession, stmt protocol.Statement, params []interface{}) (*protocol.ResultSet, error) { client := h.replica.streamClient.GetClient() if client == nil { - return nil, fmt.Errorf("ERROR 2003 (HY000): Not connected to leader") + return nil, protocol.NewMySQLError(protocol.ErrCodeUnknown, protocol.SQLStateGeneral, "Not connected to leader") } // Serialize params using msgpack serializedParams, err := marmotgrpc.SerializeParams(params) if err != nil { - return nil, fmt.Errorf("ERROR 1105 (HY000): Failed to serialize params: %v", err) + return nil, protocol.NewMySQLError(protocol.ErrCodeUnknown, protocol.SQLStateGeneral, fmt.Sprintf("Failed to serialize params: %v", err)) } requestID := session.NextForwardRequestID() @@ -388,17 +436,14 @@ func (h *ReadOnlyHandler) forwardMutation(session *protocol.ConnectionSession, s }) if err != nil { - return nil, fmt.Errorf("ERROR 2013 (HY000): Lost connection to leader during query: %v", err) + return nil, protocol.NewMySQLError(protocol.ErrCodeUnknown, protocol.SQLStateGeneral, fmt.Sprintf("Lost connection to leader during query: %v", err)) } if !resp.Success { - return nil, fmt.Errorf("ERROR 1105 (HY000): %s", resp.ErrorMessage) + return nil, forwardedError(resp.ErrorCode, resp.SqlState, resp.ErrorMessage) } - // Update session state - session.LastInsertId.Store(resp.LastInsertId) - - // Track forwarded transaction state if within a transaction - session.ForwardedTxnActive = resp.InTransaction + // Update session state, including the forwarded transaction flag. + h.applyForwardedSessionState(session, resp) // Handle database operations locally after leader confirms success // These don't replicate via change stream - replica must create/drop locally @@ -438,7 +483,7 @@ func (h *ReadOnlyHandler) forwardMutation(session *protocol.ConnectionSession, s func (h *ReadOnlyHandler) forwardTxnControl(session *protocol.ConnectionSession, stmt protocol.Statement) (*protocol.ResultSet, error) { client := h.replica.streamClient.GetClient() if client == nil { - return nil, fmt.Errorf("ERROR 2003 (HY000): Not connected to leader") + return nil, protocol.NewMySQLError(protocol.ErrCodeUnknown, protocol.SQLStateGeneral, "Not connected to leader") } var txnControl marmotgrpc.ForwardTxnControl @@ -465,10 +510,10 @@ func (h *ReadOnlyHandler) forwardTxnControl(session *protocol.ConnectionSession, }) if err != nil { - return nil, fmt.Errorf("ERROR 2013 (HY000): %v", err) + return nil, protocol.NewMySQLError(protocol.ErrCodeUnknown, protocol.SQLStateGeneral, fmt.Sprintf("Lost connection to leader during query: %v", err)) } if !resp.Success { - return nil, fmt.Errorf("ERROR 1105 (HY000): %s", resp.ErrorMessage) + return nil, forwardedError(resp.ErrorCode, resp.SqlState, resp.ErrorMessage) } // Track forwarded transaction state based on response diff --git a/replica/handler_test.go b/replica/handler_test.go index aa09185a..64f06000 100644 --- a/replica/handler_test.go +++ b/replica/handler_test.go @@ -1,9 +1,17 @@ package replica import ( + "context" + "errors" + "fmt" "os" + "strings" "testing" + "github.com/mattn/go-sqlite3" + marmotgrpc "github.com/maxpert/marmot/grpc" + "google.golang.org/grpc" + "github.com/maxpert/marmot/db" "github.com/maxpert/marmot/hlc" "github.com/maxpert/marmot/protocol" @@ -602,3 +610,379 @@ func BenchmarkHandler_RejectMutation(b *testing.B) { handler.HandleQuery(session, "INSERT INTO users (name) VALUES ('test')", nil) } } + +// TestForwardedInsertIdNeverResetBySentinelZero pins the rule that a forwarded +// insert id of 0 must not overwrite the session's LAST_INSERT_ID(). +// +// A leader reports 0 whenever a statement generated no AUTO_INCREMENT value: an +// INSERT that inserted nothing, an upsert that updated an existing row, a table +// with no auto-increment column. MySQL leaves LAST_INSERT_ID() unchanged there, +// so a replica that stores the 0 wipes a value the client is entitled to keep. +// +// Mutation: make ConnectionSession.RecordInsertId store unconditionally, or +// restore the direct session.LastInsertId.Store(resp.LastInsertId) at either +// call site in applyForwardedSessionState. Every assertion below fires. +func TestForwardedInsertIdNeverResetBySentinelZero(t *testing.T) { + h, _, cleanup := testHandler(t) + defer cleanup() + + session := &protocol.ConnectionSession{} + + // (b) a forwarded non-zero id becomes the session's value. + h.applyForwardedSessionState(session, &marmotgrpc.ForwardQueryResponse{LastInsertId: 41}) + if got := session.LastInsertId.Load(); got != 41 { + t.Fatalf("after a forwarded id of 41, session LAST_INSERT_ID() = %d, want 41", got) + } + + // (b) a later non-zero id replaces it. + h.applyForwardedSessionState(session, &marmotgrpc.ForwardQueryResponse{LastInsertId: 42}) + if got := session.LastInsertId.Load(); got != 42 { + t.Fatalf("after a forwarded id of 42, session LAST_INSERT_ID() = %d, want 42", got) + } + + // (a) a forwarded 0 leaves it alone, checked repeatedly rather than once: + // this is a safety property, so it must hold past its first success. + const noOpForwards = 3 + observed := 0 + for i := 0; i < noOpForwards; i++ { + h.applyForwardedSessionState(session, &marmotgrpc.ForwardQueryResponse{LastInsertId: 0}) + if got := session.LastInsertId.Load(); got != 42 { + t.Fatalf("forward %d of 0 changed session LAST_INSERT_ID() to %d, want it to stay 42", i, got) + } + observed++ + } + // Guards the loop against silently degrading to a single sample. + // Mutation: drop an iteration. + if observed != noOpForwards { + t.Fatalf("the zero-forward path ran %d times, want %d", observed, noOpForwards) + } + + // The unrelated state the same helper carries must still be applied, or the + // guard would have been bought by skipping the whole response. + // Mutation: delete the ForwardedTxnActive assignment. + h.applyForwardedSessionState(session, &marmotgrpc.ForwardQueryResponse{LastInsertId: 0, InTransaction: true}) + if !session.ForwardedTxnActive { + t.Fatal("a forwarded response with InTransaction=true did not set ForwardedTxnActive") + } +} + +// TestSystemQueryReturnsRetainedInsertId is (c): the value a client actually +// reads back. handleSystemQuery is the only path serving LAST_INSERT_ID() on a +// replica, so a guard that held in the session but not here would be invisible. +// +// Mutation: make RecordInsertId store unconditionally; the retained 7 becomes 0. +func TestSystemQueryReturnsRetainedInsertId(t *testing.T) { + h, _, cleanup := testHandler(t) + defer cleanup() + + session := &protocol.ConnectionSession{} + h.applyForwardedSessionState(session, &marmotgrpc.ForwardQueryResponse{LastInsertId: 7}) + h.applyForwardedSessionState(session, &marmotgrpc.ForwardQueryResponse{LastInsertId: 0}) + + stmt := protocol.ParseStatement("SELECT LAST_INSERT_ID()") + rs, err := h.handleSystemQuery(session, stmt) + if err != nil { + t.Fatalf("handleSystemQuery: %v", err) + } + if len(rs.Rows) != 1 || len(rs.Rows[0]) != 1 { + t.Fatalf("SELECT LAST_INSERT_ID() returned %d rows, want one row of one column: %#v", len(rs.Rows), rs.Rows) + } + if got := fmt.Sprintf("%v", rs.Rows[0][0]); got != "7" { + t.Fatalf("SELECT LAST_INSERT_ID() returned %s after a forwarded 0, want the retained 7", got) + } +} + +// TestForwardedErrorCarriesTheLeadersCode pins the whole replica half of the +// forwarded-error path: the leader's MySQL error code reaches the client, a +// leader that sends no code still yields ER_UNKNOWN_ERROR, and the message is +// never a formatted "ERROR (): ..." string. +// +// The formatted form this replaced had two defects. Its code was re-derived +// downstream by protocol.ConvertToMySQLError's message matching, so a leader +// message that merely contained the words "syntax error" arrived as 1064; and +// the formatted prefix was duplicated into the ERR packet's message. +func TestForwardedErrorCarriesTheLeadersCode(t *testing.T) { + cases := []struct { + name string + code uint32 + sqlState string + message string + wantCode uint16 + wantSQLState string + }{ + { + // The deliverable: D1's rule rejection keeps its own code across + // the wire instead of being flattened. + // Mutation: ignore resp.ErrorCode and always report 1105. + name: "a rule rejection keeps 1235", code: 1235, sqlState: protocol.SQLStateSyntax, + message: "This version of MySQL doesn't yet support 'INSERT ... SELECT'", + wantCode: 1235, wantSQLState: protocol.SQLStateSyntax, + }, + { + // The SQLSTATE is not on the wire; it must be derived, and derived + // the same way the coordinator's own ERR path derives it. + // Mutation: return SQLStateGeneral for every code. + name: "a duplicate key keeps 1062 and its integrity SQLSTATE", code: 1062, sqlState: protocol.SQLStateIntegrity, + message: "Duplicate entry '7' for key 'PRIMARY'", + wantCode: 1062, wantSQLState: protocol.SQLStateIntegrity, + }, + { + // Rolling upgrade: an older leader sends no code at all, and + // proto3 delivers that as 0. Behaviour must not change for it. + // Mutation: drop the `code == 0` branch; uint16(0) is reported. + name: "an old leader sending no code still yields 1105", code: 0, sqlState: "", + message: "table is read only", + wantCode: protocol.ErrCodeUnknown, wantSQLState: protocol.SQLStateGeneral, + }, + { + // The leader's wording is data, not a code: it must not be + // re-classified by the words inside it. + // Mutation: restore fmt.Errorf("ERROR 1105 (HY000): %s", message). + name: "a message containing the words syntax error is not re-classified", code: 0, sqlState: "", + message: `near "x": syntax error`, + wantCode: protocol.ErrCodeUnknown, wantSQLState: protocol.SQLStateGeneral, + }, + { + // Defensive: a current leader always fills both fields, so this + // shape can only come from a hand-built response. It must not put + // an empty SQLSTATE in the ERR packet. + // Mutation: drop the `sqlState == ""` fallback; the client gets "". + name: "a non-zero code with no SQLSTATE falls back to HY000", code: 1205, sqlState: "", + message: "Lock wait timeout exceeded", + wantCode: protocol.ErrCodeLockTimeout, wantSQLState: protocol.SQLStateGeneral, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mysqlErr := protocol.ConvertToMySQLError(forwardedError(tc.code, tc.sqlState, tc.message)) + if mysqlErr.Code != tc.wantCode { + t.Errorf("code = %d, want %d", mysqlErr.Code, tc.wantCode) + } + if mysqlErr.SQLState != tc.wantSQLState { + t.Errorf("SQLSTATE = %q, want %q", mysqlErr.SQLState, tc.wantSQLState) + } + // The client must see the leader's message, not one with a code + // spelled into it. + if mysqlErr.Message != tc.message { + t.Errorf("message = %q, want the leader's message %q verbatim", mysqlErr.Message, tc.message) + } + }) + } +} + +// leaderErrors are the shapes a leader's ConvertToMySQLError produces, with the +// (code, SQLSTATE) pair a client connected directly to that leader would see. +// The replica must reproduce the pair exactly; anything else means a client +// gets a different answer from a replica than from a coordinator for the same +// failure. +// +// The generic-constraint row is the one that matters: the leader pairs code +// 1105 with TWO different SQLSTATEs (HY000 for an unclassified error, 23000 for +// an unclassified *constraint* failure), so a replica that rebuilds the SQLSTATE +// from the code alone has to guess, and guesses HY000. +func leaderErrors() []struct { + name string + err error + state string +} { + return []struct { + name string + err error + state string + }{ + {"generic constraint failure", sqlite3.Error{Code: sqlite3.ErrConstraint, ExtendedCode: sqlite3.ErrConstraintTrigger}, protocol.SQLStateIntegrity}, + {"unique violation", sqlite3.Error{Code: sqlite3.ErrConstraint, ExtendedCode: sqlite3.ErrConstraintUnique}, protocol.SQLStateIntegrity}, + {"deadlock", protocol.ErrDeadlock(), protocol.SQLStateDeadlock}, + {"server shutdown", protocol.ErrServerShutdown(), "08S01"}, + {"read only", protocol.ErrReadOnly(), protocol.SQLStateGeneral}, + {"lock timeout", protocol.ErrLockWaitTimeout(), protocol.SQLStateGeneral}, + } +} + +// TestForwardedErrorRoundTripsTheLeadersSQLState pins that the (code, SQLSTATE) +// pair a replica client sees is the pair the leader itself produced. +// +// The two halves of the round trip are pinned separately because they live in +// different packages: TestForwardFailureCarriesTheMySQLCode (package grpc) pins +// that forwardFailure copies protocol.ConvertToMySQLError's output onto the +// wire, and this test pins that forwardedError reproduces that same output. The +// mapper is the shared seam, so neither test re-implements the other's half. +func TestForwardedErrorRoundTripsTheLeadersSQLState(t *testing.T) { + for _, tc := range leaderErrors() { + t.Run(tc.name, func(t *testing.T) { + // What the leader puts on the wire. + leader := protocol.ConvertToMySQLError(tc.err) + if leader.SQLState != tc.state { + t.Fatalf("fixture is wrong: the leader maps this to SQLSTATE %q, not %q", leader.SQLState, tc.state) + } + + // What the replica rebuilds from it. + replicaErr := protocol.ConvertToMySQLError(forwardedError(uint32(leader.Code), leader.SQLState, leader.Message)) + + if replicaErr.Code != leader.Code { + t.Errorf("code = %d, want the leader's %d", replicaErr.Code, leader.Code) + } + // Mutation: derive the SQLSTATE from the code instead of carrying + // it. The generic-constraint row then reports HY000 for 23000. + if replicaErr.SQLState != leader.SQLState { + t.Errorf("SQLSTATE = %q, want the leader's %q", replicaErr.SQLState, leader.SQLState) + } + }) + } +} + +// fakeLeaderClient stands in for the gRPC client a replica uses to reach its +// leader. It embeds the interface so only the two methods the forwarding sites +// actually call need bodies; everything else panics if it is ever reached, +// which is what we want from a stand-in. +type fakeLeaderClient struct { + marmotgrpc.MarmotServiceClient + err error +} + +func (f fakeLeaderClient) ForwardQuery(context.Context, *marmotgrpc.ForwardQueryRequest, ...grpc.CallOption) (*marmotgrpc.ForwardQueryResponse, error) { + return nil, f.err +} + +func (f fakeLeaderClient) ForwardLoadData(context.Context, *marmotgrpc.ForwardLoadDataRequest, ...grpc.CallOption) (*marmotgrpc.ForwardQueryResponse, error) { + return nil, f.err +} + +// handlerWithLeaderClient builds a forwarding-enabled handler whose leader link +// is the given client. A nil client reproduces "not connected"; a client that +// returns an error reproduces "lost connection". Neither needs a live leader. +func handlerWithLeaderClient(t *testing.T, client marmotgrpc.MarmotServiceClient) *ReadOnlyHandler { + t.Helper() + h, _, cleanup := testHandler(t) + t.Cleanup(cleanup) + h.forwardWrites = true + h.replica = &Replica{streamClient: &StreamClient{client: client}} + return h +} + +// TestReplicaEmittedErrorCodesAreServerSide drives every site where this replica +// raises an error of its own and asserts the code and SQLSTATE a client receives. +// +// The rule is that an ERR packet carries server-side codes only. 2003 +// (CR_CONN_HOST_ERROR) and 2013 (CR_SERVER_LOST) are client-library codes that no +// MySQL server emits; a driver seeing one may tear the session down or reconnect +// rather than surface a statement error, and they fire exactly when the leader +// link is down, which is when a replica most wants the client to keep its +// session. Master reported 1105/HY000 for both, with the number as decoration +// inside the message, so 1105 is also the behaviour-preserving choice. +// +// 1046/3D000 stays: ER_NO_DB_ERROR is MySQL's own server code for that condition. +func TestReplicaEmittedErrorCodesAreServerSide(t *testing.T) { + const noDatabase = "" // an unset Database is what reaches executeLocalRead + + session := func() *protocol.ConnectionSession { + return &protocol.ConnectionSession{ConnID: 1, CurrentDatabase: "testdb"} + } + insert := protocol.Statement{SQL: "INSERT INTO users (name) VALUES ('x')", Database: "testdb"} + begin := protocol.Statement{SQL: "BEGIN", Type: protocol.StatementBegin, Database: "testdb"} + + cases := []struct { + name string + client marmotgrpc.MarmotServiceClient + call func(h *ReadOnlyHandler, s *protocol.ConnectionSession) error + wantCode uint16 + wantSQLState string + wantMessage string + }{ + { + name: "HandleLoadData, not connected", client: nil, + call: func(h *ReadOnlyHandler, s *protocol.ConnectionSession) error { + _, err := h.HandleLoadData(s, "LOAD DATA LOCAL INFILE 'x' INTO TABLE users", []byte("1\n")) + return err + }, + wantCode: protocol.ErrCodeUnknown, wantSQLState: protocol.SQLStateGeneral, + wantMessage: "Not connected to leader", + }, + { + name: "forwardMutation, not connected", client: nil, + call: func(h *ReadOnlyHandler, s *protocol.ConnectionSession) error { + _, err := h.forwardMutation(s, insert, nil) + return err + }, + wantCode: protocol.ErrCodeUnknown, wantSQLState: protocol.SQLStateGeneral, + wantMessage: "Not connected to leader", + }, + { + name: "forwardTxnControl, not connected", client: nil, + call: func(h *ReadOnlyHandler, s *protocol.ConnectionSession) error { + _, err := h.forwardTxnControl(s, begin) + return err + }, + wantCode: protocol.ErrCodeUnknown, wantSQLState: protocol.SQLStateGeneral, + wantMessage: "Not connected to leader", + }, + { + name: "HandleLoadData, lost connection", client: fakeLeaderClient{err: errors.New("boom")}, + call: func(h *ReadOnlyHandler, s *protocol.ConnectionSession) error { + _, err := h.HandleLoadData(s, "LOAD DATA LOCAL INFILE 'x' INTO TABLE users", []byte("1\n")) + return err + }, + wantCode: protocol.ErrCodeUnknown, wantSQLState: protocol.SQLStateGeneral, + wantMessage: "Lost connection to leader", + }, + { + name: "forwardMutation, lost connection", client: fakeLeaderClient{err: errors.New("boom")}, + call: func(h *ReadOnlyHandler, s *protocol.ConnectionSession) error { + _, err := h.forwardMutation(s, insert, nil) + return err + }, + wantCode: protocol.ErrCodeUnknown, wantSQLState: protocol.SQLStateGeneral, + wantMessage: "Lost connection to leader", + }, + { + name: "forwardTxnControl, lost connection", client: fakeLeaderClient{err: errors.New("boom")}, + call: func(h *ReadOnlyHandler, s *protocol.ConnectionSession) error { + _, err := h.forwardTxnControl(s, begin) + return err + }, + wantCode: protocol.ErrCodeUnknown, wantSQLState: protocol.SQLStateGeneral, + wantMessage: "Lost connection to leader", + }, + { + // Not a leader-link failure, and deliberately not 1105: MySQL has + // its own server code for this one. + // Mutation: replace ErrCodeNoDB with ErrCodeUnknown. + name: "executeLocalRead, no database selected", client: nil, + call: func(h *ReadOnlyHandler, _ *protocol.ConnectionSession) error { + _, err := h.executeLocalRead(protocol.Statement{Database: noDatabase}, nil) + return err + }, + wantCode: protocol.ErrCodeNoDB, wantSQLState: protocol.SQLStateNoDB, + wantMessage: "No database selected", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := handlerWithLeaderClient(t, tc.client) + + err := tc.call(h, session()) + if err == nil { + t.Fatalf("%s returned no error; the fixture did not reach the failure site", tc.name) + } + + mysqlErr := protocol.ConvertToMySQLError(err) + // Mutation: put a client-side code back at any of these sites - + // 2003 at the not-connected sites, 2013 at the lost-connection + // ones - and the matching row fires here. + if mysqlErr.Code != tc.wantCode { + t.Errorf("code = %d, want %d", mysqlErr.Code, tc.wantCode) + } + if mysqlErr.SQLState != tc.wantSQLState { + t.Errorf("SQLSTATE = %q, want %q", mysqlErr.SQLState, tc.wantSQLState) + } + // Pins that the row reached the site it names rather than some + // other failure that happens to carry the same code. + // Mutation: swap two rows' call functions. + if !strings.Contains(mysqlErr.Message, tc.wantMessage) { + t.Errorf("message = %q, want it to contain %q", mysqlErr.Message, tc.wantMessage) + } + }) + } +} From 1d9d887316f087ca1dad63d54904eb903d1e0c3f Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Thu, 17 Sep 2026 16:17:34 -0500 Subject: [PATCH 02/17] fix(cluster): a lone restarted node must not be its own write quorum Quorum is a majority of TOTAL membership, but the membership registry started with only the local node and was never persisted. A node restarted while its peers were unreachable, or before gossip converged, computed a membership of one, was its own majority, and committed alone. Prerequisite F2 of the narrow-id design. Persisted membership. grpc/membership_snapshot.go stores {node id, address, incarnation, status} for every known node as msgpack in the node's data directory, written atomically (temp file, fsync, rename, directory fsync). The write hangs off updateClusterMetricsLocked, the choke point every membership mutator already calls and the per-heartbeat path does not; a fingerprint over the recorded fields skips the fsync when nothing the snapshot stores changed. An accepted higher-incarnation update that changes only address or incarnation now marks the record replaced so it is persisted too (found in review). Restore at construction. NewNodeRegistryWithDataDir loads the snapshot before the gRPC server serves 2PC. Restored peers enter as SUSPECT: they count toward the denominator and are excluded from replication targets until gossip hears from them. REMOVED stays REMOVED. Incarnations are restored so SWIM refutation keeps working. A missing or corrupt snapshot yields a self-only registry with a warning, never a refusal to boot. Belt. NodeProvider gains HasSeedNodes; GetClusterState refuses QUORUM, ALL and unknown levels with a retryable 1205 while a seed-configured node knows only itself. LOCAL_ONE and ONE are exempt: they make no denominator claim, and refusing them made a booting node fail SELECT 1 and look dead to liveness probes (caught by the integration suite). Single-node deployments without seeds are unchanged. Tests: snapshot round-trip and restore; a seven-step persist walk; fifty heartbeats write nothing; corrupt/absent/no-data-dir cases; address and incarnation change persisted and restored; belt table across all levels; cluster tests TestLoneRestartedNodeRefusesToBeItsOwnQuorum (kill -9, peers stopped, restart alone: "need 2 (majority of 3 total members, 1 alive)") and TestFreshNodeWithUnreachableSeedsRefusesWrites (1205). Five mutation arms fired at named assertions. Unit 24 ok, race clean on grpc/coordinator/cluster, integration 32/32. Known and accepted: the snapshot fsync (~6 ms measured) runs under the registry write lock, only on a real membership change; a peer removed while this node was down inflates the denominator until gossip delivers REMOVED (fail-closed). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- coordinator/cluster.go | 49 +++++ coordinator/cluster_test.go | 96 +++++++++ coordinator/full_replication_test.go | 6 + coordinator/node_provider.go | 10 + coordinator/read_coordinator_test.go | 2 + coordinator/vec_bench_api.go | 4 + grpc/membership_snapshot.go | 140 +++++++++++++ grpc/membership_snapshot_test.go | 296 +++++++++++++++++++++++++++ grpc/node_provider.go | 14 +- grpc/node_registry.go | 172 +++++++++++++++- grpc/server.go | 6 +- marmot.go | 3 +- test/cluster_integration_test.go | 2 +- test/quorum_of_one_test.go | 158 ++++++++++++++ test/verification_test.go | 2 + 15 files changed, 950 insertions(+), 10 deletions(-) create mode 100644 grpc/membership_snapshot.go create mode 100644 grpc/membership_snapshot_test.go create mode 100644 test/quorum_of_one_test.go diff --git a/coordinator/cluster.go b/coordinator/cluster.go index f5d9c0d6..6ea816a8 100644 --- a/coordinator/cluster.go +++ b/coordinator/cluster.go @@ -20,6 +20,36 @@ type ClusterState struct { RequiredQuorum int } +// consistencyNeedsMajority reports whether a consistency level's guarantee +// depends on the total-membership denominator. QUORUM and ALL do; ONE and +// LOCAL_ONE explicitly do not, and an unknown level is treated as needing one, +// matching QuorumSize's own default-for-safety. +func consistencyNeedsMajority(level protocol.ConsistencyLevel) bool { + switch level { + case protocol.ConsistencyLocalOne, protocol.ConsistencyOne: + return false + default: + return true + } +} + +// ErrMembershipNotLearned is returned when a node configured to join a cluster +// still knows only itself, so it cannot tell a healthy single-node deployment +// from a restart that has not yet heard from its peers. +// +// It is ErrCodeLockTimeout (1205) because the condition is transient and the +// client should retry: gossip converges within a few seconds of a peer becoming +// reachable. 1205 is the code ORM retry layers already key on, alongside 1213, +// and FINAL.md section 4 picks it for the same reason on the claim path. A +// non-retryable code would turn a few seconds of convergence into a failed +// transaction the application has to handle itself. +func ErrMembershipNotLearned() *protocol.MySQLError { + return protocol.NewMySQLError( + protocol.ErrCodeLockTimeout, + protocol.SQLStateGeneral, + "cluster membership not yet learned; refusing to act as a quorum of one") +} + // GetClusterState retrieves the current cluster state for DML operations. // It returns a ClusterState with alive nodes, total membership, and required quorum size. // @@ -42,6 +72,25 @@ func GetClusterState(nodeProvider NodeProvider, consistency protocol.Consistency // CRITICAL: Use TOTAL membership for quorum calculation, not just alive nodes totalMembership := nodeProvider.GetTotalMembershipSize() + // A node configured with seed nodes that knows only itself has not learned + // the cluster yet: it either restarted before gossip converged, or booted + // with its seeds unreachable. Quorum of a membership of one is one, so it + // would commit alone and diverge from peers that are up. Refuse, with a + // retryable code, until membership is learned. + // + // This is a belt, independent of the registry's persisted membership: it + // still holds when the snapshot is absent or unreadable, which is exactly + // when the denominator would otherwise be wrong. + // + // It applies ONLY to levels whose guarantee depends on the denominator. + // LOCAL_ONE and ONE ask for no quorum, so a wrong denominator cannot + // mislead them, and refusing there would take out health checks: the + // cluster harness's own liveness probe is a LOCAL_ONE "SELECT 1", and an + // earlier revision of this guard made a booting node look dead to it. + if totalMembership <= 1 && consistencyNeedsMajority(consistency) && nodeProvider.HasSeedNodes() { + return nil, ErrMembershipNotLearned() + } + // Validate consistency level against total membership (not just alive) if err := ValidateConsistencyLevel(consistency, totalMembership); err != nil { return nil, fmt.Errorf("invalid consistency level: %w", err) diff --git a/coordinator/cluster_test.go b/coordinator/cluster_test.go index 349a37b9..050ba923 100644 --- a/coordinator/cluster_test.go +++ b/coordinator/cluster_test.go @@ -2,6 +2,7 @@ package coordinator import ( "fmt" + "strings" "testing" "github.com/maxpert/marmot/protocol" @@ -452,6 +453,8 @@ func TestGetClusterState_SingleNode(t *testing.T) { } // errorNodeProvider is a mock NodeProvider that returns errors +func (e *errorNodeProvider) HasSeedNodes() bool { return false } + type errorNodeProvider struct { err error } @@ -467,3 +470,96 @@ func (e *errorNodeProvider) GetClusterSize() int { func (e *errorNodeProvider) GetTotalMembershipSize() int { return 0 } + +// TestGetClusterStateRefusesQuorumOfOne pins the belt that stops a node which +// was configured to join a cluster from acting as a quorum of one. +// +// A node with seed nodes whose membership is itself alone has not learned the +// cluster yet: it restarted before gossip converged, or booted with its seeds +// unreachable. Quorum of one is one, so it would commit alone and diverge from +// peers that are up. A node with no seeds is a legitimate single-node +// deployment and is unaffected. +func TestGetClusterStateRefusesQuorumOfOne(t *testing.T) { + cases := []struct { + name string + aliveNodes []uint64 + membership int + hasSeedNodes bool + wantErr bool + wantQuorum int + }{ + { + // The defect. Mutation: delete the belt in GetClusterState. + name: "seeds configured, membership of one, refuses", + aliveNodes: []uint64{1}, membership: 1, hasSeedNodes: true, + wantErr: true, + }, + { + // Ruling 4: single-node deployments are unaffected. + // Mutation: drop the HasSeedNodes() condition from the belt; this + // fires, and every single-node deployment stops accepting writes. + name: "no seeds, membership of one, accepted", + aliveNodes: []uint64{1}, membership: 1, hasSeedNodes: false, + wantErr: false, wantQuorum: 1, + }, + { + // Once membership is learned the belt lets go. + // Mutation: make the belt fire on membership <= 2. + name: "seeds configured, membership of two, accepted", + aliveNodes: []uint64{1, 2}, membership: 2, hasSeedNodes: true, + wantErr: false, wantQuorum: 2, + }, + { + name: "seeds configured, membership of three, accepted", + aliveNodes: []uint64{1, 2, 3}, membership: 3, hasSeedNodes: true, + wantErr: false, wantQuorum: 2, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + provider := &mockNodeProvider{ + nodes: tc.aliveNodes, + totalMembership: tc.membership, + hasSeedNodes: tc.hasSeedNodes, + } + + // A level that does not depend on the denominator must be served + // even in the refusing case: an earlier revision of the belt + // refused these too, which made a booting node fail "SELECT 1" and + // look dead to liveness probes. + // Mutation: drop consistencyNeedsMajority from the belt's condition. + for _, level := range []protocol.ConsistencyLevel{protocol.ConsistencyLocalOne, protocol.ConsistencyOne} { + if _, err := GetClusterState(provider, level); err != nil { + t.Errorf("GetClusterState at %v returned %v; levels that ask for no quorum must not be refused", level, err) + } + } + + state, err := GetClusterState(provider, protocol.ConsistencyQuorum) + + if !tc.wantErr { + if err != nil { + t.Fatalf("GetClusterState returned %v, want success", err) + } + if state.RequiredQuorum != tc.wantQuorum { + t.Errorf("RequiredQuorum = %d, want %d", state.RequiredQuorum, tc.wantQuorum) + } + return + } + + if err == nil { + t.Fatal("GetClusterState accepted a membership of one on a node configured with seeds") + } + // The client must be told to retry, not handed a permanent failure: + // gossip converges within seconds of a peer becoming reachable. + // Mutation: return a plain fmt.Errorf, or a non-retryable code. + mysqlErr := protocol.ConvertToMySQLError(err) + if mysqlErr.Code != protocol.ErrCodeLockTimeout { + t.Errorf("code = %d, want %d (ER_LOCK_WAIT_TIMEOUT, retryable)", mysqlErr.Code, protocol.ErrCodeLockTimeout) + } + if !strings.Contains(mysqlErr.Message, "membership") { + t.Errorf("message = %q, want it to name the membership condition", mysqlErr.Message) + } + }) + } +} diff --git a/coordinator/full_replication_test.go b/coordinator/full_replication_test.go index 7a6d6af0..76d576fb 100644 --- a/coordinator/full_replication_test.go +++ b/coordinator/full_replication_test.go @@ -12,7 +12,13 @@ import ( ) // mockNodeProvider implements NodeProvider for testing +// hasSeedNodes drives coordinator.GetClusterState's quorum-of-one belt. The +// zero value is false, which is a single-node deployment: existing tests that +// run with a membership of one keep passing unchanged. +func (m *mockNodeProvider) HasSeedNodes() bool { return m.hasSeedNodes } + type mockNodeProvider struct { + hasSeedNodes bool nodes []uint64 totalMembership int // Total membership for split-brain prevention (0 = use len(nodes)) mu sync.RWMutex diff --git a/coordinator/node_provider.go b/coordinator/node_provider.go index 9e4c1525..68f2ee7e 100644 --- a/coordinator/node_provider.go +++ b/coordinator/node_provider.go @@ -15,4 +15,14 @@ type NodeProvider interface { // to prevent split-brain: quorum must be majority of TOTAL membership, // not just currently reachable nodes. GetTotalMembershipSize() int + + // HasSeedNodes reports whether this node was configured to join an existing + // cluster. It is threaded through the provider rather than read from the + // global config so the coordinator stays testable and has no config + // dependency of its own. + // + // It exists for one decision: a node with seeds whose membership is itself + // alone has not yet learned who its peers are, and must not act as a quorum + // of one. A node without seeds is a legitimate single-node deployment. + HasSeedNodes() bool } diff --git a/coordinator/read_coordinator_test.go b/coordinator/read_coordinator_test.go index 04d04a10..b6e83310 100644 --- a/coordinator/read_coordinator_test.go +++ b/coordinator/read_coordinator_test.go @@ -25,6 +25,8 @@ func (m *mockReaderOld) ReadSnapshot(ctx context.Context, nodeID uint64, req *Re } // Mock NodeProvider for testing (old tests) +func (m *mockNodeProviderOld) HasSeedNodes() bool { return false } + type mockNodeProviderOld struct { nodes []uint64 } diff --git a/coordinator/vec_bench_api.go b/coordinator/vec_bench_api.go index 2315703b..c9d1c4ac 100644 --- a/coordinator/vec_bench_api.go +++ b/coordinator/vec_bench_api.go @@ -25,6 +25,10 @@ func (p *BenchNodeProvider) GetAliveNodes() ([]uint64, error) { return p.nodes, func (p *BenchNodeProvider) GetClusterSize() int { return len(p.nodes) } func (p *BenchNodeProvider) GetTotalMembershipSize() int { return len(p.nodes) } +// HasSeedNodes is false: the benchmark harness is a single-node deployment, so +// the quorum-of-one belt in GetClusterState does not apply to it. +func (p *BenchNodeProvider) HasSeedNodes() bool { return false } + // NewBenchHandler builds a CoordinatorHandler wired only for the read + vector // rewrite paths — suitable for benchmark tooling (cmd/vec-bench). Writes, // DDL, schema-version replication, and node-membership broadcasts are all diff --git a/grpc/membership_snapshot.go b/grpc/membership_snapshot.go new file mode 100644 index 00000000..d7255260 --- /dev/null +++ b/grpc/membership_snapshot.go @@ -0,0 +1,140 @@ +package grpc + +import ( + "fmt" + "os" + "path/filepath" + "sort" + + "github.com/maxpert/marmot/encoding" + "github.com/rs/zerolog/log" +) + +// membershipSnapshotFile is the snapshot's name inside the node's data directory. +const membershipSnapshotFile = "membership.msgpack" + +// membershipSnapshotNode is one node's durable membership record. +// +// Only the fields a restart needs to reconstruct a quorum denominator and to +// keep SWIM's refutation logic working are stored. Liveness is deliberately NOT +// among them: a restored peer's status is decided by the restore, not by what it +// was when the snapshot was written (see restoreMembershipLocked). +type membershipSnapshotNode struct { + NodeID uint64 `msgpack:"node_id"` + Address string `msgpack:"address"` + Incarnation uint64 `msgpack:"incarnation"` + Status int32 `msgpack:"status"` +} + +// membershipSnapshot is the whole durable record. +type membershipSnapshot struct { + Nodes []membershipSnapshotNode `msgpack:"nodes"` +} + +// membershipStore persists cluster membership across restarts. +// +// It exists because quorum is a majority of TOTAL membership, and a registry +// that starts with only itself computes a majority of one. A node that has ever +// known a multi-node membership must not compute a quorum from fewer members +// than it last knew until it has re-learned membership from a live peer, and +// the only way to know what it last knew is to have written it down. +// +// It is a plain file, not the system Pebble store, because the registry is +// built inside the gRPC server (marmot.go initializeGRPCServer) before the +// database manager exists, so no Pebble store is open at that point. +type membershipStore struct { + path string +} + +// newMembershipStore returns a store rooted at dataDir, or nil when dataDir is +// empty. A nil store disables persistence, which is what the in-process tests +// and any embedded use without a data directory want; every method tolerates a +// nil receiver. +func newMembershipStore(dataDir string) *membershipStore { + if dataDir == "" { + return nil + } + return &membershipStore{path: filepath.Join(dataDir, membershipSnapshotFile)} +} + +// save writes the snapshot atomically: a temp file in the same directory, +// fsynced, then renamed over the target, then the directory fsynced so the +// rename itself survives a crash. A half-written snapshot would be worse than +// none, because it would be read back as a smaller membership. +func (s *membershipStore) save(snapshot membershipSnapshot) error { + if s == nil { + return nil + } + + // A stable order keeps the bytes identical for identical membership, so a + // no-op save cannot look like a change to anything watching the file. + sort.Slice(snapshot.Nodes, func(i, j int) bool { + return snapshot.Nodes[i].NodeID < snapshot.Nodes[j].NodeID + }) + + data, err := encoding.Marshal(snapshot) + if err != nil { + return fmt.Errorf("encode membership snapshot: %w", err) + } + + dir := filepath.Dir(s.path) + tmp, err := os.CreateTemp(dir, membershipSnapshotFile+".tmp-*") + if err != nil { + return fmt.Errorf("create temp membership snapshot: %w", err) + } + tmpName := tmp.Name() + + if _, err := tmp.Write(data); err != nil { + tmp.Close() + os.Remove(tmpName) + return fmt.Errorf("write membership snapshot: %w", err) + } + if err := tmp.Sync(); err != nil { + tmp.Close() + os.Remove(tmpName) + return fmt.Errorf("fsync membership snapshot: %w", err) + } + if err := tmp.Close(); err != nil { + os.Remove(tmpName) + return fmt.Errorf("close membership snapshot: %w", err) + } + if err := os.Rename(tmpName, s.path); err != nil { + os.Remove(tmpName) + return fmt.Errorf("rename membership snapshot: %w", err) + } + + // Without this the rename can be lost on a crash even though the file's + // contents were synced. + if dirFile, err := os.Open(dir); err == nil { + _ = dirFile.Sync() + _ = dirFile.Close() + } + return nil +} + +// load reads the snapshot. A missing or unreadable snapshot is not an error the +// caller should act on: it means "nothing known", and the registry starts with +// self only, exactly as it did before this file existed. Returning an error here +// instead would turn a corrupt file into a node that refuses to boot. +func (s *membershipStore) load() (membershipSnapshot, bool) { + if s == nil { + return membershipSnapshot{}, false + } + + data, err := os.ReadFile(s.path) + if err != nil { + if !os.IsNotExist(err) { + log.Warn().Err(err).Str("path", s.path). + Msg("Membership snapshot unreadable; starting with self-only membership") + } + return membershipSnapshot{}, false + } + + var snapshot membershipSnapshot + if err := encoding.Unmarshal(data, &snapshot); err != nil { + log.Warn().Err(err).Str("path", s.path). + Msg("Membership snapshot corrupt; starting with self-only membership") + return membershipSnapshot{}, false + } + return snapshot, true +} diff --git a/grpc/membership_snapshot_test.go b/grpc/membership_snapshot_test.go new file mode 100644 index 00000000..a10883b4 --- /dev/null +++ b/grpc/membership_snapshot_test.go @@ -0,0 +1,296 @@ +package grpc + +import ( + "os" + "path/filepath" + "testing" +) + +// registryAt builds a registry rooted at dir, as production does. +func registryAt(t *testing.T, nodeID uint64, address, dir string) *NodeRegistry { + t.Helper() + return NewNodeRegistryWithDataDir(nodeID, address, dir) +} + +// TestMembershipSurvivesRestart is the defect this step exists to close: a node +// restarted while its peers are unreachable must not compute a quorum from a +// membership of one. +// +// Mutation: delete the restoreMembershipLocked call in the constructor. Count() +// after restart drops to 1 and the first assertion fires. +func TestMembershipSurvivesRestart(t *testing.T) { + dir := t.TempDir() + + first := registryAt(t, 1, "10.0.0.1:7100", dir) + first.Add(&NodeState{NodeId: 2, Address: "10.0.0.2:7100", Status: NodeStatus_ALIVE, Incarnation: 4}) + first.Add(&NodeState{NodeId: 3, Address: "10.0.0.3:7100", Status: NodeStatus_ALIVE, Incarnation: 7}) + if got := first.Count(); got != 3 { + t.Fatalf("fixture: membership before restart = %d, want 3", got) + } + + // A restart: a brand new registry over the same data directory, with no + // gossip and no reachable peer. + restarted := registryAt(t, 1, "10.0.0.1:7100", dir) + + if got := restarted.Count(); got != 3 { + t.Fatalf("membership after restart = %d, want 3; a lone node would be its own quorum", got) + } + + // Liveness is not restored: nothing has been heard from these peers. + // Mutation: restore peers as ALIVE instead of SUSPECT. This fires, and so + // would the cluster's notion of who can receive a write. + if got := restarted.CountAlive(); got != 1 { + t.Errorf("alive after restart = %d, want 1 (self only)", got) + } + for _, id := range []uint64{2, 3} { + node, ok := restarted.Get(id) + if !ok { + t.Fatalf("peer %d missing after restart", id) + } + if node.Status != NodeStatus_SUSPECT { + t.Errorf("peer %d restored as %s, want SUSPECT", id, node.Status) + } + } + + // Incarnation must survive, or SWIM refutation compares against zero and a + // stale rumour can overwrite a peer that has moved on. + // Mutation: drop Incarnation from the snapshot record. + if node, _ := restarted.Get(2); node.Incarnation != 4 { + t.Errorf("peer 2 incarnation = %d, want 4", node.Incarnation) + } + if node, _ := restarted.Get(3); node.Incarnation != 7 { + t.Errorf("peer 3 incarnation = %d, want 7", node.Incarnation) + } + // The address is what a restored node needs in order to gossip back. + // Mutation: drop Address from the snapshot record. + if node, _ := restarted.Get(2); node.Address != "10.0.0.2:7100" { + t.Errorf("peer 2 address = %q, want 10.0.0.2:7100", node.Address) + } +} + +// TestRemovedNodeStaysRemovedAcrossRestart pins that a decommissioned peer does +// not come back as a member and inflate the quorum denominator forever. +// +// Mutation: restore every peer as SUSPECT unconditionally; Count() becomes 3. +func TestRemovedNodeStaysRemovedAcrossRestart(t *testing.T) { + dir := t.TempDir() + + first := registryAt(t, 1, "10.0.0.1:7100", dir) + first.Add(&NodeState{NodeId: 2, Address: "10.0.0.2:7100", Status: NodeStatus_ALIVE}) + first.Add(&NodeState{NodeId: 3, Address: "10.0.0.3:7100", Status: NodeStatus_ALIVE}) + if err := first.MarkRemoved(3); err != nil { + t.Fatalf("MarkRemoved: %v", err) + } + if got := first.Count(); got != 2 { + t.Fatalf("fixture: membership after removal = %d, want 2", got) + } + + restarted := registryAt(t, 1, "10.0.0.1:7100", dir) + + if got := restarted.Count(); got != 2 { + t.Errorf("membership after restart = %d, want 2; a REMOVED peer must not count", got) + } + node, ok := restarted.Get(3) + if !ok { + t.Fatal("removed peer absent from the registry entirely; it must stay REMOVED, not vanish") + } + if node.Status != NodeStatus_REMOVED { + t.Errorf("removed peer restored as %s, want REMOVED", node.Status) + } +} + +// TestMembershipPersistedOnEveryMembershipChange walks the mutators that change +// membership and asserts the snapshot on disk follows each one. It is the +// enumeration, made executable: the persist hook sits on +// updateClusterMetricsLocked, and this is what proves each operation reaches it. +// +// Mutation: move the persist call to TouchLastSeen (persist on heartbeat only). +// Every row fires. +func TestMembershipPersistedOnEveryMembershipChange(t *testing.T) { + dir := t.TempDir() + nr := registryAt(t, 1, "10.0.0.1:7100", dir) + store := newMembershipStore(dir) + + countOnDisk := func(t *testing.T) int { + t.Helper() + snapshot, ok := store.load() + if !ok { + t.Fatal("no membership snapshot on disk") + } + counted := 0 + for _, n := range snapshot.Nodes { + if NodeStatus(n.Status) != NodeStatus_REMOVED && NodeStatus(n.Status) != NodeStatus_LEAVING { + counted++ + } + } + return counted + } + + steps := []struct { + name string + op func() + want int + }{ + {"Add peer 2", func() { nr.Add(&NodeState{NodeId: 2, Address: "a2", Status: NodeStatus_ALIVE}) }, 2}, + {"Add peer 3", func() { nr.Add(&NodeState{NodeId: 3, Address: "a3", Status: NodeStatus_ALIVE}) }, 3}, + {"MarkSuspect 2 (still a member)", func() { nr.MarkSuspect(2) }, 3}, + {"MarkDead 2 (still a member)", func() { nr.MarkDead(2) }, 3}, + {"MarkLeaving 3", func() { _ = nr.MarkLeaving(3) }, 2}, + {"MarkRemoved 2", func() { _ = nr.MarkRemoved(2) }, 1}, + {"Remove 3 entirely", func() { nr.Remove(3) }, 1}, + } + + observed := 0 + for _, step := range steps { + t.Run(step.name, func(t *testing.T) { + step.op() + if got := countOnDisk(t); got != step.want { + t.Errorf("membership on disk = %d, want %d", got, step.want) + } + }) + observed++ + } + // Guards the walk against silently skipping operations. + if observed != len(steps) { + t.Fatalf("walked %d operations, want %d", observed, len(steps)) + } +} + +// TestHeartbeatDoesNotRewriteTheSnapshot pins that the per-heartbeat path stays +// off the persistence path: an fsync per heartbeat would put a disk write on +// gossip's hot path. +// +// Mutation: call persistMembershipLocked from TouchLastSeen, or drop the +// fingerprint guard so any mutator rewrites the file. +func TestHeartbeatDoesNotRewriteTheSnapshot(t *testing.T) { + dir := t.TempDir() + nr := registryAt(t, 1, "10.0.0.1:7100", dir) + nr.Add(&NodeState{NodeId: 2, Address: "a2", Status: NodeStatus_ALIVE}) + + path := filepath.Join(dir, membershipSnapshotFile) + before, err := os.Stat(path) + if err != nil { + t.Fatalf("snapshot missing after Add: %v", err) + } + + const heartbeats = 50 + for i := 0; i < heartbeats; i++ { + nr.TouchLastSeen(2) + } + + after, err := os.Stat(path) + if err != nil { + t.Fatalf("stat after heartbeats: %v", err) + } + if !after.ModTime().Equal(before.ModTime()) || after.Size() != before.Size() { + t.Errorf("%d heartbeats rewrote the snapshot (mtime %v -> %v); persistence must not be on the heartbeat path", + heartbeats, before.ModTime(), after.ModTime()) + } +} + +// TestCorruptOrAbsentSnapshotStartsSelfOnly pins the failure path: a snapshot +// that cannot be read must leave a working self-only registry, never a panic +// and never a refusal to boot. The seed-node belt in the coordinator is what +// still protects a node in this state. +// +// Mutation: return an error from load() and let the constructor propagate it. +func TestCorruptOrAbsentSnapshotStartsSelfOnly(t *testing.T) { + t.Run("absent", func(t *testing.T) { + nr := registryAt(t, 1, "10.0.0.1:7100", t.TempDir()) + if got := nr.Count(); got != 1 { + t.Errorf("membership = %d, want 1", got) + } + }) + + t.Run("corrupt", func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, membershipSnapshotFile), []byte("not msgpack at all"), 0o600); err != nil { + t.Fatalf("writing corrupt snapshot: %v", err) + } + nr := registryAt(t, 1, "10.0.0.1:7100", dir) + if got := nr.Count(); got != 1 { + t.Errorf("membership = %d, want 1", got) + } + // And the node must still be able to learn membership afterwards. + nr.Add(&NodeState{NodeId: 2, Address: "a2", Status: NodeStatus_ALIVE}) + if got := nr.Count(); got != 2 { + t.Errorf("membership after Add = %d, want 2", got) + } + }) + + t.Run("no data directory disables persistence", func(t *testing.T) { + nr := NewNodeRegistry(1, "10.0.0.1:7100") + nr.Add(&NodeState{NodeId: 2, Address: "a2", Status: NodeStatus_ALIVE}) + if got := nr.Count(); got != 2 { + t.Errorf("membership = %d, want 2", got) + } + }) +} + +// TestAddressAndIncarnationChangeIsPersisted covers the gossip update that +// changes a peer's address or incarnation without changing its status: a peer +// that restarted on a new address, or refuted a suspicion, while staying ALIVE. +// +// The higher-incarnation branch of Update replaces the whole record, but for a +// while it only reported a change when the STATUS changed, so these updates were +// applied in memory and never written. A restart then restored the stale address +// and the stale incarnation - and a stale incarnation is worse than a stale +// address, because SWIM compares incarnations to decide who wins, so an older +// rumour could overwrite the record. +// +// Mutation: gate the persist hook on the status change alone again (drop +// recordReplaced from the condition in Update). Every assertion below fires. +func TestAddressAndIncarnationChangeIsPersisted(t *testing.T) { + dir := t.TempDir() + nr := registryAt(t, 1, "10.0.0.1:7100", dir) + nr.Add(&NodeState{NodeId: 2, Address: "10.0.0.2:7100", Status: NodeStatus_ALIVE, Incarnation: 1}) + + // Same status, new address, higher incarnation. + nr.Update(&NodeState{NodeId: 2, Address: "10.0.0.99:7100", Status: NodeStatus_ALIVE, Incarnation: 5}) + + // Precondition: the update was accepted in memory. If it was not, the + // on-disk assertions below would pass for the wrong reason. + inMemory, ok := nr.Get(2) + if !ok { + t.Fatal("peer 2 missing from the registry") + } + if inMemory.Address != "10.0.0.99:7100" || inMemory.Incarnation != 5 { + t.Fatalf("fixture: in-memory record is %s/%d, want 10.0.0.99:7100/5", + inMemory.Address, inMemory.Incarnation) + } + + // On disk. + snapshot, loaded := newMembershipStore(dir).load() + if !loaded { + t.Fatal("no membership snapshot on disk") + } + var found bool + for _, record := range snapshot.Nodes { + if record.NodeID != 2 { + continue + } + found = true + if record.Address != "10.0.0.99:7100" { + t.Errorf("snapshot address = %q, want 10.0.0.99:7100", record.Address) + } + if record.Incarnation != 5 { + t.Errorf("snapshot incarnation = %d, want 5", record.Incarnation) + } + } + if !found { + t.Fatal("peer 2 absent from the snapshot") + } + + // And through a restart, which is what the snapshot exists for. + restarted := registryAt(t, 1, "10.0.0.1:7100", dir) + restoredPeer, ok := restarted.Get(2) + if !ok { + t.Fatal("peer 2 missing after restart") + } + if restoredPeer.Address != "10.0.0.99:7100" { + t.Errorf("restored address = %q, want 10.0.0.99:7100", restoredPeer.Address) + } + if restoredPeer.Incarnation != 5 { + t.Errorf("restored incarnation = %d, want 5; a stale incarnation lets an older rumour win", restoredPeer.Incarnation) + } +} diff --git a/grpc/node_provider.go b/grpc/node_provider.go index 7a744f07..34148945 100644 --- a/grpc/node_provider.go +++ b/grpc/node_provider.go @@ -7,16 +7,24 @@ import ( // GossipNodeProvider implements coordinator.NodeProvider using the gossip protocol's // node registry. It returns all alive nodes in the cluster for full database replication. type GossipNodeProvider struct { - registry *NodeRegistry + registry *NodeRegistry + hasSeedNodes bool } // NewGossipNodeProvider creates a new NodeProvider backed by the gossip node registry -func NewGossipNodeProvider(registry *NodeRegistry) coordinator.NodeProvider { +func NewGossipNodeProvider(registry *NodeRegistry, hasSeedNodes bool) coordinator.NodeProvider { return &GossipNodeProvider{ - registry: registry, + registry: registry, + hasSeedNodes: hasSeedNodes, } } +// HasSeedNodes reports whether this node was configured to join an existing +// cluster. See coordinator.NodeProvider for why the coordinator needs it. +func (gnp *GossipNodeProvider) HasSeedNodes() bool { + return gnp.hasSeedNodes +} + // GetAliveNodes returns all ALIVE nodes for replication func (gnp *GossipNodeProvider) GetAliveNodes() ([]uint64, error) { allNodes := gnp.registry.GetReplicationEligible() diff --git a/grpc/node_registry.go b/grpc/node_registry.go index d43c9430..eb271eb2 100644 --- a/grpc/node_registry.go +++ b/grpc/node_registry.go @@ -1,7 +1,10 @@ package grpc import ( + "encoding/binary" "fmt" + "hash/fnv" + "sort" "sync" "time" @@ -43,10 +46,34 @@ type NodeRegistry struct { onNodeDeadFunc func(*NodeState) // Callback when node transitions to DEAD onNodeLeavingFunc func() // Callback when local node marked LEAVING via remote decommission callbackMu sync.RWMutex + + // store persists membership so a restarted node does not compute a quorum + // from a membership of one. nil disables persistence. + store *membershipStore + // persistedFingerprint is the membership the snapshot on disk describes. + // It lets the persist hook skip the write when a mutator changed something + // the snapshot does not record, so an fsync only happens on a real change. + persistedFingerprint uint64 } -// NewNodeRegistry creates a new node registry +// NewNodeRegistry creates a new node registry with no durable membership. +// Used by tests and by any embedding that has no data directory; production +// goes through NewNodeRegistryWithDataDir so a restart can re-learn what the +// cluster looked like. func NewNodeRegistry(localNodeID uint64, advertiseAddress string) *NodeRegistry { + return NewNodeRegistryWithDataDir(localNodeID, advertiseAddress, "") +} + +// NewNodeRegistryWithDataDir creates a registry that persists membership under +// dataDir and restores it before returning. +// +// Restoring matters because quorum is a majority of TOTAL membership. A registry +// that starts with self only makes a lone restarted node its own majority, so it +// can commit a write - or, once narrow id ranges exist, grant itself a range - +// that overlaps a live peer's. The invariant this establishes: a node that has +// ever known a multi-node membership does not compute a quorum from fewer +// members than it last knew, until gossip re-learns membership from a live peer. +func NewNodeRegistryWithDataDir(localNodeID uint64, advertiseAddress string, dataDir string) *NodeRegistry { log.Debug(). Uint64("node_id", localNodeID). Str("advertise_address", advertiseAddress). @@ -57,6 +84,7 @@ func NewNodeRegistry(localNodeID uint64, advertiseAddress string) *NodeRegistry localNodeID: localNodeID, nodes: make(map[uint64]*NodeState), lastSeen: make(map[uint64]time.Time), + store: newMembershipStore(dataDir), } // Add self to registry as ALIVE @@ -69,11 +97,20 @@ func NewNodeRegistry(localNodeID uint64, advertiseAddress string) *NodeRegistry } nr.lastSeen[localNodeID] = now - // Initialize cluster metrics with self nr.mu.Lock() + restored := nr.restoreMembershipLocked() + // Initialize cluster metrics with the membership we start from. nr.updateClusterMetricsLocked() nr.mu.Unlock() + if restored > 0 { + log.Info(). + Uint64("node_id", localNodeID). + Int("restored_peers", restored). + Int("membership", nr.Count()). + Msg("BOOT: Restored persisted cluster membership; peers start SUSPECT until gossip confirms them") + } + log.Debug(). Uint64("node_id", localNodeID). Str("timestamp", time.Now().Format("15:04:05.000")). @@ -127,6 +164,15 @@ func (nr *NodeRegistry) Update(node *NodeState) { // Rule 4: Apply SWIM state update rules stateChanged := false + // recordReplaced is separate from stateChanged because the higher-incarnation + // branch swaps the whole record: address and incarnation can change with the + // status staying put. Persistence must follow the record, not just the + // status - a peer that restarted on a new address, or refuted a suspicion, + // otherwise stayed at its old address and incarnation on disk, and a restart + // restored the stale one. The stale incarnation is the dangerous half: SWIM + // compares incarnations to decide who wins, so an older rumour could then + // overwrite the record. + recordReplaced := false if node.Incarnation > existing.Incarnation { // Higher incarnation always wins, EXCEPT: // - REMOVED status is sticky (can only be cleared via admin API AllowRejoin) @@ -151,6 +197,7 @@ func (nr *NodeRegistry) Update(node *NodeState) { Uint64("new_inc", node.Incarnation). Msg("REGISTRY: Updating node (higher incarnation)") nr.nodes[node.NodeId] = node + recordReplaced = true // Record state transition if status changed if oldStatus != node.Status { @@ -189,8 +236,10 @@ func (nr *NodeRegistry) Update(node *NodeState) { } // Ignore updates with same/older incarnation that don't escalate - // Update cluster metrics if state changed - if stateChanged { + // Refresh metrics and persist when the record changed at all. The persist + // hook fingerprints exactly the fields the snapshot stores, so a call here + // that changed nothing it records performs no write. + if stateChanged || recordReplaced { nr.updateClusterMetricsLocked() } @@ -873,6 +922,121 @@ func (nr *NodeRegistry) updateClusterMetricsLocked() { } else { telemetry.ClusterQuorumAvailable.Set(0) } + + nr.persistMembershipLocked() +} + +// membershipFingerprintLocked summarises exactly the state the snapshot records, +// so the persist hook can tell a real membership change from a mutator that +// touched something the snapshot does not store. +func (nr *NodeRegistry) membershipFingerprintLocked() uint64 { + ids := make([]uint64, 0, len(nr.nodes)) + for id := range nr.nodes { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + + h := fnv.New64a() + var buf [8]byte + write := func(v uint64) { + binary.LittleEndian.PutUint64(buf[:], v) + _, _ = h.Write(buf[:]) + } + for _, id := range ids { + node := nr.nodes[id] + write(id) + write(uint64(node.Status)) + write(node.Incarnation) + _, _ = h.Write([]byte(node.Address)) + } + return h.Sum64() +} + +// persistMembershipLocked writes the snapshot when the membership actually +// changed. It runs under the registry's write lock, which is acceptable because +// it is reached only on a real membership change - never from TouchLastSeen, +// the per-heartbeat path - and a synchronous write has no window in which a +// crash loses the very change that would have widened the quorum denominator. +func (nr *NodeRegistry) persistMembershipLocked() { + if nr.store == nil { + return + } + + fingerprint := nr.membershipFingerprintLocked() + if fingerprint == nr.persistedFingerprint { + return + } + + snapshot := membershipSnapshot{Nodes: make([]membershipSnapshotNode, 0, len(nr.nodes))} + for id, node := range nr.nodes { + snapshot.Nodes = append(snapshot.Nodes, membershipSnapshotNode{ + NodeID: id, + Address: node.Address, + Incarnation: node.Incarnation, + Status: int32(node.Status), + }) + } + + if err := nr.store.save(snapshot); err != nil { + // Not fatal: the running node's in-memory membership is still correct. + // The cost is that a restart falls back to self-only membership, which + // the seed-node belt in coordinator.GetClusterState still refuses to + // treat as a quorum. + log.Error().Err(err).Msg("Failed to persist cluster membership snapshot") + return + } + nr.persistedFingerprint = fingerprint +} + +// restoreMembershipLocked loads the persisted membership and returns how many +// peers it restored. Caller holds the write lock and has already added self. +// +// Restored peers enter as SUSPECT rather than at their persisted status: this +// node has heard from none of them since booting, so their liveness is unknown. +// SUSPECT counts toward TOTAL membership (Count) and is excluded from +// GetAliveNodes, which is exactly the fail-closed shape wanted - the quorum +// denominator is restored immediately while nothing is treated as reachable +// until gossip says so. +// +// REMOVED stays REMOVED, so a decommissioned peer does not come back as a +// member. Incarnation is restored because SWIM refutation compares incarnations: +// Update() rejects an update whose incarnation is not newer (see the SWIM rules +// in Update), so a peer that has moved on refutes our stale record and wins, +// while a restored record is not silently overwritten by an older rumour. +func (nr *NodeRegistry) restoreMembershipLocked() int { + snapshot, ok := nr.store.load() + if !ok { + return 0 + } + + restored := 0 + for _, record := range snapshot.Nodes { + if record.NodeID == nr.localNodeID { + // Self is ALIVE by construction and was added by the caller. + continue + } + + status := NodeStatus_SUSPECT + if NodeStatus(record.Status) == NodeStatus_REMOVED { + status = NodeStatus_REMOVED + } + + nr.nodes[record.NodeID] = &NodeState{ + NodeId: record.NodeID, + Address: record.Address, + Status: status, + Incarnation: record.Incarnation, + } + // Deliberately NOT time.Now(): a restored peer has not been seen, and + // dating it now would delay the failure detector by a full timeout. + nr.lastSeen[record.NodeID] = time.Time{} + restored++ + } + + // The snapshot we just read is what is on disk; record it so an unchanged + // membership does not rewrite the same bytes on the first mutation. + nr.persistedFingerprint = nr.membershipFingerprintLocked() + return restored } // QuorumInfo returns quorum calculation information diff --git a/grpc/server.go b/grpc/server.go index 99496cec..f66efbc7 100644 --- a/grpc/server.go +++ b/grpc/server.go @@ -79,6 +79,10 @@ type ServerConfig struct { Address string Port int AdvertiseAddress string + // DataDir is where the node registry persists cluster membership so a + // restart does not compute a quorum from a membership of one. Empty + // disables persistence (tests, embedded use without a data directory). + DataDir string } // NewServer creates a new gRPC server @@ -93,7 +97,7 @@ func NewServer(config ServerConfig) (*Server, error) { } // Initialize components with advertise address - s.registry = NewNodeRegistry(config.NodeID, config.AdvertiseAddress) + s.registry = NewNodeRegistryWithDataDir(config.NodeID, config.AdvertiseAddress, config.DataDir) s.gossip = NewGossipProtocol(config.NodeID, s.registry) // Set up callback to connect to nodes when they become ALIVE diff --git a/marmot.go b/marmot.go index a0fd736b..f9e6a2da 100644 --- a/marmot.go +++ b/marmot.go @@ -419,7 +419,7 @@ func main() { // Phase 7: Setup transaction coordinators for full database replication log.Info().Msg("Setting up transaction coordinators") - nodeProvider := marmotgrpc.NewGossipNodeProvider(gossip.GetNodeRegistry()) + nodeProvider := marmotgrpc.NewGossipNodeProvider(gossip.GetNodeRegistry(), len(cfg.Config.Cluster.SeedNodes) > 0) replicator := marmotgrpc.NewGRPCReplicator(client) writeTimeout := time.Duration(cfg.Config.Replication.WriteTimeoutMS) * time.Millisecond @@ -609,6 +609,7 @@ func initializeGRPCServer() (*marmotgrpc.Server, error) { Address: cfg.Config.Cluster.GRPCBindAddress, Port: cfg.Config.Cluster.GRPCPort, AdvertiseAddress: cfg.Config.Cluster.GRPCAdvertiseAddress, + DataDir: cfg.Config.DataDir, } server, err := marmotgrpc.NewServer(config) diff --git a/test/cluster_integration_test.go b/test/cluster_integration_test.go index 466ded03..d5441f79 100644 --- a/test/cluster_integration_test.go +++ b/test/cluster_integration_test.go @@ -456,7 +456,7 @@ func startNode(t *testing.T, node *testNode, seedNodes []string) { node.antiEntropy.Start() // Setup coordinators - nodeProvider := marmotgrpc.NewGossipNodeProvider(gossip.GetNodeRegistry()) + nodeProvider := marmotgrpc.NewGossipNodeProvider(gossip.GetNodeRegistry(), false) replicator := marmotgrpc.NewGRPCReplicator(client) localReplicator := db.NewLocalReplicator(node.nodeID, dbMgr, node.clock) diff --git a/test/quorum_of_one_test.go b/test/quorum_of_one_test.go new file mode 100644 index 00000000..b948f14d --- /dev/null +++ b/test/quorum_of_one_test.go @@ -0,0 +1,158 @@ +package test + +import ( + "fmt" + "strings" + "testing" + "time" +) + +// TestLoneRestartedNodeRefusesToBeItsOwnQuorum is F2: a node restarted while its +// peers are unreachable must not commit alone. +// +// Before this step the registry started with self only and was never persisted, +// so a restarted node computed a total membership of 1, was its own majority, +// and committed writes its peers never saw. Two mechanisms now prevent it: the +// registry restores its last known membership from disk, so the quorum +// denominator is 3 rather than 1; and, when no snapshot is readable, the +// coordinator refuses to act as a quorum of one on any node configured with +// seeds. +// +// Mutation: drop restoreMembershipLocked in the registry constructor AND the +// belt in GetClusterState. The lone node accepts the write and the row-count +// assertion below fires. +func TestLoneRestartedNodeRefusesToBeItsOwnQuorum(t *testing.T) { + harness := NewClusterHarness(t) + defer harness.Cleanup() + + if err := harness.StartCluster(); err != nil { + t.Fatalf("Failed to start cluster: %v", err) + } + + const table = "f2_quorum" + allNodes := []int{1, 2, 3} + + if _, err := harness.ExecNode(1, fmt.Sprintf( + "CREATE TABLE %s (id INT PRIMARY KEY, v TEXT)", table)); err != nil { + t.Fatalf("CREATE TABLE: %v", err) + } + if err := harness.WaitForTableExists(table, allNodes, 15*time.Second); err != nil { + harness.dumpNodeLogs("f2_ddl") + t.Fatalf("DDL did not replicate: %v", err) + } + + if _, err := harness.ExecNode(1, fmt.Sprintf( + "INSERT INTO %s (id, v) VALUES (1, 'before')", table)); err != nil { + t.Fatalf("initial INSERT: %v", err) + } + if err := harness.WaitForRowCount(table, allNodes, 1, 15*time.Second); err != nil { + harness.dumpNodeLogs("f2_initial") + t.Fatalf("initial row did not replicate: %v", err) + } + + // Node 3 dies abruptly, then its peers go away. This is the window the + // defect lived in: node 3 comes back knowing nothing and reaching no one. + if err := harness.KillNode(3); err != nil { + t.Fatalf("KillNode(3): %v", err) + } + if err := harness.StopNode(1); err != nil { + t.Fatalf("StopNode(1): %v", err) + } + if err := harness.StopNode(2); err != nil { + t.Fatalf("StopNode(2): %v", err) + } + + if err := harness.StartNode(3); err != nil { + t.Fatalf("StartNode(3): %v", err) + } + if err := harness.WaitForAlive(3, 30*time.Second); err != nil { + harness.dumpNodeLogs("f2_restart") + t.Fatalf("node 3 did not come up alone: %v", err) + } + + // The invariant. Whichever mechanism fires - a restored denominator of 3 + // that cannot be met, or the seed-node belt - the write must be refused. + _, err := harness.ExecNode(3, fmt.Sprintf( + "INSERT INTO %s (id, v) VALUES (2, 'alone')", table)) + if err == nil { + harness.dumpNodeLogs("f2_lone_accepted") + t.Fatal("a lone restarted node accepted a write: it acted as its own quorum") + } + t.Logf("lone node refused the write, as required: %v", err) + + // And it did not write locally either. An error returned after a local + // commit would still have diverged from the peers that were down. + // Mutation: refuse only after applying locally. + if got := harness.getRowCount(3, table); got != 1 { + t.Errorf("row count on the lone node = %d, want 1; the refused write must not have been applied", got) + } + + // Once the cluster is back, the same statement must succeed and replicate: + // the refusal is transient, not a node that has wedged itself. + // Mutation: make the refusal permanent (e.g. latch it); this fires. + if err := harness.StartNode(1); err != nil { + t.Fatalf("StartNode(1): %v", err) + } + if err := harness.StartNode(2); err != nil { + t.Fatalf("StartNode(2): %v", err) + } + if err := harness.WaitForClusterConvergence(60 * time.Second); err != nil { + harness.dumpNodeLogs("f2_reconverge") + t.Fatalf("cluster did not reconverge: %v", err) + } + + var insertErr error + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + _, insertErr = harness.ExecNode(3, fmt.Sprintf( + "INSERT INTO %s (id, v) VALUES (2, 'after')", table)) + if insertErr == nil { + break + } + time.Sleep(time.Second) + } + if insertErr != nil { + harness.dumpNodeLogs("f2_after_reconverge") + t.Fatalf("INSERT still refused after the cluster reconverged: %v", insertErr) + } + if err := harness.WaitForRowCount(table, allNodes, 2, 30*time.Second); err != nil { + harness.dumpNodeLogs("f2_after_replicate") + t.Fatalf("the write did not replicate after reconvergence: %v", err) + } +} + +// TestFreshNodeWithUnreachableSeedsRefusesWrites covers the belt on its own: a +// node that has never known any membership, so there is no snapshot to restore, +// and whose seeds are down. Its membership is genuinely one, and the only thing +// distinguishing it from a legitimate single-node deployment is that it was +// configured with seeds. +// +// Mutation: drop the HasSeedNodes() condition and this passes for the wrong +// reason; drop the belt entirely and the write is accepted. +func TestFreshNodeWithUnreachableSeedsRefusesWrites(t *testing.T) { + harness := NewClusterHarness(t) + defer harness.Cleanup() + + // Start only node 2, whose configured seeds (node 1) are never started. + if err := harness.StartNode(2); err != nil { + t.Fatalf("StartNode(2): %v", err) + } + if err := harness.WaitForAlive(2, 30*time.Second); err != nil { + harness.dumpNodeLogs("f2_fresh_boot") + t.Fatalf("node 2 did not come up: %v", err) + } + + _, err := harness.ExecNode(2, "CREATE TABLE f2_fresh (id INT PRIMARY KEY, v TEXT)") + if err == nil { + // A DDL is a quorum write too; if it is accepted the belt did not fire. + harness.dumpNodeLogs("f2_fresh_accepted") + t.Fatal("a fresh node with unreachable seeds accepted a write") + } + + // The client must be told to retry: the seeds may come up at any moment. + // Mutation: return a non-retryable code from ErrMembershipNotLearned. + if !strings.Contains(err.Error(), "1205") && !strings.Contains(strings.ToLower(err.Error()), "membership") { + t.Errorf("error %q names neither the retryable code 1205 nor the membership condition", err) + } + t.Logf("fresh node with unreachable seeds refused the write: %v", err) +} diff --git a/test/verification_test.go b/test/verification_test.go index 9700bef3..60aae32f 100644 --- a/test/verification_test.go +++ b/test/verification_test.go @@ -19,6 +19,8 @@ import ( ) // MockNodeProvider implements coordinator.NodeProvider +func (m *MockNodeProvider) HasSeedNodes() bool { return false } + type MockNodeProvider struct { nodes []uint64 } From ef6d388f610f91bfe93e8e2ea124f143455ee449 Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Thu, 17 Sep 2026 18:45:56 -0500 Subject: [PATCH 03/17] feat(schema): record declared MySQL integer widths beside SQLite's INTEGER Step 3a of the narrow AUTO_INCREMENT id design. Marmot collapses every MySQL integer type to SQLite's INTEGER so INTEGER PRIMARY KEY keeps aliasing rowid, which is what LAST_INSERT_ID() depends on. That collapse lost the declared width, so a 53-bit id could be minted into an INT column. This change keeps the width where SQLite preserves it verbatim: in the CREATE TABLE text held by sqlite_master. Marker grammar. protocol/query/transform/intmarker defines `/*M:[u][a]*/` (bits 8/16/24/32, u = UNSIGNED, a = explicit AUTO_INCREMENT), Encode, a column-scoped Decode over sqlite_master text that skips string literals, quoted identifiers and table constraints, Strip, and the signed/unsigned widthMax table. BIGINT emits no marker and its DDL text is byte-identical to before. INTEGER spelled out is a MySQL synonym for INT and is marked as 32 bits. Emit. CREATE TABLE and ALTER TABLE ADD/MODIFY/CHANGE share one collapse, collapseIntegerTypeWithMarker, emitting `INTEGER /*M:..*/`. The ALTER path previously never collapsed integer types at all: ADD COLUMN age INT AUTO_INCREMENT reached SQLite as an unparseable keyword. Read. The schema cache decodes markers from CreateSQL, never from PRAGMA table_info, which normalises the comment away; ColumnSchema carries DeclaredWidth/Unsigned/ExplicitAutoInc and the transpiler SchemaInfo carries the auto-increment column's width and signedness for the allocator that follows on this branch. Client surface. SHOW CREATE TABLE strips the marker; SHOW COLUMNS and information_schema.COLUMNS read PRAGMA and are unchanged. Known limitation: during a rolling upgrade an old binary's SHOW CREATE TABLE shows the marker verbatim for a table created by a new node. Error codes. protocol/mysqlcode is a leaf package holding the MySQL error codes and SQLSTATEs so rule packages can name a code without importing protocol; protocol and transform alias it, and the bare 1064/HY000/08S01 literals in protocol/server.go, mysql_errors.go and error_mapper.go are replaced. Tests: encode/decode across width x signedness x autoinc; decode against hand-written DDL with DEFAULT (expr), string literals, quoted identifiers and table constraints; rowid aliasing with a marker present; ALTER shapes; schema-cache read from CreateSQL vs PRAGMA; SHOW CREATE strip; transpiler schema width/unsigned selection with a decoy narrow column first. Four mutation arms fired at named assertions. Unit 26 ok, race clean across protocol/..., integration 32/32, benchmarks with identical allocs/op. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- db/database_manager.go | 9 + db/database_manager_transpiler_schema_test.go | 86 ++++++ db/schema.go | 15 + db/schema_cache.go | 29 ++ db/width_marker_test.go | 149 ++++++++++ protocol/error_mapper.go | 2 +- protocol/handlers/metadata.go | 14 +- protocol/handlers/metadata_marker_test.go | 72 +++++ protocol/mysql_errors.go | 65 +++-- protocol/mysqlcode/mysqlcode.go | 46 +++ protocol/query/pipeline_ddl_having_test.go | 12 +- .../transform/alter_table_column_type.go | 5 + .../transform/alter_table_column_type_test.go | 90 ++++++ protocol/query/transform/errors.go | 15 +- protocol/query/transform/int_type.go | 22 +- protocol/query/transform/int_type_test.go | 151 +++++++++- protocol/query/transform/intmarker/decode.go | 268 ++++++++++++++++++ .../query/transform/intmarker/intmarker.go | 131 +++++++++ .../transform/intmarker/intmarker_test.go | 215 ++++++++++++++ protocol/query/transform/rule.go | 9 + protocol/query/transform/table_utils.go | 64 +++++ protocol/server.go | 16 +- 22 files changed, 1409 insertions(+), 76 deletions(-) create mode 100644 db/width_marker_test.go create mode 100644 protocol/handlers/metadata_marker_test.go create mode 100644 protocol/mysqlcode/mysqlcode.go create mode 100644 protocol/query/transform/alter_table_column_type_test.go create mode 100644 protocol/query/transform/intmarker/decode.go create mode 100644 protocol/query/transform/intmarker/intmarker.go create mode 100644 protocol/query/transform/intmarker/intmarker_test.go diff --git a/db/database_manager.go b/db/database_manager.go index 4e548fc3..cde2959e 100644 --- a/db/database_manager.go +++ b/db/database_manager.go @@ -1411,6 +1411,15 @@ func (dm *DatabaseManager) GetTranspilerSchema(database, table string) (*transfo AutoIncrementColumn: autoIncCol, AutoIncrementOrdinal: columnOrdinal(schema.Columns, autoIncCol), } + // The declared width comes from the marker the transpiler wrote into the + // CREATE TABLE text; a column without one keeps the 64-bit path. + for _, col := range schema.FullColumns { + if autoIncCol != "" && strings.EqualFold(col.Name, autoIncCol) { + info.AutoIncrementWidth = col.DeclaredWidth + info.AutoIncrementUnsigned = col.Unsigned + break + } + } // PrimaryKeys uses "rowid" sentinel when no explicit PRIMARY KEY is defined. // Keep it out of transpiler conflict targeting to preserve fallback behavior. diff --git a/db/database_manager_transpiler_schema_test.go b/db/database_manager_transpiler_schema_test.go index 3482a09c..74d0a782 100644 --- a/db/database_manager_transpiler_schema_test.go +++ b/db/database_manager_transpiler_schema_test.go @@ -134,3 +134,89 @@ func TestGetTranspilerSchema_AutoIncrementOrdinal(t *testing.T) { t.Fatalf("AutoIncrementOrdinal = %d, want -1", info.AutoIncrementOrdinal) } } + +// TestGetTranspilerSchema_AutoIncrementWidth pins the fields the transpiler's +// id injection will read: the declared width and UNSIGNED flag of the +// AUTO-INCREMENT column specifically, not of whichever narrow column happens to +// come first. +// +// The column order here is deliberate. "q" is narrow (16-bit, signed) and sits +// BEFORE the auto-increment column, so a selection loop that took the first +// marked column instead of the named one would report 16/false where the answer +// is 32/true. With the columns the other way round that mistake would be +// invisible. +func TestGetTranspilerSchema_AutoIncrementWidth(t *testing.T) { + dm, _ := setupTestDatabaseManager(t) + defer dm.Close() + + if err := dm.CreateDatabase("testdb"); err != nil { + t.Fatalf("CreateDatabase failed: %v", err) + } + mdb, err := dm.GetDatabase("testdb") + if err != nil { + t.Fatalf("GetDatabase failed: %v", err) + } + cache, ok := mdb.GetSchemaCache().(*SchemaCache) + if !ok || cache == nil { + t.Fatal("schema cache is unavailable") + } + + // "CREATE TABLE marked (q SMALLINT, id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY)" + // as the schema cache holds it after parsing the width markers. + cache.Update("marked", &TableSchema{ + Columns: []string{"q", "id"}, + PrimaryKeys: []string{"id"}, + AutoIncrementCol: "id", + FullColumns: []ColumnSchema{ + {Name: "q", Type: "INTEGER", DeclaredWidth: 16}, + {Name: "id", Type: "INTEGER", DeclaredWidth: 32, Unsigned: true, ExplicitAutoInc: true, IsPK: true, PKOrder: 1}, + }, + }) + // A BIGINT auto-increment column carries no marker and keeps the 64-bit path. + cache.Update("wide", &TableSchema{ + Columns: []string{"id", "v"}, + PrimaryKeys: []string{"id"}, + AutoIncrementCol: "id", + FullColumns: []ColumnSchema{ + {Name: "id", Type: "INTEGER", IsPK: true, PKOrder: 1}, + {Name: "v", Type: "TEXT"}, + }, + }) + + // Mutation: have the selection loop in GetTranspilerSchema take the first + // column with a non-zero DeclaredWidth instead of matching + // AutoIncrementColumn by name. Both assertions below fire, reporting q's + // 16/false in place of id's 32/true. + info, err := dm.GetTranspilerSchema("testdb", "marked") + if err != nil { + t.Fatalf("GetTranspilerSchema failed: %v", err) + } + if info.AutoIncrementWidth != 32 { + t.Errorf("AutoIncrementWidth = %d, want 32 (id's width, not q's)", info.AutoIncrementWidth) + } + if !info.AutoIncrementUnsigned { + t.Error("AutoIncrementUnsigned = false, want true (id is UNSIGNED, q is not)") + } + // The ordinal must still point at the auto-increment column, which is + // second here. Mutation: return the index of the marked column found by + // the loop above instead of columnOrdinal's result. + if info.AutoIncrementOrdinal != 1 { + t.Errorf("AutoIncrementOrdinal = %d, want 1 (id is the second column)", info.AutoIncrementOrdinal) + } + if info.AutoIncrementColumn != "id" { + t.Errorf("AutoIncrementColumn = %q, want id", info.AutoIncrementColumn) + } + + // Unmarked means the 64-bit path, not a default width. + // Mutation: default AutoIncrementWidth to 64 when no marker is present. + wide, err := dm.GetTranspilerSchema("testdb", "wide") + if err != nil { + t.Fatalf("GetTranspilerSchema failed: %v", err) + } + if wide.AutoIncrementWidth != 0 { + t.Errorf("AutoIncrementWidth = %d for an unmarked BIGINT column, want 0", wide.AutoIncrementWidth) + } + if wide.AutoIncrementUnsigned { + t.Error("AutoIncrementUnsigned = true for an unmarked column, want false") + } +} diff --git a/db/schema.go b/db/schema.go index c4f01474..b7b166b7 100644 --- a/db/schema.go +++ b/db/schema.go @@ -22,6 +22,21 @@ type ColumnSchema struct { Nullable bool // true if NULL allowed (notnull column == 0) IsPK bool // true if part of primary key (pk > 0) PKOrder int // 1-based order in composite PK (0 if not PK) + + // DeclaredWidth is the MySQL integer width the column was declared with: + // 8, 16, 24 or 32. Zero means the column carries no width marker, which is + // every column declared BIGINT and every table created before markers + // existed; those keep the 64-bit path. + // + // It cannot come from PRAGMA table_info, which reports the type as plain + // INTEGER with the marker normalised away. Only sqlite_master.sql keeps it. + DeclaredWidth int + // Unsigned mirrors the MySQL UNSIGNED modifier. Meaningless when + // DeclaredWidth is zero. + Unsigned bool + // ExplicitAutoInc records that the column was declared AUTO_INCREMENT, + // as opposed to merely being a narrow integer. + ExplicitAutoInc bool } // ToPublisherSchema converts db.TableSchema to publisher.TableSchema. diff --git a/db/schema_cache.go b/db/schema_cache.go index a3b1e54b..3c4e1f54 100644 --- a/db/schema_cache.go +++ b/db/schema_cache.go @@ -6,6 +6,7 @@ package db import ( "database/sql/driver" "fmt" + "github.com/maxpert/marmot/protocol/query/transform/intmarker" "io" "strings" "sync" @@ -192,6 +193,33 @@ func (c *SchemaCache) Update(tableName string, schema *TableSchema) { c.cache[tableName] = schema } +// applyWidthMarkers copies each column's declared MySQL integer width out of +// the CREATE TABLE text and onto its ColumnSchema. +// +// The text is the only place the width survives: the transpiler collapses +// TINYINT..INT to SQLite's single INTEGER type and records the original width +// as a marker comment, which SQLite preserves verbatim in sqlite_master.sql and +// normalises away from PRAGMA table_info. Reading the PRAGMA instead would +// silently give every narrow column the 64-bit path. +// +// A table with no markers leaves every field zero, which is exactly the +// behaviour every table had before markers existed. +func applyWidthMarkers(schema *TableSchema, createSQL string) { + markers := intmarker.Decode(createSQL) + if len(markers) == 0 { + return + } + for i := range schema.FullColumns { + attrs, ok := markers[strings.ToLower(schema.FullColumns[i].Name)] + if !ok { + continue + } + schema.FullColumns[i].DeclaredWidth = attrs.Bits + schema.FullColumns[i].Unsigned = attrs.Unsigned + schema.FullColumns[i].ExplicitAutoInc = attrs.ExplicitAutoInc + } +} + // loadSchema fetches schema from DB using the raw SQLite connection. // Uses PRAGMA table_xinfo (a superset of table_info) rather than table_info: // - cid: column's TRUE ordinal position in the table, including hidden @@ -356,6 +384,7 @@ func loadSchema(conn *sqlite3.SQLiteConn, tableName string) (*TableSchema, error if createRows.Next(dest) == nil { if sqlText, ok := dest[0].(string); ok { schema.CreateSQL = sqlText + applyWidthMarkers(schema, sqlText) } } } diff --git a/db/width_marker_test.go b/db/width_marker_test.go new file mode 100644 index 00000000..9bf10e50 --- /dev/null +++ b/db/width_marker_test.go @@ -0,0 +1,149 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "testing" +) + +// TestWidthMarkerSurvivesToTheSchemaCache walks the whole path this step +// builds: the transpiler writes a marker into the CREATE TABLE text, SQLite +// stores that text verbatim, and the schema cache reads the width back out of +// it. Each hop is checked, because a marker that is written but never read is +// indistinguishable from one that is read but never written. +func TestWidthMarkerSurvivesToTheSchemaCache(t *testing.T) { + source := newRowidTestDatabase(t, 1) + + // The DDL as the transpiler would emit it for + // "CREATE TABLE widths (id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + // small SMALLINT, big BIGINT, name TEXT)". + const ddl = "CREATE TABLE widths (" + + "id INTEGER /*M:32ua*/ PRIMARY KEY, " + + "small INTEGER /*M:16*/, " + + "big INTEGER, " + + "name TEXT)" + if err := execAndReload(source, ddl); err != nil { + t.Fatalf("CREATE TABLE: %v", err) + } + + // Hop 1: SQLite kept the marker in sqlite_master. + // Mutation: none needed - if SQLite normalised it away the next hop fails. + schema, err := source.GetCachedTableSchema("widths") + if err != nil { + t.Fatalf("GetCachedTableSchema: %v", err) + } + + // Hop 2: PRAGMA-derived type is plain INTEGER, which is why the width + // cannot be read from there. This is the hazard the design names. + // Mutation: read the width from ColumnSchema.Type instead of the marker. + byName := map[string]ColumnSchema{} + for _, col := range schema.FullColumns { + byName[col.Name] = col + if col.Name != "name" && col.Type != "INTEGER" { + t.Errorf("column %s has PRAGMA type %q, want plain INTEGER", col.Name, col.Type) + } + } + + // Hop 3: the width came back. + // Mutation: parse markers from PRAGMA table_info instead of CreateSQL; + // every width below becomes zero. + if got := byName["id"]; got.DeclaredWidth != 32 || !got.Unsigned || !got.ExplicitAutoInc { + t.Errorf("id = width %d unsigned %v autoinc %v, want 32/true/true", + got.DeclaredWidth, got.Unsigned, got.ExplicitAutoInc) + } + if got := byName["small"]; got.DeclaredWidth != 16 || got.Unsigned || got.ExplicitAutoInc { + t.Errorf("small = width %d unsigned %v autoinc %v, want 16/false/false", + got.DeclaredWidth, got.Unsigned, got.ExplicitAutoInc) + } + // BIGINT and non-integer columns carry no marker and keep the 64-bit path. + // Mutation: default DeclaredWidth to 64 instead of 0. + if got := byName["big"]; got.DeclaredWidth != 0 { + t.Errorf("big = width %d, want 0 (unmarked, 64-bit path)", got.DeclaredWidth) + } + if got := byName["name"]; got.DeclaredWidth != 0 { + t.Errorf("name = width %d, want 0", got.DeclaredWidth) + } +} + +// TestMarkerDoesNotBreakRowidAliasing is the hazard the design calls out: the +// stored type token must remain exactly INTEGER, because only that spelling +// makes a PRIMARY KEY an alias of the rowid. "INT PRIMARY KEY" does not, and +// storing the MySQL type word there would make LAST_INSERT_ID() report an +// unrelated internal rowid - a silent wrong answer. +// +// Mutation: emit "INT /*M:32a*/" instead of "INTEGER /*M:32a*/". The inserted +// id comes back NULL and the first assertion fires. +func TestMarkerDoesNotBreakRowidAliasing(t *testing.T) { + source := newRowidTestDatabase(t, 1) + if err := execAndReload(source, "CREATE TABLE aliased (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)"); err != nil { + t.Fatalf("CREATE TABLE: %v", err) + } + + if _, err := source.GetWriteDB().Exec("INSERT INTO aliased (id, v) VALUES (NULL, 'x')"); err != nil { + t.Fatalf("INSERT: %v", err) + } + + var id, rowid int64 + if err := source.GetWriteDB().QueryRow("SELECT id, rowid FROM aliased").Scan(&id, &rowid); err != nil { + t.Fatalf("SELECT: %v", err) + } + if id == 0 { + t.Fatal("id came back 0 or NULL: the marker broke rowid aliasing") + } + if id != rowid { + t.Errorf("id = %d but rowid = %d; the column is no longer a rowid alias", id, rowid) + } + + // And the schema cache still recognises it as the auto-increment column, + // which is what the injection rule keys on. + // Mutation: have loadSchema reject a type string containing a marker. + schema, err := source.GetCachedTableSchema("aliased") + if err != nil { + t.Fatalf("GetCachedTableSchema: %v", err) + } + if schema.GetAutoIncrementCol() != "id" { + t.Errorf("auto-increment column = %q, want id", schema.GetAutoIncrementCol()) + } +} + +// TestUnmarkedTableIsByteIdentical pins the migration promise: a table whose +// DDL carries no marker behaves exactly as it did before markers existed, and +// its stored text is unchanged. +// +// The byte-identical guarantee is about BIGINT and about pre-existing tables, +// NOT about the word INTEGER. A column declared INTEGER in MySQL is 32 bits - +// INTEGER and INT are the same type there - so a new table declaring INTEGER is +// marked, and that is asserted separately by +// TestIntegerSpelledOutIsMarked. This test therefore feeds DDL that already +// looks like stored SQLite output, which is what an old table's sqlite_master +// text looks like. +// +// Mutation: emit a marker for BIGINT, or for unmarked INTEGER columns read +// back from an old table. +func TestUnmarkedTableIsByteIdentical(t *testing.T) { + source := newRowidTestDatabase(t, 1) + const ddl = "CREATE TABLE legacy (id INTEGER PRIMARY KEY, n INTEGER, v TEXT)" + if err := execAndReload(source, ddl); err != nil { + t.Fatalf("CREATE TABLE: %v", err) + } + + var stored string + if err := source.GetWriteDB().QueryRow( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='legacy'").Scan(&stored); err != nil { + t.Fatalf("SELECT sql: %v", err) + } + if stored != ddl { + t.Errorf("stored DDL =\n %q\nwant\n %q", stored, ddl) + } + + schema, err := source.GetCachedTableSchema("legacy") + if err != nil { + t.Fatalf("GetCachedTableSchema: %v", err) + } + for _, col := range schema.FullColumns { + if col.DeclaredWidth != 0 || col.Unsigned || col.ExplicitAutoInc { + t.Errorf("column %s picked up marker attributes from an unmarked table: %+v", col.Name, col) + } + } +} diff --git a/protocol/error_mapper.go b/protocol/error_mapper.go index 600f5ca4..7885ba20 100644 --- a/protocol/error_mapper.go +++ b/protocol/error_mapper.go @@ -78,7 +78,7 @@ func sqlStateForCode(code uint16) string { case ErrCodeNoDB: return SQLStateNoDB case ErrCodeServerShutdown: - return "08S01" + return SQLStateConnFailure default: return SQLStateGeneral } diff --git a/protocol/handlers/metadata.go b/protocol/handlers/metadata.go index 14d868e7..a5e65bfa 100644 --- a/protocol/handlers/metadata.go +++ b/protocol/handlers/metadata.go @@ -3,6 +3,7 @@ package handlers import ( "database/sql" "fmt" + "github.com/maxpert/marmot/protocol/query/transform/intmarker" "github.com/maxpert/marmot/protocol" "github.com/rs/zerolog/log" @@ -203,13 +204,24 @@ func (m *MetadataHandler) HandleShowCreateTable(dbName, tableName string) (*prot return nil, err } + // sqlite_master.sql is returned verbatim, so it would carry the width + // marker the transpiler writes beside narrow integer types. The marker is + // Marmot's own bookkeeping; a client must never see it, and a tool that + // round-trips SHOW CREATE TABLE output must not be handed a comment it + // would then feed back in. + // + // This is the ONLY client-facing reader of the DDL text: every other + // sqlite_master query in protocol/handlers selects `name`, and the column + // types shown by SHOW COLUMNS and information_schema.COLUMNS come from + // PRAGMA table_info, which reports plain INTEGER with the marker + // normalised away. return &protocol.ResultSet{ Columns: []protocol.ColumnDef{ {Name: "Table", Type: 0xFD}, {Name: "Create Table", Type: 0xFD}, }, Rows: [][]interface{}{ - {tableName, createSQL}, + {tableName, intmarker.Strip(createSQL)}, }, }, nil } diff --git a/protocol/handlers/metadata_marker_test.go b/protocol/handlers/metadata_marker_test.go new file mode 100644 index 00000000..aaf75e53 --- /dev/null +++ b/protocol/handlers/metadata_marker_test.go @@ -0,0 +1,72 @@ +package handlers + +import ( + "database/sql" + "strings" + "testing" + + _ "github.com/mattn/go-sqlite3" +) + +// fakeDatabaseProvider serves one in-memory SQLite database. +type fakeDatabaseProvider struct{ db *sql.DB } + +func (f fakeDatabaseProvider) ListDatabases() []string { return []string{"testdb"} } +func (f fakeDatabaseProvider) DatabaseExists(name string) bool { return name == "testdb" } +func (f fakeDatabaseProvider) GetDatabaseConnection(string) (*sql.DB, error) { + return f.db, nil +} + +// TestShowCreateTableStripsWidthMarker pins what a client sees. SHOW CREATE +// TABLE hands sqlite_master.sql back verbatim, so without stripping, Marmot's +// own width bookkeeping would appear in every schema dump - and a tool that +// round-trips the output would feed the comment back in. +// +// Mutation: return createSQL instead of intmarker.Strip(createSQL); the first +// assertion fires. +func TestShowCreateTableStripsWidthMarker(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open: %v", err) + } + defer db.Close() + + const ddl = "CREATE TABLE marked (id INTEGER /*M:32ua*/ PRIMARY KEY, qty INTEGER /*M:16*/, v TEXT)" + if _, err := db.Exec(ddl); err != nil { + t.Fatalf("CREATE TABLE: %v", err) + } + + // Precondition: SQLite really did keep the marker, or the assertion below + // would pass for the wrong reason. + var stored string + if err := db.QueryRow("SELECT sql FROM sqlite_master WHERE name='marked'").Scan(&stored); err != nil { + t.Fatalf("SELECT sql: %v", err) + } + if !strings.Contains(stored, "/*M:") { + t.Fatalf("fixture: SQLite did not preserve the marker in sqlite_master: %q", stored) + } + + handler := NewMetadataHandler(fakeDatabaseProvider{db: db}, "__marmot_system") + rs, err := handler.HandleShowCreateTable("testdb", "marked") + if err != nil { + t.Fatalf("HandleShowCreateTable: %v", err) + } + if len(rs.Rows) != 1 || len(rs.Rows[0]) != 2 { + t.Fatalf("result shape = %d rows, want one row of two columns", len(rs.Rows)) + } + + shown, ok := rs.Rows[0][1].(string) + if !ok { + t.Fatalf("Create Table column is %T, want string", rs.Rows[0][1]) + } + if strings.Contains(shown, "/*M:") { + t.Errorf("SHOW CREATE TABLE leaked a width marker to the client:\n%s", shown) + } + // The rest of the DDL must survive intact, or stripping has eaten schema. + // Mutation: strip everything between the first "/*" and the last "*/". + for _, want := range []string{"CREATE TABLE marked", "id INTEGER PRIMARY KEY", "qty INTEGER", "v TEXT"} { + if !strings.Contains(shown, want) { + t.Errorf("stripped DDL lost %q:\n%s", want, shown) + } + } +} diff --git a/protocol/mysql_errors.go b/protocol/mysql_errors.go index 6b29d763..2a55cd8e 100644 --- a/protocol/mysql_errors.go +++ b/protocol/mysql_errors.go @@ -1,38 +1,45 @@ package protocol -import "fmt" +import ( + "fmt" -// MySQL error code constants + "github.com/maxpert/marmot/protocol/mysqlcode" +) + +// MySQL error code constants. Values live in protocol/mysqlcode so packages +// below protocol - which cannot import protocol without an import cycle - +// can reference the same numbers instead of duplicating them. const ( - ErrCodeUnknown uint16 = 1105 - ErrCodeBadNull uint16 = 1048 - ErrCodeTableExists uint16 = 1050 - ErrCodeBadField uint16 = 1054 - ErrCodeDupFieldName uint16 = 1060 - ErrCodeDupEntry uint16 = 1062 - ErrCodeParseError uint16 = 1064 - ErrCodeTooBigRowsize uint16 = 1118 - ErrCodeNoSuchTable uint16 = 1146 - ErrCodeNoDB uint16 = 1046 - ErrCodeLockTimeout uint16 = 1205 - ErrCodeDeadlock uint16 = 1213 - ErrCodeReadOnly uint16 = 1290 - ErrCodeServerShutdown uint16 = 1053 - ErrCodeNoReferencedRow uint16 = 1452 - ErrCodeCheckConstraint uint16 = 3819 + ErrCodeUnknown = mysqlcode.ErrCodeUnknown + ErrCodeBadNull = mysqlcode.ErrCodeBadNull + ErrCodeTableExists = mysqlcode.ErrCodeTableExists + ErrCodeBadField = mysqlcode.ErrCodeBadField + ErrCodeDupFieldName = mysqlcode.ErrCodeDupFieldName + ErrCodeDupEntry = mysqlcode.ErrCodeDupEntry + ErrCodeParseError = mysqlcode.ErrCodeParseError + ErrCodeTooBigRowsize = mysqlcode.ErrCodeTooBigRowsize + ErrCodeNoSuchTable = mysqlcode.ErrCodeNoSuchTable + ErrCodeNoDB = mysqlcode.ErrCodeNoDB + ErrCodeLockTimeout = mysqlcode.ErrCodeLockTimeout + ErrCodeDeadlock = mysqlcode.ErrCodeDeadlock + ErrCodeReadOnly = mysqlcode.ErrCodeReadOnly + ErrCodeServerShutdown = mysqlcode.ErrCodeServerShutdown + ErrCodeNoReferencedRow = mysqlcode.ErrCodeNoReferencedRow + ErrCodeCheckConstraint = mysqlcode.ErrCodeCheckConstraint ) -// SQLSTATE constants +// SQLSTATE constants. Values live in protocol/mysqlcode; see above. const ( - SQLStateGeneral = "HY000" - SQLStateIntegrity = "23000" - SQLStateSyntax = "42000" - SQLStateDeadlock = "40001" - SQLStateTableExists = "42S01" - SQLStateNoSuchTable = "42S02" - SQLStateNoSuchCol = "42S22" - SQLStateDupColumn = "42S21" - SQLStateNoDB = "3D000" + SQLStateGeneral = mysqlcode.SQLStateGeneral + SQLStateIntegrity = mysqlcode.SQLStateIntegrity + SQLStateSyntax = mysqlcode.SQLStateSyntax + SQLStateDeadlock = mysqlcode.SQLStateDeadlock + SQLStateTableExists = mysqlcode.SQLStateTableExists + SQLStateNoSuchTable = mysqlcode.SQLStateNoSuchTable + SQLStateNoSuchCol = mysqlcode.SQLStateNoSuchCol + SQLStateDupColumn = mysqlcode.SQLStateDupColumn + SQLStateConnFailure = mysqlcode.SQLStateConnFailure + SQLStateNoDB = mysqlcode.SQLStateNoDB ) // MySQLError represents a MySQL protocol error with error code and SQLSTATE @@ -74,5 +81,5 @@ func ErrReadOnly() *MySQLError { // ErrServerShutdown returns error 1053 - server shutdown in progress func ErrServerShutdown() *MySQLError { - return NewMySQLError(ErrCodeServerShutdown, "08S01", "Server shutdown in progress") + return NewMySQLError(ErrCodeServerShutdown, SQLStateConnFailure, "Server shutdown in progress") } diff --git a/protocol/mysqlcode/mysqlcode.go b/protocol/mysqlcode/mysqlcode.go new file mode 100644 index 00000000..66a0b8c7 --- /dev/null +++ b/protocol/mysqlcode/mysqlcode.go @@ -0,0 +1,46 @@ +// Package mysqlcode is the single home for MySQL wire-protocol error-code and +// SQLSTATE numeric/string constants. It imports nothing from this repo, so +// any package - including ones below protocol, which cannot import protocol +// itself without creating an import cycle - can depend on it without pulling +// in protocol's error-mapping logic. +package mysqlcode + +// MySQL error code constants. +const ( + ErrCodeUnknown uint16 = 1105 + ErrCodeBadNull uint16 = 1048 + ErrCodeTableExists uint16 = 1050 + ErrCodeBadField uint16 = 1054 + ErrCodeDupFieldName uint16 = 1060 + ErrCodeDupEntry uint16 = 1062 + ErrCodeParseError uint16 = 1064 + ErrCodeTooBigRowsize uint16 = 1118 + ErrCodeNoSuchTable uint16 = 1146 + ErrCodeNoDB uint16 = 1046 + ErrCodeLockTimeout uint16 = 1205 + ErrCodeDeadlock uint16 = 1213 + ErrCodeReadOnly uint16 = 1290 + ErrCodeServerShutdown uint16 = 1053 + ErrCodeNoReferencedRow uint16 = 1452 + ErrCodeCheckConstraint uint16 = 3819 + + // ErrCodeNotSupportedYet is MySQL's ER_NOT_SUPPORTED_YET: the server + // understands the statement but does not implement this form of it. + ErrCodeNotSupportedYet uint16 = 1235 +) + +// SQLSTATE constants. +const ( + SQLStateGeneral = "HY000" + SQLStateIntegrity = "23000" + SQLStateSyntax = "42000" + SQLStateDeadlock = "40001" + SQLStateTableExists = "42S01" + SQLStateNoSuchTable = "42S02" + SQLStateNoSuchCol = "42S22" + SQLStateDupColumn = "42S21" + // SQLStateConnFailure is MySQL's class for a connection that is going away, + // used for ER_SERVER_SHUTDOWN. + SQLStateConnFailure = "08S01" + SQLStateNoDB = "3D000" +) diff --git a/protocol/query/pipeline_ddl_having_test.go b/protocol/query/pipeline_ddl_having_test.go index 0806626c..ed3e74da 100644 --- a/protocol/query/pipeline_ddl_having_test.go +++ b/protocol/query/pipeline_ddl_having_test.go @@ -52,7 +52,15 @@ func TestPipeline_WellFormedDDLStillParses(t *testing.T) { } // TestPipeline_AlterTableColumnRegressions locks in that plain ADD/DROP/RENAME -// COLUMN still pass through unchanged after introducing AlterTableConstraintRule. +// COLUMN are not mangled by AlterTableConstraintRule, which is what this test +// was written for. +// +// The ADD COLUMN row's expected output changed when declared-width markers were +// introduced: an integer column's type is now collapsed to SQLite's single +// INTEGER type with the MySQL width recorded beside it, on ALTER exactly as on +// CREATE. The assertion is not weakened - it is still an exact match on the +// whole statement - it asserts the new intended output. DROP and RENAME carry +// no column type and are unchanged, so the original guard still stands on them. func TestPipeline_AlterTableColumnRegressions(t *testing.T) { pipeline, err := NewPipeline(100, nil) if err != nil { @@ -64,7 +72,7 @@ func TestPipeline_AlterTableColumnRegressions(t *testing.T) { sql string want string }{ - {"ALTER TABLE users ADD COLUMN age INT", "alter table users add column age INT"}, + {"ALTER TABLE users ADD COLUMN age INT", "alter table users add column age INTEGER /*M:32*/"}, {"ALTER TABLE users DROP COLUMN age", "alter table users drop column age"}, {"ALTER TABLE users RENAME COLUMN age TO years", "alter table users rename column age to years"}, } diff --git a/protocol/query/transform/alter_table_column_type.go b/protocol/query/transform/alter_table_column_type.go index a35d1188..f37dfc7e 100644 --- a/protocol/query/transform/alter_table_column_type.go +++ b/protocol/query/transform/alter_table_column_type.go @@ -39,14 +39,19 @@ func (r *AlterTableColumnTypeRule) Transform(stmt sqlparser.Statement, params [] case *sqlparser.AddColumns: for _, col := range o.Columns { stripMySQLColumnType(col.Type) + collapseIntegerTypeWithMarker(col.Type) } case *sqlparser.ModifyColumn: if o.NewColDefinition != nil { stripMySQLColumnType(o.NewColDefinition.Type) + collapseIntegerTypeWithMarker(o.NewColDefinition.Type) } case *sqlparser.ChangeColumn: + // CHANGE renames as well as retypes; the marker travels with the + // new column definition, so it follows the new name. if o.NewColDefinition != nil { stripMySQLColumnType(o.NewColDefinition.Type) + collapseIntegerTypeWithMarker(o.NewColDefinition.Type) } } } diff --git a/protocol/query/transform/alter_table_column_type_test.go b/protocol/query/transform/alter_table_column_type_test.go new file mode 100644 index 00000000..2d975a10 --- /dev/null +++ b/protocol/query/transform/alter_table_column_type_test.go @@ -0,0 +1,90 @@ +package transform + +import ( + "strings" + "testing" + + "vitess.io/vitess/go/vt/sqlparser" +) + +// TestAlterTableEmitsWidthMarker pins the ALTER path. Before the collapse was +// shared, this rule ran only stripMySQLColumnType, which removes display widths +// and charsets but does NOT collapse the type and does NOT remove +// AUTO_INCREMENT - so "ALTER TABLE t ADD COLUMN x INT AUTO_INCREMENT" reached +// SQLite carrying a keyword it cannot parse, and no width was ever recorded for +// a column added after the table was created. +// +// Mutation: drop the collapseIntegerTypeWithMarker call from any of the three +// arms; that arm's row fires. +func TestAlterTableEmitsWidthMarker(t *testing.T) { + cases := []struct { + name string + input string + column string + want string + }{ + {"ADD COLUMN INT AUTO_INCREMENT", "ALTER TABLE t ADD COLUMN x INT AUTO_INCREMENT", "x", "INTEGER /*M:32a*/"}, + {"ADD COLUMN SMALLINT UNSIGNED", "ALTER TABLE t ADD COLUMN q SMALLINT UNSIGNED", "q", "INTEGER /*M:16u*/"}, + {"MODIFY COLUMN MEDIUMINT", "ALTER TABLE t MODIFY COLUMN n MEDIUMINT", "n", "INTEGER /*M:24*/"}, + // CHANGE renames as well as retypes; the marker must follow the NEW name. + {"CHANGE COLUMN renames and retypes", "ALTER TABLE t CHANGE COLUMN old new TINYINT UNSIGNED", "new", "INTEGER /*M:8u*/"}, + // BIGINT keeps the existing 64-bit path and is deliberately unmarked. + {"ADD COLUMN BIGINT is unmarked", "ALTER TABLE t ADD COLUMN big BIGINT AUTO_INCREMENT", "big", "INTEGER"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stmt, err := sqlparser.NewTestParser().Parse(tc.input) + if err != nil { + t.Fatalf("parse: %v", err) + } + rule := &AlterTableColumnTypeRule{} + if _, err := rule.Transform(stmt, nil, nil, "testdb", &SQLiteSerializer{}); err != ErrRuleNotApplicable { + t.Fatalf("Transform returned %v, want ErrRuleNotApplicable", err) + } + + got, ok := alteredColumnType(stmt, tc.column) + if !ok { + t.Fatalf("column %q not found in the altered statement", tc.column) + } + if got != tc.want { + t.Errorf("declared type = %q, want %q", got, tc.want) + } + + // AUTO_INCREMENT must be gone from the AST, or SQLite rejects the + // statement outright. + // Mutation: keep Options.Autoincrement set. + serialized := (&SQLiteSerializer{}).Serialize(stmt) + if strings.Contains(strings.ToUpper(serialized), "AUTO_INCREMENT") { + t.Errorf("serialized ALTER still carries AUTO_INCREMENT, which SQLite cannot parse:\n%s", serialized) + } + }) + } +} + +// alteredColumnType returns the declared type of one column in an ALTER. +func alteredColumnType(stmt sqlparser.Statement, column string) (string, bool) { + alter, ok := stmt.(*sqlparser.AlterTable) + if !ok { + return "", false + } + for _, opt := range alter.AlterOptions { + switch o := opt.(type) { + case *sqlparser.AddColumns: + for _, col := range o.Columns { + if col.Name.String() == column { + return col.Type.Type, true + } + } + case *sqlparser.ModifyColumn: + if o.NewColDefinition != nil && o.NewColDefinition.Name.String() == column { + return o.NewColDefinition.Type.Type, true + } + case *sqlparser.ChangeColumn: + if o.NewColDefinition != nil && o.NewColDefinition.Name.String() == column { + return o.NewColDefinition.Type.Type, true + } + } + } + return "", false +} diff --git a/protocol/query/transform/errors.go b/protocol/query/transform/errors.go index d5882dc8..19c02659 100644 --- a/protocol/query/transform/errors.go +++ b/protocol/query/transform/errors.go @@ -1,17 +1,20 @@ package transform -import "fmt" +import ( + "fmt" + + "github.com/maxpert/marmot/protocol/mysqlcode" +) // MySQL server error codes raised by transformation rules. // -// These live here, rather than beside the protocol package's error codes, -// because transform is a leaf package: protocol imports it and never the -// reverse, so a rule cannot name protocol.ErrCode*. Only codes a rule actually -// raises belong here, and none of them is declared by protocol as well. +// The value lives in protocol/mysqlcode, the shared leaf package for MySQL +// error-code constants, and is referenced here under its historical name so +// existing callers in this package's API are unaffected. const ( // ErrCodeNotSupportedYet is MySQL's ER_NOT_SUPPORTED_YET (1235): the // server understands the statement but does not implement this form of it. - ErrCodeNotSupportedYet uint16 = 1235 + ErrCodeNotSupportedYet = mysqlcode.ErrCodeNotSupportedYet ) // CodedError is the error a transformation rule returns when it refuses a diff --git a/protocol/query/transform/int_type.go b/protocol/query/transform/int_type.go index 9272fe7e..fe00850a 100644 --- a/protocol/query/transform/int_type.go +++ b/protocol/query/transform/int_type.go @@ -50,27 +50,7 @@ func (r *IntTypeRule) Transform(stmt sqlparser.Statement, params []interface{}, continue } - // Strip AUTO_INCREMENT keyword (SQLite doesn't use it) - if col.Type.Options != nil && col.Type.Options.Autoincrement { - col.Type.Options.Autoincrement = false - modified = true - } - - // Strip UNSIGNED - if col.Type.Unsigned { - col.Type.Unsigned = false - modified = true - } - - // Strip ZEROFILL - if col.Type.Zerofill { - col.Type.Zerofill = false - modified = true - } - - // Convert all integer types to INTEGER for SQLite - if upperType != "INTEGER" { - col.Type.Type = "INTEGER" + if collapseIntegerTypeWithMarker(col.Type) { modified = true } } diff --git a/protocol/query/transform/int_type_test.go b/protocol/query/transform/int_type_test.go index 5724b458..1d60dbaa 100644 --- a/protocol/query/transform/int_type_test.go +++ b/protocol/query/transform/int_type_test.go @@ -4,6 +4,7 @@ import ( "strings" "testing" + "github.com/maxpert/marmot/protocol/query/transform/intmarker" "vitess.io/vitess/go/vt/sqlparser" ) @@ -119,8 +120,8 @@ func TestIntTypeRule_ModifiesAST(t *testing.T) { } // Check integer type was converted to INTEGER - if tt.checkIntType && isIntegerType(strings.ToUpper(col.Type.Type)) { - if col.Type.Type != "INTEGER" { + if tt.checkIntType && isIntegerType(strings.ToUpper(storedTypeToken(col.Type.Type))) { + if storedTypeToken(col.Type.Type) != "INTEGER" { t.Errorf("integer type should have been converted to INTEGER for column %s, got %s", col.Name.String(), col.Type.Type) } } @@ -170,8 +171,8 @@ func TestIntTypeRule_WordPressSchema(t *testing.T) { } // Check integer types were converted to INTEGER - if isIntegerType(strings.ToUpper(col.Type.Type)) { - if col.Type.Type != "INTEGER" { + if isIntegerType(strings.ToUpper(storedTypeToken(col.Type.Type))) { + if storedTypeToken(col.Type.Type) != "INTEGER" { t.Errorf("integer type should be INTEGER for column %s, got %s", col.Name.String(), col.Type.Type) } } @@ -212,8 +213,8 @@ func TestIntTypeRule_AllIntTypes(t *testing.T) { if col.Type.Unsigned { t.Errorf("%s: UNSIGNED should have been stripped", tt.name) } - if col.Type.Type != "INTEGER" { - t.Errorf("%s: type should be INTEGER, got %s", tt.name, col.Type.Type) + if storedTypeToken(col.Type.Type) != "INTEGER" { + t.Errorf("%s: stored type token should be INTEGER, got %s", tt.name, col.Type.Type) } } }) @@ -272,8 +273,8 @@ func TestIntTypeRule_DeferToCreateTableRule(t *testing.T) { t.Errorf("AUTO_INCREMENT should have been removed from column %s", col.Name.String()) } // Check that integer types were converted to INTEGER - if isIntegerType(strings.ToUpper(col.Type.Type)) { - if col.Type.Type != "INTEGER" { + if isIntegerType(strings.ToUpper(storedTypeToken(col.Type.Type))) { + if storedTypeToken(col.Type.Type) != "INTEGER" { t.Errorf("integer type should have been converted to INTEGER for column %s, got %s", col.Name.String(), col.Type.Type) } } @@ -281,3 +282,137 @@ func TestIntTypeRule_DeferToCreateTableRule(t *testing.T) { }) } } + +// storedTypeToken returns the SQLite type word without the width marker the +// transpiler now appends. The assertions below are NOT loosened by it: the +// stored token must still be exactly "INTEGER", because only that spelling +// aliases the rowid. What the marker adds is checked separately, by +// TestIntTypeRuleEmitsWidthMarker. +func storedTypeToken(declared string) string { + if i := strings.Index(declared, "/*M:"); i >= 0 { + return strings.TrimSpace(declared[:i]) + } + return declared +} + +// TestIntTypeRuleEmitsWidthMarker pins what the marker adds on CREATE TABLE. +// The stored token must still be exactly "INTEGER" - that is asserted by the +// tests above and by the storedTypeToken helper - and the declared width, the +// UNSIGNED modifier and an explicit AUTO_INCREMENT must survive beside it. +func TestIntTypeRuleEmitsWidthMarker(t *testing.T) { + cases := []struct { + name string + input string + column string + want string + }{ + // Mutation: capture the attributes AFTER the strips instead of before. + // Unsigned and ExplicitAutoInc are already false by then, so every row + // with u or a loses it. + {"INT AUTO_INCREMENT", "CREATE TABLE t (id INT AUTO_INCREMENT PRIMARY KEY)", "id", "INTEGER /*M:32a*/"}, + {"INT UNSIGNED AUTO_INCREMENT", "CREATE TABLE t (id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY)", "id", "INTEGER /*M:32ua*/"}, + {"TINYINT", "CREATE TABLE t (flag TINYINT)", "flag", "INTEGER /*M:8*/"}, + {"SMALLINT UNSIGNED", "CREATE TABLE t (qty SMALLINT UNSIGNED)", "qty", "INTEGER /*M:16u*/"}, + {"MEDIUMINT", "CREATE TABLE t (n MEDIUMINT)", "n", "INTEGER /*M:24*/"}, + {"INTEGER spelled out", "CREATE TABLE t (id INTEGER AUTO_INCREMENT PRIMARY KEY)", "id", "INTEGER /*M:32a*/"}, + // BIGINT is deliberately unmarked: it keeps the existing 64-bit path, + // and an existing BIGINT table's DDL text must be byte-identical to + // what it was before markers existed. + // Mutation: return 64, true from BitsForType for BIGINT. + {"BIGINT is unmarked", "CREATE TABLE t (id BIGINT AUTO_INCREMENT PRIMARY KEY)", "id", "INTEGER"}, + {"BIGINT UNSIGNED is unmarked", "CREATE TABLE t (id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY)", "id", "INTEGER"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stmt, err := sqlparser.NewTestParser().Parse(tc.input) + if err != nil { + t.Fatalf("parse: %v", err) + } + rule := &IntTypeRule{} + if _, err := rule.Transform(stmt, nil, nil, "testdb", &SQLiteSerializer{}); err != ErrRuleNotApplicable { + t.Fatalf("Transform returned %v, want ErrRuleNotApplicable", err) + } + + create := stmt.(*sqlparser.CreateTable) + var found bool + for _, col := range create.TableSpec.Columns { + if col.Name.String() != tc.column { + continue + } + found = true + if col.Type.Type != tc.want { + t.Errorf("declared type = %q, want %q", col.Type.Type, tc.want) + } + } + if !found { + t.Fatalf("column %q not found", tc.column) + } + }) + } +} + +// TestIntTypeRuleMarkerSurvivesSerialization pins the marker through the step +// that actually produces the DDL SQLite stores. A marker held only in the AST +// and dropped by the serializer would be invisible until a restart read +// sqlite_master back. +// +// Mutation: emit the marker as a separate AST field the serializer ignores. +func TestIntTypeRuleMarkerSurvivesSerialization(t *testing.T) { + stmt, err := sqlparser.NewTestParser().Parse( + "CREATE TABLE t (id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, v TEXT)") + if err != nil { + t.Fatalf("parse: %v", err) + } + rule := &IntTypeRule{} + if _, err := rule.Transform(stmt, nil, nil, "testdb", &SQLiteSerializer{}); err != ErrRuleNotApplicable { + t.Fatalf("Transform returned %v, want ErrRuleNotApplicable", err) + } + + serialized := (&SQLiteSerializer{}).Serialize(stmt) + if !strings.Contains(serialized, "/*M:32ua*/") { + t.Fatalf("serialized DDL lost the marker:\n%s", serialized) + } + // And it must decode back to the column it belongs to. + decoded := intmarker.Decode(serialized) + if got := decoded["id"]; got != (intmarker.Attributes{Bits: 32, Unsigned: true, ExplicitAutoInc: true}) { + t.Errorf("decoded %+v from %q", got, serialized) + } +} + +// TestIntegerSpelledOutIsMarked records a deliberate deviation from FINAL.md +// touch point 1, which says Encode returns "" for BIGINT *and* INTEGER. +// +// Marking INTEGER is correct and the design text is loose: in MySQL, INTEGER is +// a synonym for INT and is 32 bits, so a column declared INTEGER AUTO_INCREMENT +// overflows at 2147483647 exactly as INT does. Treating it as unmarked would +// route it to the 64-bit path and reproduce the overflow this work exists to +// fix. Only BIGINT is genuinely 64-bit and genuinely unmarked. +// +// Mutation: make BitsForType return ok == false for "INTEGER"; the first row +// fires. Make it return true for "BIGINT"; the second fires. +func TestIntegerSpelledOutIsMarked(t *testing.T) { + cases := []struct { + name string + input string + want string + }{ + {"INTEGER is 32 bits, like INT", "CREATE TABLE t (id INTEGER AUTO_INCREMENT PRIMARY KEY)", "INTEGER /*M:32a*/"}, + {"BIGINT is the only unmarked integer", "CREATE TABLE t (id BIGINT AUTO_INCREMENT PRIMARY KEY)", "INTEGER"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stmt, err := sqlparser.NewTestParser().Parse(tc.input) + if err != nil { + t.Fatalf("parse: %v", err) + } + if _, err := (&IntTypeRule{}).Transform(stmt, nil, nil, "testdb", &SQLiteSerializer{}); err != ErrRuleNotApplicable { + t.Fatalf("Transform returned %v, want ErrRuleNotApplicable", err) + } + got := stmt.(*sqlparser.CreateTable).TableSpec.Columns[0].Type.Type + if got != tc.want { + t.Errorf("declared type = %q, want %q", got, tc.want) + } + }) + } +} diff --git a/protocol/query/transform/intmarker/decode.go b/protocol/query/transform/intmarker/decode.go new file mode 100644 index 00000000..d57a4207 --- /dev/null +++ b/protocol/query/transform/intmarker/decode.go @@ -0,0 +1,268 @@ +package intmarker + +import "strings" + +// Decode reads every marker in one table's sqlite_master.sql text and returns +// them keyed by column name, lowercased. +// +// It must attribute each marker to the column whose definition contains it, so +// it walks the column list rather than searching the whole string: a marker +// sits immediately after its column's type, and a naive scan would hand the +// first marker to the first column whatever the text between them said. +// +// Text that is not DDL structure is skipped rather than parsed: single-quoted +// strings, double-quoted, backticked and bracketed identifiers, line comments, +// and block comments that are not markers. A marker appearing inside any of +// those is not a marker, and a column named like one is a name. +// +// Anything it cannot parse yields no entry rather than an error. The caller's +// fallback for an absent marker is the 64-bit path, which is the behaviour +// every table had before markers existed, so a malformed marker degrades to +// today's behaviour rather than failing a schema load. +func Decode(createSQL string) map[string]Attributes { + body, ok := columnListBody(createSQL) + if !ok { + return nil + } + + var out map[string]Attributes + for _, part := range splitTopLevel(body) { + name, ok := leadingIdentifier(part) + if !ok { + continue + } + attrs, ok := findMarker(part) + if !ok { + continue + } + if out == nil { + out = make(map[string]Attributes) + } + out[strings.ToLower(name)] = attrs + } + return out +} + +// columnListBody returns the text between the parentheses that open the column +// list, skipping anything quoted so a parenthesis inside a table name or a +// comment does not open it. +func columnListBody(createSQL string) (string, bool) { + i, ok := scanToColumnList(createSQL) + if !ok { + return "", false + } + depth := 0 + for j := i; j < len(createSQL); j++ { + if skip, next := skipNonCode(createSQL, j); skip { + j = next - 1 + continue + } + switch createSQL[j] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + return createSQL[i+1 : j], true + } + } + } + return "", false +} + +// scanToColumnList finds the opening parenthesis of the column list. +func scanToColumnList(s string) (int, bool) { + for i := 0; i < len(s); i++ { + if skip, next := skipNonCode(s, i); skip { + i = next - 1 + continue + } + if s[i] == '(' { + return i, true + } + } + return 0, false +} + +// skipNonCode reports whether position i begins a region that is not DDL +// structure, and where that region ends. A marker is deliberately NOT skipped: +// it is the one block comment this package reads. +func skipNonCode(s string, i int) (bool, int) { + switch { + case strings.HasPrefix(s[i:], markerOpen): + return false, i + case s[i] == '\'': + return true, closingQuote(s, i, '\'') + case s[i] == '"': + return true, closingQuote(s, i, '"') + case s[i] == '`': + return true, closingQuote(s, i, '`') + case s[i] == '[': + return true, indexAfter(s, i+1, "]") + case strings.HasPrefix(s[i:], "--"): + if nl := strings.IndexByte(s[i:], '\n'); nl >= 0 { + return true, i + nl + 1 + } + return true, len(s) + case strings.HasPrefix(s[i:], "/*"): + return true, indexAfter(s, i+2, "*/") + } + return false, i +} + +// closingQuote finds the end of a quoted region, honouring SQL's doubled-quote +// escape (” inside '...', "" inside "...", “ inside `...`). +func closingQuote(s string, start int, q byte) int { + for i := start + 1; i < len(s); i++ { + if s[i] != q { + continue + } + if i+1 < len(s) && s[i+1] == q { + i++ + continue + } + return i + 1 + } + return len(s) +} + +func indexAfter(s string, from int, sep string) int { + if idx := strings.Index(s[from:], sep); idx >= 0 { + return from + idx + len(sep) + } + return len(s) +} + +// splitTopLevel splits a column list on commas that are not inside parentheses +// or any non-code region, so "DEFAULT (a, b)" stays with its column. +func splitTopLevel(body string) []string { + var parts []string + depth := 0 + start := 0 + for i := 0; i < len(body); i++ { + if skip, next := skipNonCode(body, i); skip { + i = next - 1 + continue + } + switch body[i] { + case '(': + depth++ + case ')': + depth-- + case ',': + if depth == 0 { + parts = append(parts, body[start:i]) + start = i + 1 + } + } + } + return append(parts, body[start:]) +} + +// leadingIdentifier returns a column definition's name, unquoting whichever +// form the DDL used. +func leadingIdentifier(part string) (string, bool) { + i := 0 + for i < len(part) && isSpace(part[i]) { + i++ + } + if i >= len(part) { + return "", false + } + switch part[i] { + case '`', '"': + q := part[i] + end := closingQuote(part, i, q) + if end <= i+1 { + return "", false + } + inner := part[i+1 : end-1] + return strings.ReplaceAll(inner, string([]byte{q, q}), string(q)), true + case '[': + end := indexAfter(part, i+1, "]") + if end <= i+1 { + return "", false + } + return part[i+1 : end-1], true + } + start := i + for i < len(part) && !isSpace(part[i]) && part[i] != '(' { + i++ + } + if i == start { + return "", false + } + return part[start:i], true +} + +// findMarker returns the first marker in one column definition, skipping any +// that appear inside a quoted region. +func findMarker(part string) (Attributes, bool) { + for i := 0; i < len(part); i++ { + if strings.HasPrefix(part[i:], markerOpen) { + end := strings.Index(part[i+len(markerOpen):], markerClose) + if end < 0 { + return Attributes{}, false + } + body := part[i+len(markerOpen) : i+len(markerOpen)+end] + attrs, err := parseMarkerBody(body) + if err != nil { + return Attributes{}, false + } + return attrs, true + } + if skip, next := skipNonCode(part, i); skip { + i = next - 1 + } + } + return Attributes{}, false +} + +// Strip removes every marker from DDL text, for the paths that hand +// sqlite_master.sql to a client. Markers inside quoted regions are left alone, +// because there they are data. +// +// It also repairs the spacing the removal would otherwise leave: a marker is +// emitted as "INTEGER /*M:32a*/ PRIMARY KEY", so removing just the comment +// leaves a double space, and "INTEGER /*M:8*/," leaves a space before the +// comma. Both are visible to anyone reading SHOW CREATE TABLE. +func Strip(createSQL string) string { + out := make([]byte, 0, len(createSQL)) + for i := 0; i < len(createSQL); i++ { + if strings.HasPrefix(createSQL[i:], markerOpen) { + if end := strings.Index(createSQL[i+len(markerOpen):], markerClose); end >= 0 { + i = i + len(markerOpen) + end + len(markerClose) - 1 + out = repairSpacing(out, createSQL, i+1) + continue + } + } + if skip, next := skipNonCode(createSQL, i); skip { + out = append(out, createSQL[i:next]...) + i = next - 1 + continue + } + out = append(out, createSQL[i]) + } + return string(out) +} + +// repairSpacing removes the one space a deleted marker leaves behind. It drops +// the space that preceded the marker when what follows is a delimiter or more +// whitespace, so exactly one separator survives between the tokens that were +// on either side. +func repairSpacing(out []byte, src string, next int) []byte { + if len(out) == 0 || !isSpace(out[len(out)-1]) { + return out + } + if next >= len(src) { + return out[:len(out)-1] + } + if isSpace(src[next]) || src[next] == ',' || src[next] == ')' { + return out[:len(out)-1] + } + return out +} + +func isSpace(c byte) bool { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' +} diff --git a/protocol/query/transform/intmarker/intmarker.go b/protocol/query/transform/intmarker/intmarker.go new file mode 100644 index 00000000..504f0455 --- /dev/null +++ b/protocol/query/transform/intmarker/intmarker.go @@ -0,0 +1,131 @@ +// Package intmarker encodes a MySQL integer column's declared width into the +// SQLite DDL text, and reads it back. +// +// SQLite has one integer type, so the transpiler collapses TINYINT..INT to +// INTEGER and the declared width is lost. It cannot be kept by storing the +// MySQL type word instead: only the exact token INTEGER makes a PRIMARY KEY an +// alias of the rowid, and "INT PRIMARY KEY" does not, which would make +// LAST_INSERT_ID() report an unrelated internal rowid. So the width travels as +// a comment beside the type, which SQLite preserves verbatim in +// sqlite_master.sql and normalises away everywhere else. +// +// The grammar is /*M:[u][a]*/ - bits in {8,16,24,32}, u for UNSIGNED, +// a for an explicitly declared AUTO_INCREMENT. It is deliberately NOT MySQL's +// /*! ... */ executable-comment syntax: that syntax asks another MySQL to +// execute the contents, which is the opposite of what this is for. +package intmarker + +import ( + "fmt" + "strconv" + "strings" +) + +const ( + markerOpen = "/*M:" + markerClose = "*/" +) + +// Attributes are what a marker records about one column. +type Attributes struct { + // Bits is the declared width: 8, 16, 24 or 32. Zero means "no marker", + // which is the 64-bit path and every table created before markers existed. + Bits int + // Unsigned mirrors the MySQL UNSIGNED modifier. + Unsigned bool + // ExplicitAutoInc records that the column was declared AUTO_INCREMENT, + // as opposed to merely being a narrow integer. + ExplicitAutoInc bool +} + +// Marked reports whether a marker was present. +func (a Attributes) Marked() bool { return a.Bits != 0 } + +// WidthMax returns the largest value the declared column can hold. +// +// It is the single source of this number: the range allocator, the claim +// precondition and the participant that validates a claim must all derive the +// ceiling from the same code, or a claim accepted by one is rejected by +// another. Zero bits means no marker and therefore no narrow ceiling; the +// caller uses its existing 64-bit path. +func (a Attributes) WidthMax() uint64 { + if a.Bits == 0 { + return 0 + } + if a.Unsigned { + return uint64(1)<= '0' && body[i] <= '9' { + i++ + } + if i == 0 { + return a, fmt.Errorf("marker has no width: %q", body) + } + bits, err := strconv.Atoi(body[:i]) + if err != nil { + return a, fmt.Errorf("marker width %q: %w", body[:i], err) + } + switch bits { + case 8, 16, 24, 32: + a.Bits = bits + default: + return a, fmt.Errorf("marker width %d is not one of 8, 16, 24, 32", bits) + } + for ; i < len(body); i++ { + switch body[i] { + case 'u': + a.Unsigned = true + case 'a': + a.ExplicitAutoInc = true + default: + return a, fmt.Errorf("marker flag %q is not u or a", body[i]) + } + } + return a, nil +} diff --git a/protocol/query/transform/intmarker/intmarker_test.go b/protocol/query/transform/intmarker/intmarker_test.go new file mode 100644 index 00000000..6bd219f2 --- /dev/null +++ b/protocol/query/transform/intmarker/intmarker_test.go @@ -0,0 +1,215 @@ +package intmarker + +import "testing" + +// TestEncodeRoundTrip pins the grammar in both directions. +func TestEncodeRoundTrip(t *testing.T) { + cases := []struct { + name string + attrs Attributes + want string + }{ + {"int, signed, auto-increment", Attributes{Bits: 32, ExplicitAutoInc: true}, "/*M:32a*/"}, + // Mutation: drop the Unsigned branch in Encode. This and the next fire, + // and every unsigned column silently loses half its range. + {"int, unsigned, auto-increment", Attributes{Bits: 32, Unsigned: true, ExplicitAutoInc: true}, "/*M:32ua*/"}, + {"smallint, unsigned, no auto-increment", Attributes{Bits: 16, Unsigned: true}, "/*M:16u*/"}, + {"tinyint, plain", Attributes{Bits: 8}, "/*M:8*/"}, + {"mediumint, auto-increment", Attributes{Bits: 24, ExplicitAutoInc: true}, "/*M:24a*/"}, + // BIGINT and anything unmarked must render nothing at all, or the DDL + // text of an existing table stops being byte-identical. + // Mutation: return "/*M:0*/" for the zero value. + {"unmarked renders nothing", Attributes{}, ""}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := Encode(tc.attrs) + if got != tc.want { + t.Fatalf("Encode = %q, want %q", got, tc.want) + } + if tc.want == "" { + return + } + // Round-trip through a minimal column definition. + decoded := Decode("CREATE TABLE t (id INTEGER " + got + " PRIMARY KEY)") + attrs, ok := decoded["id"] + if !ok { + t.Fatalf("Decode lost the marker %q entirely", got) + } + if attrs != tc.attrs { + t.Errorf("round trip = %+v, want %+v", attrs, tc.attrs) + } + }) + } +} + +// TestWidthMax is the single source of the ceiling every other component +// derives its bound from, so the numbers are pinned against FINAL.md's table +// rather than recomputed at each use. +// +// Mutation: use 1< nth column); + // "name" gains event_id's marker and the assertions below fire. + const ddl = "CREATE TABLE `events` (\n" + + " tenant TEXT NOT NULL,\n" + + " name TEXT DEFAULT 'no /*M:8*/ marker here',\n" + + " event_id INTEGER /*M:32a*/ PRIMARY KEY,\n" + + " qty INTEGER /*M:16u*/ DEFAULT (1 + 2),\n" + + " note TEXT\n" + + ")" + + got := Decode(ddl) + + if len(got) != 2 { + t.Fatalf("decoded %d markers, want 2: %+v", len(got), got) + } + if attrs := got["event_id"]; attrs != (Attributes{Bits: 32, ExplicitAutoInc: true}) { + t.Errorf("event_id = %+v, want 32/signed/auto", attrs) + } + // A marker after which a parenthesised DEFAULT follows must still land on + // its own column, and the paren must not swallow the next column. + // Mutation: split the column list on every comma regardless of depth. + if attrs := got["qty"]; attrs != (Attributes{Bits: 16, Unsigned: true}) { + t.Errorf("qty = %+v, want 16/unsigned/no-auto", attrs) + } + // The marker-looking text inside a string literal is data, not a marker. + // Mutation: drop the string-literal skip in findMarker. + if _, ok := got["name"]; ok { + t.Error("a marker inside a string literal was decoded as a marker") + } + if _, ok := got["tenant"]; ok { + t.Error("an unmarked column gained a marker") + } +} + +// TestDecodeEdgeCases covers the shapes that would otherwise be found in +// production rather than in a test. +func TestDecodeEdgeCases(t *testing.T) { + t.Run("table with no markers decodes to nothing", func(t *testing.T) { + // Mutation: return a non-nil empty map and have the caller treat + // "present" as "marked". + if got := Decode("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"); len(got) != 0 { + t.Errorf("decoded %+v, want nothing", got) + } + }) + + t.Run("a column named like a marker is a name", func(t *testing.T) { + got := Decode("CREATE TABLE t (`/*M:32a*/` TEXT, id INTEGER /*M:8u*/ PRIMARY KEY)") + if attrs := got["id"]; attrs != (Attributes{Bits: 8, Unsigned: true}) { + t.Errorf("id = %+v, want 8/unsigned", attrs) + } + if len(got) != 1 { + t.Errorf("decoded %d markers, want 1: %+v", len(got), got) + } + }) + + t.Run("table constraints are not columns", func(t *testing.T) { + got := Decode("CREATE TABLE t (a INTEGER /*M:24*/, b TEXT, PRIMARY KEY (a, b))") + if len(got) != 1 { + t.Errorf("decoded %d markers, want 1: %+v", len(got), got) + } + }) + + t.Run("a malformed marker degrades to unmarked", func(t *testing.T) { + // 64 is not a legal width; the column must fall back to the 64-bit + // path rather than failing the schema load. + // Mutation: accept any integer width in parseMarkerBody. + if got := Decode("CREATE TABLE t (id INTEGER /*M:64a*/ PRIMARY KEY)"); len(got) != 0 { + t.Errorf("decoded %+v from a malformed marker, want nothing", got) + } + if got := Decode("CREATE TABLE t (id INTEGER /*M:32z*/ PRIMARY KEY)"); len(got) != 0 { + t.Errorf("decoded %+v from an unknown flag, want nothing", got) + } + }) +} + +// TestStrip pins what a client sees. SHOW CREATE TABLE hands sqlite_master.sql +// to the client verbatim, so an unstripped marker would appear in every schema +// dump and in any tool that round-trips DDL. +func TestStrip(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + { + // Mutation: return the input unchanged; the marker leaks. + name: "marker and its trailing space are removed", + in: "CREATE TABLE t (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)", + want: "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)", + }, + { + name: "several markers", + in: "CREATE TABLE t (a INTEGER /*M:8*/, b INTEGER /*M:16u*/ DEFAULT (1))", + want: "CREATE TABLE t (a INTEGER, b INTEGER DEFAULT (1))", + }, + { + // Inside a string literal the text is data and must survive. + // Mutation: strip with a plain string replace. + name: "marker-shaped text inside a literal survives", + in: "CREATE TABLE t (v TEXT DEFAULT 'keep /*M:8*/ me')", + want: "CREATE TABLE t (v TEXT DEFAULT 'keep /*M:8*/ me')", + }, + { + // Unmarked DDL must come back byte-identical, or every existing + // table's SHOW CREATE output changes. + // Mutation: unconditionally collapse whitespace. + name: "unmarked DDL is unchanged", + in: "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)", + want: "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := Strip(tc.in); got != tc.want { + t.Errorf("Strip =\n %q\nwant\n %q", got, tc.want) + } + }) + } +} diff --git a/protocol/query/transform/rule.go b/protocol/query/transform/rule.go index 916d3501..9281f08c 100644 --- a/protocol/query/transform/rule.go +++ b/protocol/query/transform/rule.go @@ -56,6 +56,15 @@ type SchemaInfo struct { // index into such a statement's VALUES tuple that holds the id. // -1 when AutoIncrementColumn is empty. AutoIncrementOrdinal int + + // AutoIncrementWidth is the MySQL width the auto-increment column was + // declared with: 8, 16, 24 or 32. Zero means no width marker, which is + // BIGINT and every table created before markers existed; those keep the + // 64-bit id path. + AutoIncrementWidth int + // AutoIncrementUnsigned mirrors the MySQL UNSIGNED modifier on that column. + // Meaningless when AutoIncrementWidth is zero. + AutoIncrementUnsigned bool } // HasAutoIncrement returns true if table has an auto-increment column diff --git a/protocol/query/transform/table_utils.go b/protocol/query/transform/table_utils.go index 9f80ad7b..b7fb6269 100644 --- a/protocol/query/transform/table_utils.go +++ b/protocol/query/transform/table_utils.go @@ -2,7 +2,9 @@ package transform import ( "fmt" + "strings" + "github.com/maxpert/marmot/protocol/query/transform/intmarker" "vitess.io/vitess/go/vt/sqlparser" ) @@ -80,3 +82,65 @@ func searchTableExpr(expr sqlparser.TableExpr, targetAlias string) (tableName, a return "", "", false } + +// collapseIntegerTypeWithMarker rewrites a MySQL integer column type into the +// single type SQLite has, carrying the declared width as a marker comment, and +// strips the modifiers SQLite rejects. It reports whether it changed anything. +// +// Shared by CREATE TABLE (IntTypeRule) and ALTER TABLE ADD/MODIFY/CHANGE COLUMN +// (AlterTableColumnTypeRule). Before it was shared, ALTER only ran +// stripMySQLColumnType, which does not collapse the type and does not remove +// AUTO_INCREMENT, so "ALTER TABLE t ADD COLUMN x INT AUTO_INCREMENT" reached +// SQLite with a keyword it cannot parse. +// +// The stored token must stay exactly "INTEGER": only that spelling makes a +// PRIMARY KEY an alias of the rowid, and "INT PRIMARY KEY" does not, which +// would make LAST_INSERT_ID() report an unrelated internal rowid. BIGINT and +// non-integer types get no marker, so their DDL text is byte-identical to what +// it was before markers existed. +func collapseIntegerTypeWithMarker(colType *sqlparser.ColumnType) bool { + if colType == nil { + return false + } + + upperType := strings.ToUpper(colType.Type) + if !isIntegerType(upperType) { + return false + } + + // Capture the declared width BEFORE the strips below erase it. + marker := "" + if bits, narrow := intmarker.BitsForType(upperType); narrow { + marker = intmarker.Encode(intmarker.Attributes{ + Bits: bits, + Unsigned: colType.Unsigned, + ExplicitAutoInc: colType.Options != nil && colType.Options.Autoincrement, + }) + } + + modified := false + if colType.Options != nil && colType.Options.Autoincrement { + colType.Options.Autoincrement = false + modified = true + } + if colType.Unsigned { + colType.Unsigned = false + modified = true + } + if colType.Zerofill { + colType.Zerofill = false + modified = true + } + + want := "INTEGER" + if marker != "" { + // Spaced, matching the form verified against sqlite3 in the design: + // "id INTEGER /*M:32a*/ PRIMARY KEY". + want = "INTEGER " + marker + } + if colType.Type != want { + colType.Type = want + modified = true + } + return modified +} diff --git a/protocol/server.go b/protocol/server.go index f4405b4f..8a79f945 100644 --- a/protocol/server.go +++ b/protocol/server.go @@ -681,7 +681,7 @@ func (s *MySQLServer) writeEOF(w io.Writer, seq byte, session *ConnectionSession } func (s *MySQLServer) writeError(w io.Writer, seq byte, code uint16, msg string) error { - return s.writeErrorWithState(w, seq, code, "HY000", msg) + return s.writeErrorWithState(w, seq, code, SQLStateGeneral, msg) } func (s *MySQLServer) writeErrorWithState(w io.Writer, seq byte, code uint16, sqlState, msg string) error { @@ -1075,7 +1075,7 @@ func (s *MySQLServer) handleStmtPrepare(conn net.Conn, session *ConnectionSessio Str("query", sql). Err(err). Msg("Failed to process SQL in PREPARE") - _ = s.writeError(conn, 1, 1064, err.Error()) + _ = s.writeError(conn, 1, ErrCodeParseError, err.Error()) return } @@ -1089,7 +1089,7 @@ func (s *MySQLServer) handleStmtPrepare(conn net.Conn, session *ConnectionSessio Str("query", sql). Str("error", errorMsg). Msg("Invalid SQL in PREPARE") - _ = s.writeError(conn, 1, 1064, errorMsg) + _ = s.writeError(conn, 1, ErrCodeParseError, errorMsg) return } @@ -1259,7 +1259,7 @@ func (s *MySQLServer) handleStmtSendLongData(session *ConnectionSession, payload // accumulated via COM_STMT_SEND_LONG_DATA for the statement and responds OK. func (s *MySQLServer) handleStmtReset(conn net.Conn, session *ConnectionSession, payload []byte) { if len(payload) < 4 { - _ = s.writeError(conn, 1, 1064, "Invalid COM_STMT_RESET packet") + _ = s.writeError(conn, 1, ErrCodeParseError, "Invalid COM_STMT_RESET packet") return } stmtID := binary.LittleEndian.Uint32(payload[0:4]) @@ -1280,7 +1280,7 @@ func (s *MySQLServer) handleStmtReset(conn net.Conn, session *ConnectionSession, func (s *MySQLServer) handleStmtExecute(conn net.Conn, session *ConnectionSession, payload []byte) { if len(payload) < 9 { - _ = s.writeError(conn, 1, 1064, "Invalid COM_STMT_EXECUTE packet") + _ = s.writeError(conn, 1, ErrCodeParseError, "Invalid COM_STMT_EXECUTE packet") return } @@ -1313,7 +1313,7 @@ func (s *MySQLServer) handleStmtExecute(conn net.Conn, session *ConnectionSessio // Parse NULL bitmap and new-params-bound-flag nullBitmapLen := (int(stmt.ParamCount) + 7) / 8 if len(payload) < 9+nullBitmapLen+1 { - _ = s.writeError(conn, 1, 1064, "Invalid parameter data") + _ = s.writeError(conn, 1, ErrCodeParseError, "Invalid parameter data") return } @@ -1327,7 +1327,7 @@ func (s *MySQLServer) handleStmtExecute(conn net.Conn, session *ConnectionSessio if newParamsBoundFlag == 1 { // New types provided - parse and cache them if len(payload) < offset+int(stmt.ParamCount)*2 { - _ = s.writeError(conn, 1, 1064, "Invalid parameter types") + _ = s.writeError(conn, 1, ErrCodeParseError, "Invalid parameter types") return } paramTypes = make([]byte, int(stmt.ParamCount)*2) @@ -1386,7 +1386,7 @@ func (s *MySQLServer) handleStmtExecute(conn net.Conn, session *ConnectionSessio _ = s.writeErrorWithState(conn, 1, 1264, "22003", fmt.Sprintf("Out of range value for parameter %d", i)) } else { - _ = s.writeError(conn, 1, 1064, fmt.Sprintf("Failed to parse parameter %d: %v", i, err)) + _ = s.writeError(conn, 1, ErrCodeParseError, fmt.Sprintf("Failed to parse parameter %d: %v", i, err)) } return } From 541fc0fa1ad8cba17dbda4b1a21cf322f57f20ef Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Wed, 23 Sep 2026 13:32:59 -0500 Subject: [PATCH 04/17] feat(autoinc): cluster-wide range claims for narrow AUTO_INCREMENT columns Step 3b of the narrow AUTO_INCREMENT id design. A node that must mint ids for an INT (or narrower) AUTO_INCREMENT column first claims a range of ids through a quorum, so no two nodes can hand out the same id. Nothing calls the allocator yet; Step 3c wires it into the insert path. On its own this commit changes no insert behaviour, and it is not meant to ship without 3c. Claim protocol. Claim rows live in the existing system database, keyed by (db, tbl), in a hidden table the client cannot write (1142/42000) and CDC never replicates. A claim rides an existing statement type with a flag and a msgpack payload (proto Statement fields 9 and 10), so an older binary rejects it at the CDC-row gate. A participant votes under the claim key's intent lock, reading its own stored base and deriving the width ceiling from its own sqlite_master: accept only when storedBase <= prevBase, newBase == prevBase, size >= 1 and the range fits the column. A rejection returns the participant's base (TransactionResponse field 7) and the coordinator retries above it, pinned to QUORUM, with jittered backoff and a retry on declined PREPARE rounds, then 1205. COMMIT applies the claim only if the stored base has not moved, and refuses (never ACKs) when the claim intent is gone. The system database syncs every commit, with fullfsync on darwin, so an ACKed base survives an OS crash. Seeding. DDL seeds the base to max(MAX(col), declared AUTO_INCREMENT=N-1), raise-only, atomically with the DDL. The width marker gains an optional floor, /*M:[u][a][:]*/. A column whose existing ids already exceed its width is refused with 1264/22003, and that code now survives the participant-to-coordinator hop (TransactionResponse field 8). An ALTER that sets a floor without tagging a column, or sets only AUTO_INCREMENT=N, is refused with 1235 instead of being silently dropped. Pre-existing defects fixed on the way, each with a regression test: - Row locks: a stale GC marker let a later writer evict a live intent holder, and emptying a table's row-lock map could drop a lock another transaction had just taken. Markers are gone and maps are kept. - A writer could take over a prepared transaction's lock after the heartbeat timeout, so two conflicting DML commits were both ACKed and one write was silently lost. Heartbeat eviction is removed; a dead coordinator's prepared rows are freed by the stale-transaction GC, including transactions recovered from Pebble after a restart. A DML COMMIT whose prepared rows are gone is refused. - Anti-entropy catch-up on a running node swapped a database's files under open connections and could replace the system database. It now detaches only the database being restored: a SQLite commit hook refuses writes from the moment of the detach (reaching clients as 1213/40001), in-flight writes are drained, lifecycle operations no longer hold the manager lock while stopping GC, and startup restores keep the higher claim base. - Snapshot exports are reference-counted, so overlapping catch-ups no longer leak full database copies; orphans are swept at startup. Benchmarks (same session, alone, medians): claim apply 3.1 ms with fullfsync on darwin, 42 us without; a user write commit costs about 1-4% more for the gate; PREPARE and row-lock paths are unchanged or faster. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- common/internal_tables.go | 30 + coordinator/autoinc_claim.go | 186 +++ coordinator/autoinc_claim_bench_test.go | 34 + coordinator/autoinc_claim_test.go | 385 ++++++ coordinator/autoinc_commit_request_test.go | 55 + coordinator/cluster.go | 2 +- coordinator/errors.go | 45 +- coordinator/errors_test.go | 48 + coordinator/write_coordinator.go | 80 +- .../write_coordinator_benchmarks_test.go | 2 +- .../write_coordinator_executeprepare_test.go | 38 +- ...write_coordinator_remote_rejection_test.go | 2 +- db/autoinc_claim.go | 363 ++++++ db/autoinc_claim_bench_test.go | 85 ++ db/autoinc_claim_test.go | 269 ++++ db/autoinc_commit_test.go | 454 +++++++ db/autoinc_hookdb_test.go | 131 ++ db/autoinc_prepare_test.go | 368 ++++++ db/autoinc_restore_merge_test.go | 164 +++ db/autoinc_seed.go | 124 ++ db/autoinc_seed_test.go | 334 +++++ db/database_manager.go | 510 ++++++-- db/database_manager_cdc_test.go | 9 +- db/database_manager_test.go | 2 +- db/db_integration.go | 160 ++- db/ddl_prepare_validate.go | 151 ++- db/ddl_prepare_validate_autoinc_code_test.go | 100 ++ db/ddl_prepare_validate_test.go | 106 ++ db/detach_drain_test.go | 433 +++++++ db/meta_schema.go | 16 + db/meta_store.go | 6 +- db/meta_store_gc_intent_test.go | 234 ++-- db/meta_store_memory.go | 37 +- db/meta_store_pebble.go | 279 +++-- db/prepared_intent_test.go | 383 ++++++ db/prepared_recovery_test.go | 329 +++++ db/replication_engine.go | 246 +++- db/row_lock_store.go | 241 +--- db/row_lock_store_test.go | 334 ++--- db/snapshot/restorer.go | 38 + db/snapshot/restorer_test.go | 60 + db/sqlite_batch_committer.go | 22 +- db/sqlite_batch_committer_test.go | 29 + db/sqlite_driver.go | 75 +- db/transaction.go | 65 + db/write_gate.go | 78 ++ db/write_gate_bench_test.go | 36 + grpc/anti_entropy.go | 27 + grpc/autoinc_claim_handler_test.go | 150 +++ grpc/autoinc_claim_wire_test.go | 97 ++ grpc/catch_up.go | 210 ++-- grpc/catch_up_detach_test.go | 400 ++++++ grpc/catch_up_test.go | 193 ++- grpc/marmot.pb.go | 110 +- grpc/marmot.proto | 45 + grpc/replication_handler.go | 2 + grpc/replicator_adapter.go | 43 +- grpc/server.go | 297 ++--- grpc/snapshot_export.go | 130 ++ grpc/snapshot_export_test.go | 256 ++++ grpc/vector_statement.go | 12 +- marmot.go | 26 +- protocol/autoinc_claim.go | 54 + protocol/autoinc_claim_test.go | 35 + protocol/error_mapper.go | 12 +- protocol/error_mapper_test.go | 26 + protocol/mysql_errors.go | 55 +- protocol/mysqlcode/mysqlcode.go | 11 + protocol/query/pipeline.go | 17 + protocol/query/rules/claim_table_guard.go | 160 +++ .../query/rules/claim_table_guard_test.go | 98 ++ .../transform/alter_table_column_type.go | 90 +- .../transform/alter_table_column_type_test.go | 104 ++ protocol/query/transform/int_type.go | 6 +- protocol/query/transform/int_type_test.go | 87 +- .../query/transform/intmarker/intmarker.go | 28 +- .../transform/intmarker/intmarker_test.go | 84 +- protocol/query/transform/table_utils.go | 55 +- protocol/query/transpiler_rule_error_test.go | 25 + protocol/server.go | 2 +- protocol/transaction.go | 15 + telemetry/collector.go | 9 +- telemetry/metrics.go | 10 +- test/autoinc_claim_cluster_test.go | 503 ++++++++ test/autoinc_claim_inprocess_test.go | 1083 +++++++++++++++++ test/cluster_integration_test.go | 13 +- test/crash_recovery_test.go | 26 +- test/ddl_replication_test.go | 6 +- 88 files changed, 10496 insertions(+), 1294 deletions(-) create mode 100644 common/internal_tables.go create mode 100644 coordinator/autoinc_claim.go create mode 100644 coordinator/autoinc_claim_bench_test.go create mode 100644 coordinator/autoinc_claim_test.go create mode 100644 coordinator/autoinc_commit_request_test.go create mode 100644 db/autoinc_claim.go create mode 100644 db/autoinc_claim_bench_test.go create mode 100644 db/autoinc_claim_test.go create mode 100644 db/autoinc_commit_test.go create mode 100644 db/autoinc_hookdb_test.go create mode 100644 db/autoinc_prepare_test.go create mode 100644 db/autoinc_restore_merge_test.go create mode 100644 db/autoinc_seed.go create mode 100644 db/autoinc_seed_test.go create mode 100644 db/ddl_prepare_validate_autoinc_code_test.go create mode 100644 db/detach_drain_test.go create mode 100644 db/prepared_intent_test.go create mode 100644 db/prepared_recovery_test.go create mode 100644 db/write_gate.go create mode 100644 db/write_gate_bench_test.go create mode 100644 grpc/autoinc_claim_handler_test.go create mode 100644 grpc/autoinc_claim_wire_test.go create mode 100644 grpc/catch_up_detach_test.go create mode 100644 grpc/snapshot_export.go create mode 100644 grpc/snapshot_export_test.go create mode 100644 protocol/autoinc_claim.go create mode 100644 protocol/autoinc_claim_test.go create mode 100644 protocol/query/rules/claim_table_guard.go create mode 100644 protocol/query/rules/claim_table_guard_test.go create mode 100644 test/autoinc_claim_cluster_test.go create mode 100644 test/autoinc_claim_inprocess_test.go diff --git a/common/internal_tables.go b/common/internal_tables.go new file mode 100644 index 00000000..accc7278 --- /dev/null +++ b/common/internal_tables.go @@ -0,0 +1,30 @@ +package common + +import "strings" + +// InternalTablePrefix marks every SQLite table Marmot creates for its own +// protocol state rather than for client data - e.g. AutoIncClaimTableName +// below. It is the single Go definition of this literal: both the protocol +// package (which must refuse client SQL naming such a table) and the db +// package (which owns the tables themselves) import it from here, so the two +// can never drift apart. +// +// NOTE: several "name NOT LIKE '__marmot__%'" filters exist elsewhere in this +// codebase (protocol/handlers) as SQLite queries. SQLite's LIKE treats "_" as +// a single-character wildcard, so that pattern matches more than the literal +// prefix - it is not a substitute for this constant, and this constant must +// not be turned into a LIKE pattern either. +const InternalTablePrefix = "__marmot__" + +// AutoIncClaimTableName is the hidden per-database table holding each +// narrow AUTO_INCREMENT table's allocation base. db.AutoIncClaimTable aliases +// this constant; see db/autoinc_claim.go for the schema and write path. +const AutoIncClaimTableName = InternalTablePrefix + "autoinc" + +// IsInternalTableName reports whether name - compared case-insensitively, as +// MySQL/SQLite identifiers are - names one of Marmot's own internal tables +// rather than a client table. Used to refuse client SQL that targets Marmot's +// internal state as a class, not just the one table that exists today. +func IsInternalTableName(name string) bool { + return strings.HasPrefix(strings.ToLower(name), InternalTablePrefix) +} diff --git a/coordinator/autoinc_claim.go b/coordinator/autoinc_claim.go new file mode 100644 index 00000000..b8187366 --- /dev/null +++ b/coordinator/autoinc_claim.go @@ -0,0 +1,186 @@ +package coordinator + +import ( + "context" + "errors" + "fmt" + "math/rand/v2" + "time" + + "github.com/maxpert/marmot/protocol" +) + +// maxClaimAttempts bounds how many PREPARE rounds ClaimRange spends on one +// call. A rejection that carries a usable base means real progress - another +// claimant is ahead, not that this claim is wrong - so retrying is the right +// response; this bound exists only so a caller always gets a definitive +// answer within one call instead of spinning against sustained contention. +const maxClaimAttempts = 8 + +// claimBackoffBase and claimBackoffCap shape the wait before a claim retries +// a round that a participant declined without a verdict - in practice a lost +// race for the claim key, held by another claim until its COMMIT has synced +// the system database on a majority. With c = min(claimBackoffCap, +// claimBackoffBase<= 1 && +// newBase+size <= widthMax. +// +// WriteConsistency is pinned to ConsistencyQuorum unconditionally - it never +// reads a caller-supplied level or the cluster's configured write default. +// QuorumSize(ConsistencyOne, n) and QuorumSize(ConsistencyLocalOne, n) are +// both 1 (coordinator/quorum.go), and a quorum of 1 does not intersect +// another quorum of 1: two claimants could each get a "yes" from a disjoint +// single node and mint the same range. The claim protocol's soundness +// (db/replication_engine.go's prepareAutoIncClaim doc) depends on any two +// quorums sharing a participant. A client can otherwise steer consistency +// with a SQL hint (coordinator/handler.go's routeQuery) or via the cluster's +// configured default (cfg.Config.Replication.DefaultWriteConsist) - neither +// must be allowed to downgrade a claim to a single node. +// +// On a PREPARE rejection, ClaimRange retries above the highest base any +// participant reported that round (executePreparePhase tracks this as +// maxAutoIDStoredBase and attaches it to whichever of *LocalPrepareError or +// *RemotePrepareRejectedError is on record - see coordinator/write_coordinator.go). +// That retry is immediate: the rejection means another claim committed. +// +// A round that failed because a participant declined without a verdict - +// quorum not achieved while at least one participant answered, which for a +// claim means it lost the race for the claim key - is retried with the same +// proposal after a jittered backoff (claimBackoffBase, claimBackoffCap). Any +// other failure that carries no usable base - quorum not achieved with no +// participant answering, a transport failure - is not something retrying can +// fix by itself, so it ends the attempt loop immediately rather than spending +// the full budget. +// +// On exhausting maxClaimAttempts, or on hitting a failure with no usable +// base, ClaimRange returns protocol.ErrLockWaitTimeout() (MySQL 1205): the +// standard signal this codebase already uses for "retry me" (see +// runPreparePhase's handling of write-write conflicts). It is wrapped with +// %w around the last attempt's cause so a caller using errors.As can still +// recover the retryable code, and errors.Unwrap the underlying reason for +// logs. +// +// It never touches hookDB and never calls ExecuteLocalWithHooks: it only +// builds a statement and drives it through WriteTransaction, exactly like +// any other 2PC write. +func (wc *WriteCoordinator) ClaimRange(ctx context.Context, database, table string, prevBase, size uint64) (newBase uint64, granted uint64, err error) { + newBase = prevBase + + var lastErr error + declinedRounds := 0 + for attempt := 0; attempt < maxClaimAttempts; attempt++ { + payload, encErr := protocol.EncodeAutoIncClaim(protocol.AutoIncClaim{ + Table: table, + PrevBase: prevBase, + NewBase: newBase, + Size: size, + }) + if encErr != nil { + return 0, 0, fmt.Errorf("encode auto-increment claim for %s.%s: %w", database, table, encErr) + } + + startTS := wc.clock.Now() + txn := &Transaction{ + ID: startTS.ToTxnID(), + NodeID: wc.nodeID, + StartTS: startTS, + Database: database, + // Pinned regardless of any caller or cluster default - see the + // method doc for why a quorum of 1 cannot be trusted here. + WriteConsistency: protocol.ConsistencyQuorum, + Statements: []protocol.Statement{{ + Type: protocol.StatementInsert, + TableName: table, + Database: database, + IntentKey: []byte(protocol.AutoIncClaimKey(database, table)), + AutoIDClaim: true, + AutoIDClaimPayload: payload, + }}, + } + + attemptErr := wc.WriteTransaction(ctx, txn) + if attemptErr == nil { + return newBase, size, nil + } + lastErr = attemptErr + + if base, ok := autoIDStoredBaseFromRejection(attemptErr); ok && base > prevBase { + prevBase = base + newBase = base + continue + } + if !declinedByParticipant(attemptErr) || attempt == maxClaimAttempts-1 { + break + } + if err := sleepClaimBackoff(ctx, declinedRounds); err != nil { + lastErr = err + break + } + declinedRounds++ + } + + return 0, 0, fmt.Errorf("auto-increment claim for %s.%s could not be granted: %w (last cause: %v)", + database, table, protocol.ErrLockWaitTimeout(), lastErr) +} + +// autoIDStoredBaseFromRejection extracts the highest base a rejecting +// participant reported from either shape runPreparePhase can return: a +// *RemotePrepareRejectedError (quorum failed to form and a remote rejection +// is on record) or a *LocalPrepareError (this node's own PREPARE rejected, +// possibly wrapped in *CoordinatorNotParticipatedError). Neither wraps the +// other, so both are checked. A zero base reports "no usable base" (ok=false): +// per PrepareResult.AutoIDStoredBase's own contract, zero means this +// response was not an AUTO_INCREMENT claim rejection at all. +func autoIDStoredBaseFromRejection(err error) (base uint64, ok bool) { + var remoteRejection *RemotePrepareRejectedError + if errors.As(err, &remoteRejection) && remoteRejection.AutoIDStoredBase > 0 { + return remoteRejection.AutoIDStoredBase, true + } + var localErr *LocalPrepareError + if errors.As(err, &localErr) && localErr.AutoIDStoredBase > 0 { + return localErr.AutoIDStoredBase, true + } + return 0, false +} + +// declinedByParticipant reports whether a claim round failed to reach quorum +// while at least one participant answered without a verdict. +func declinedByParticipant(err error) bool { + var quorumErr *QuorumNotAchievedError + return errors.As(err, &quorumErr) && quorumErr.Phase == "prepare" && quorumErr.Declined > 0 +} + +// sleepClaimBackoff waits the k-th jittered claim backoff, or until ctx ends. +func sleepClaimBackoff(ctx context.Context, k int) error { + ceiling := claimBackoffBase << k + if ceiling <= 0 || ceiling > claimBackoffCap { + ceiling = claimBackoffCap + } + half := ceiling / 2 + timer := time.NewTimer(half + time.Duration(rand.Int64N(int64(ceiling-half)))) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/coordinator/autoinc_claim_bench_test.go b/coordinator/autoinc_claim_bench_test.go new file mode 100644 index 00000000..19f1073c --- /dev/null +++ b/coordinator/autoinc_claim_bench_test.go @@ -0,0 +1,34 @@ +package coordinator + +import ( + "context" + "testing" + "time" + + "github.com/maxpert/marmot/hlc" +) + +// BenchmarkClaimRange measures one uncontested claim end to end through the +// coordinator's 2PC against an in-process fake participant, so what it times +// is the claim's own work - building the statement, encoding the payload, +// pinning quorum, running PREPARE and COMMIT - and not a network. +// +// It is the allocation budget for the range allocator, which amortises +// one ClaimRange over a whole range of ids, so this cost is paid once per +// range, not once per insert. +func BenchmarkClaimRange(b *testing.B) { + InitTestTelemetry() + + fake := newClaimStepReplicator() + nodeProvider := newMockNodeProvider([]uint64{1}) + wc := NewWriteCoordinator(1, nodeProvider, fake, fake, 200*time.Millisecond, hlc.NewClock(1)) + ctx := context.Background() + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, _, err := wc.ClaimRange(ctx, "testdb", "orders", uint64(i), 50); err != nil { + b.Fatalf("ClaimRange: %v", err) + } + } +} diff --git a/coordinator/autoinc_claim_test.go b/coordinator/autoinc_claim_test.go new file mode 100644 index 00000000..f7c52b44 --- /dev/null +++ b/coordinator/autoinc_claim_test.go @@ -0,0 +1,385 @@ +package coordinator + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" +) + +// claimStepReplicator is a minimal fake Replicator used only by the +// ClaimRange tests below. It answers PREPARE from a per-node queue of canned +// responses, consumed one per call, so a test can script a participant +// rejecting on an early attempt and accepting once ClaimRange retries above +// the base it reported. Once a node's queue is empty, PREPARE succeeds. +// COMMIT and ABORT always succeed: ClaimRange's retry logic only inspects +// PREPARE-phase errors. +type claimStepReplicator struct { + mu sync.Mutex + prepareQueue map[uint64][]*ReplicationResponse + prepareCalls map[uint64][]*ReplicationRequest + fail map[uint64]bool // simulate a transport failure (missing ACK), not a rejection +} + +func newClaimStepReplicator() *claimStepReplicator { + return &claimStepReplicator{ + prepareQueue: make(map[uint64][]*ReplicationResponse), + prepareCalls: make(map[uint64][]*ReplicationRequest), + fail: make(map[uint64]bool), + } +} + +func (r *claimStepReplicator) push(nodeID uint64, resp *ReplicationResponse) { + r.mu.Lock() + defer r.mu.Unlock() + r.prepareQueue[nodeID] = append(r.prepareQueue[nodeID], resp) +} + +func (r *claimStepReplicator) setTransportFailure(nodeID uint64) { + r.mu.Lock() + defer r.mu.Unlock() + r.fail[nodeID] = true +} + +func (r *claimStepReplicator) prepareCallCount(nodeID uint64) int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.prepareCalls[nodeID]) +} + +func (r *claimStepReplicator) prepareCall(nodeID uint64, i int) *ReplicationRequest { + r.mu.Lock() + defer r.mu.Unlock() + return r.prepareCalls[nodeID][i] +} + +func (r *claimStepReplicator) ReplicateTransaction(_ context.Context, nodeID uint64, req *ReplicationRequest) (*ReplicationResponse, error) { + if req.Phase != PhasePrep { + return &ReplicationResponse{Success: true}, nil + } + + r.mu.Lock() + defer r.mu.Unlock() + r.prepareCalls[nodeID] = append(r.prepareCalls[nodeID], req) + + if r.fail[nodeID] { + return nil, errors.New("simulated transport failure") + } + + q := r.prepareQueue[nodeID] + if len(q) == 0 { + return &ReplicationResponse{Success: true}, nil + } + resp := q[0] + r.prepareQueue[nodeID] = q[1:] + return resp, nil +} + +func rejectionWithBase(base uint64) *ReplicationResponse { + return &ReplicationResponse{ + Success: false, + Rejected: true, + Error: "auto-increment claim is stale", + AutoIDStoredBase: base, + } +} + +// decodePreparedClaim pulls the AutoIncClaim payload off the single claim +// statement a PREPARE request must carry. +func decodePreparedClaim(t *testing.T, req *ReplicationRequest) protocol.AutoIncClaim { + t.Helper() + if len(req.Statements) != 1 { + t.Fatalf("prepare request carries %d statements, want 1", len(req.Statements)) + } + stmt := req.Statements[0] + if !stmt.AutoIDClaim { + t.Fatal("statement does not carry the claim flag") + } + if len(stmt.EncodedRow) != 0 { + t.Fatal("claim statement carries an EncodedRow; the design requires it carry none") + } + if len(stmt.IntentKey) == 0 { + t.Fatal("claim statement carries no IntentKey") + } + claim, err := protocol.DecodeAutoIncClaim(stmt.AutoIDClaimPayload) + if err != nil { + t.Fatalf("DecodeAutoIncClaim: %v", err) + } + return claim +} + +// TestClaimRange_SuccessfulClaimSingleRound pins the shape of an +// uncontested claim: one PREPARE round, a statement carrying the flag, a +// non-empty intent key, a decodable payload with NewBase == PrevBase, and no +// EncodedRow. +func TestClaimRange_SuccessfulClaimSingleRound(t *testing.T) { + InitTestTelemetry() + + fake := newClaimStepReplicator() + nodeProvider := newMockNodeProvider([]uint64{1}) + wc := NewWriteCoordinator(1, nodeProvider, fake, fake, 200*time.Millisecond, hlc.NewClock(1)) + + newBase, granted, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, 50) + if err != nil { + t.Fatalf("ClaimRange: %v", err) + } + if newBase != 100 { + t.Errorf("newBase = %d, want 100", newBase) + } + if granted != 50 { + t.Errorf("granted = %d, want 50", granted) + } + + if got := fake.prepareCallCount(1); got != 1 { + t.Fatalf("PREPARE calls = %d, want exactly 1 (single round)", got) + } + req := fake.prepareCall(1, 0) + claim := decodePreparedClaim(t, req) + if claim.Table != "orders" { + t.Errorf("claim table = %q, want orders", claim.Table) + } + if claim.PrevBase != 100 || claim.NewBase != 100 { + t.Errorf("claim base = {prev:%d new:%d}, want both 100", claim.PrevBase, claim.NewBase) + } + if claim.Size != 50 { + t.Errorf("claim size = %d, want 50", claim.Size) + } +} + +// TestClaimRange_ConsistencyIsAlwaysQuorum pins that ClaimRange pins +// ConsistencyQuorum unconditionally: quorums of 1 (ConsistencyOne, +// ConsistencyLocalOne) do not intersect, so a claim under either would not +// have the majority-intersection soundness argument prepareAutoIncClaim +// relies on. +// +// Proven behaviourally, not by inspecting a field: with a 3-node cluster, +// only the coordinator's own PREPARE succeeds (1 ack) while the other two +// nodes never answer. A quorum of 1 (ConsistencyOne/LocalOne) would let this +// succeed; requiring the majority of 3 (ConsistencyQuorum = 2) must fail +// quorum instead. +// +// Mutation: make ClaimRange honour a caller-supplied or config-default +// consistency level instead of pinning quorum. +func TestClaimRange_ConsistencyIsAlwaysQuorum(t *testing.T) { + InitTestTelemetry() + + fake := newClaimStepReplicator() + fake.setTransportFailure(2) + fake.setTransportFailure(3) + nodeProvider := newMockNodeProvider([]uint64{1, 2, 3}) + wc := NewWriteCoordinator(1, nodeProvider, fake, fake, 100*time.Millisecond, hlc.NewClock(1)) + + _, _, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, 50) + if err == nil { + t.Fatal("expected ClaimRange to fail with only 1 of 3 nodes acking - it would succeed under a quorum of 1, proving consistency was not pinned to QUORUM") + } + + var mysqlErr *protocol.MySQLError + if !errors.As(err, &mysqlErr) { + t.Fatalf("expected a *protocol.MySQLError, got %T: %v", err, err) + } + if mysqlErr.Code != protocol.ErrCodeLockTimeout { + t.Errorf("MySQL code = %d, want %d (lock wait timeout / retry)", mysqlErr.Code, protocol.ErrCodeLockTimeout) + } +} + +// TestClaimRange_RetriesAboveRejectedBase pins the single-participant retry +// path: a PREPARE rejection reporting a higher base causes exactly one retry +// whose payload carries that base, and the retry succeeds. +func TestClaimRange_RetriesAboveRejectedBase(t *testing.T) { + InitTestTelemetry() + + fake := newClaimStepReplicator() + fake.push(1, rejectionWithBase(150)) + nodeProvider := newMockNodeProvider([]uint64{1}) + wc := NewWriteCoordinator(1, nodeProvider, fake, fake, 200*time.Millisecond, hlc.NewClock(1)) + + newBase, granted, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, 50) + if err != nil { + t.Fatalf("ClaimRange: %v", err) + } + if newBase != 150 { + t.Errorf("newBase = %d, want 150 (the rejected base)", newBase) + } + if granted != 50 { + t.Errorf("granted = %d, want 50", granted) + } + + if got := fake.prepareCallCount(1); got != 2 { + t.Fatalf("PREPARE calls = %d, want exactly 2 (one rejection, one retry)", got) + } + retryClaim := decodePreparedClaim(t, fake.prepareCall(1, 1)) + if retryClaim.PrevBase != 150 || retryClaim.NewBase != 150 { + t.Errorf("retry claim base = {prev:%d new:%d}, want both 150", retryClaim.PrevBase, retryClaim.NewBase) + } +} + +// TestClaimRange_RetriesAboveMaximumRejectedBase pins the multi-participant +// case: two participants reject in the same round with different bases, and +// the retry uses the MAXIMUM, not merely the first rejection the coordinator +// happened to record. +// +// Mutation: retry using RemotePrepareRejectedError's own base (the first +// remote rejection) instead of executePreparePhase's tracked round maximum. +func TestClaimRange_RetriesAboveMaximumRejectedBase(t *testing.T) { + InitTestTelemetry() + + fake := newClaimStepReplicator() + fake.push(2, rejectionWithBase(50)) + fake.push(3, rejectionWithBase(80)) + nodeProvider := newMockNodeProvider([]uint64{1, 2, 3}) + wc := NewWriteCoordinator(1, nodeProvider, fake, fake, 200*time.Millisecond, hlc.NewClock(1)) + + newBase, granted, err := wc.ClaimRange(context.Background(), "testdb", "orders", 40, 10) + if err != nil { + t.Fatalf("ClaimRange: %v", err) + } + if newBase != 80 { + t.Errorf("newBase = %d, want 80 (the MAXIMUM of the two rejected bases, not 50)", newBase) + } + if granted != 10 { + t.Errorf("granted = %d, want 10", granted) + } + + if got := fake.prepareCallCount(2); got != 2 { + t.Fatalf("node 2 PREPARE calls = %d, want 2", got) + } + if got := fake.prepareCallCount(3); got != 2 { + t.Fatalf("node 3 PREPARE calls = %d, want 2", got) + } + retryClaim := decodePreparedClaim(t, fake.prepareCall(2, 1)) + if retryClaim.PrevBase != 80 || retryClaim.NewBase != 80 { + t.Errorf("retry claim base = {prev:%d new:%d}, want both 80 (the max)", retryClaim.PrevBase, retryClaim.NewBase) + } +} + +// TestClaimRange_EightConsecutiveRejectionsReturn1205 pins the retry bound: +// after 8 attempts all rejected (each carrying a usable, strictly increasing +// base so the loop always has a reason to retry), ClaimRange gives up and +// returns the retryable MySQL 1205 rather than looping forever. +func TestClaimRange_EightConsecutiveRejectionsReturn1205(t *testing.T) { + InitTestTelemetry() + + fake := newClaimStepReplicator() + base := uint64(100) + for i := 0; i < 8; i++ { + base += 10 + fake.push(1, rejectionWithBase(base)) + } + nodeProvider := newMockNodeProvider([]uint64{1}) + wc := NewWriteCoordinator(1, nodeProvider, fake, fake, 200*time.Millisecond, hlc.NewClock(1)) + + _, _, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, 50) + if err == nil { + t.Fatal("expected ClaimRange to give up after 8 rejections, got nil error") + } + var mysqlErr *protocol.MySQLError + if !errors.As(err, &mysqlErr) { + t.Fatalf("expected a *protocol.MySQLError, got %T: %v", err, err) + } + if mysqlErr.Code != protocol.ErrCodeLockTimeout { + t.Errorf("MySQL code = %d, want %d (lock wait timeout / retry)", mysqlErr.Code, protocol.ErrCodeLockTimeout) + } + + if got := fake.prepareCallCount(1); got != 8 { + t.Fatalf("PREPARE calls = %d, want exactly 8", got) + } +} + +// claimKeyHolder is a fake participant set that declines every PREPARE, +// without a verdict, until a deadline: another claim holds the claim key on +// each of them until its COMMIT has synced, the way a participant answers a +// lost race for the key. +type claimKeyHolder struct { + until time.Time + mu sync.Mutex + calls int +} + +func (h *claimKeyHolder) ReplicateTransaction(_ context.Context, _ uint64, req *ReplicationRequest) (*ReplicationResponse, error) { + if req.Phase != PhasePrep { + return &ReplicationResponse{Success: true}, nil + } + h.mu.Lock() + h.calls++ + h.mu.Unlock() + if time.Now().Before(h.until) { + return &ReplicationResponse{Success: false, Error: "write-write conflict: claim key locked"}, nil + } + return &ReplicationResponse{Success: true}, nil +} + +// TestClaimRange_BacksOffWhileAnotherClaimHoldsTheKey: every participant +// declines while another claim holds the key for 60ms. ClaimRange must wait +// that out within one call: its declined-round waits are at least 2, 4, 8, +// 16 and 32ms, so by the sixth round more than 60ms have passed. Retrying at +// once spends all maxClaimAttempts rounds inside the hold and returns 1205, +// which is what contending claimants did once a claim COMMIT synced with +// F_FULLFSYNC. +// +// Mutation: drop the backoff sleep (retry at once). "a claim gave up while +// another claim held the key" fires. Mutation: stop retrying declined rounds. +// The same assertion fires after one round. +func TestClaimRange_BacksOffWhileAnotherClaimHoldsTheKey(t *testing.T) { + InitTestTelemetry() + + const hold = 60 * time.Millisecond + holder := &claimKeyHolder{until: time.Now().Add(hold)} + nodeProvider := newMockNodeProvider([]uint64{1, 2, 3}) + wc := NewWriteCoordinator(1, nodeProvider, holder, holder, 200*time.Millisecond, hlc.NewClock(1)) + + start := time.Now() + newBase, granted, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, 50) + if err != nil { + t.Fatalf("a claim gave up while another claim held the key: %v", err) + } + if newBase != 100 || granted != 50 { + t.Fatalf("claim = (%d, %d), want (100, 50)", newBase, granted) + } + elapsed := time.Since(start) + if elapsed < hold { + t.Fatalf("claim granted after %v, inside the %v hold", elapsed, hold) + } + holder.mu.Lock() + calls := holder.calls + holder.mu.Unlock() + if rounds := calls / 3; rounds > maxClaimAttempts { + t.Fatalf("claim used %d PREPARE rounds, more than maxClaimAttempts", rounds) + } + t.Logf("granted after %v and %d PREPARE calls", elapsed.Round(time.Millisecond), calls) +} + +// TestClaimRange_QuorumNotAchievedReturns1205WithoutRetrying pins the other +// terminal case: a failure that carries no usable base (plain quorum +// failure, no participant explicitly rejecting) must not spend all 8 +// attempts retrying something that cannot change - it returns 1205 +// immediately, after a single round. +func TestClaimRange_QuorumNotAchievedReturns1205WithoutRetrying(t *testing.T) { + InitTestTelemetry() + + fake := newClaimStepReplicator() + fake.setTransportFailure(2) + fake.setTransportFailure(3) + nodeProvider := newMockNodeProvider([]uint64{1, 2, 3}) + wc := NewWriteCoordinator(1, nodeProvider, fake, fake, 100*time.Millisecond, hlc.NewClock(1)) + + _, _, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, 50) + if err == nil { + t.Fatal("expected an error when quorum cannot be achieved") + } + var mysqlErr *protocol.MySQLError + if !errors.As(err, &mysqlErr) { + t.Fatalf("expected a *protocol.MySQLError, got %T: %v", err, err) + } + if mysqlErr.Code != protocol.ErrCodeLockTimeout { + t.Errorf("MySQL code = %d, want %d (lock wait timeout / retry)", mysqlErr.Code, protocol.ErrCodeLockTimeout) + } + + if got := fake.prepareCallCount(1); got != 1 { + t.Fatalf("PREPARE calls to the coordinator's own node = %d, want exactly 1 (no retry spin on an unusable failure)", got) + } +} diff --git a/coordinator/autoinc_commit_request_test.go b/coordinator/autoinc_commit_request_test.go new file mode 100644 index 00000000..05290eb7 --- /dev/null +++ b/coordinator/autoinc_commit_request_test.go @@ -0,0 +1,55 @@ +package coordinator + +import ( + "testing" + + "github.com/maxpert/marmot/protocol" +) + +// TestCommitRequestCarriesClaimFlagNotPayload pins the split that makes the +// AUTO_INCREMENT range claim safe across the two phases. +// +// buildCommitRequest rebuilds every statement from a fixed list of fields, so a +// field that is not named there does not reach COMMIT. The claim FLAG must: +// it is what tells a participant to apply the claim it prepared, and without it +// every participant commits the transaction while leaving its base untouched, +// so the next claimant anywhere mints the same ids. The PAYLOAD must not: a +// participant reads the claim's values from the intent it wrote at PREPARE, +// after checking them against its own schema and its own stored base, and a +// payload on the COMMIT wire is an invitation to read them from an unchecked +// source instead. +// +// Mutations: drop the AutoIDClaim line from the struct literal (the first +// assertion fires); add AutoIDClaimPayload beside it (the second fires). +func TestCommitRequestCarriesClaimFlagNotPayload(t *testing.T) { + wc := &WriteCoordinator{nodeID: 9} + + req := wc.buildCommitRequest(&Transaction{ + ID: 4242, + Database: "testdb", + Statements: []protocol.Statement{{ + // StatementInsert matters: a claim rides a DML statement type, so + // it takes the branch that copies the fewest fields. + Type: protocol.StatementInsert, + Database: "testdb", + TableName: "__marmot__autoinc", + IntentKey: []byte("autoinc:testdb:users"), + AutoIDClaim: true, + AutoIDClaimPayload: []byte("prepare-only"), + }}, + }) + + if len(req.Statements) != 1 { + t.Fatalf("commit request carries %d statements, want 1", len(req.Statements)) + } + stmt := req.Statements[0] + if !stmt.AutoIDClaim { + t.Error("the claim flag was dropped at COMMIT, so no participant would apply the claim it prepared") + } + if stmt.AutoIDClaimPayload != nil { + t.Errorf("the claim payload reached COMMIT (%q); participants must read it from their own intent", stmt.AutoIDClaimPayload) + } + if len(stmt.IntentKey) == 0 { + t.Error("the intent key was dropped, so the claim's intent could not be addressed") + } +} diff --git a/coordinator/cluster.go b/coordinator/cluster.go index 6ea816a8..d6f99754 100644 --- a/coordinator/cluster.go +++ b/coordinator/cluster.go @@ -40,7 +40,7 @@ func consistencyNeedsMajority(level protocol.ConsistencyLevel) bool { // It is ErrCodeLockTimeout (1205) because the condition is transient and the // client should retry: gossip converges within a few seconds of a peer becoming // reachable. 1205 is the code ORM retry layers already key on, alongside 1213, -// and FINAL.md section 4 picks it for the same reason on the claim path. A +// and the AUTO_INCREMENT claim path returns it for the same reason. A // non-retryable code would turn a few seconds of convergence into a failed // transaction the application has to handle itself. func ErrMembershipNotLearned() *protocol.MySQLError { diff --git a/coordinator/errors.go b/coordinator/errors.go index b3ac48a9..084b5ff0 100644 --- a/coordinator/errors.go +++ b/coordinator/errors.go @@ -1,6 +1,10 @@ package coordinator -import "fmt" +import ( + "fmt" + + "github.com/maxpert/marmot/protocol/query/transform" +) // PrepareConflictError represents a write-write conflict detected during the prepare phase type PrepareConflictError struct { @@ -20,6 +24,10 @@ type QuorumNotAchievedError struct { TotalMembership int AliveNodes int IsRemoteQuorum bool // True if this is remote commit quorum (quorum-1) + // Declined counts participants that answered PREPARE without preparing + // and without a verdict - a lost race for a row lock, a storage error - + // as opposed to participants that never answered. + Declined int } func (e *QuorumNotAchievedError) Error() string { @@ -33,12 +41,31 @@ func (e *QuorumNotAchievedError) Error() string { // of the retry signal used for write-write conflicts. type LocalPrepareError struct { Reason string + // AutoIDStoredBase is the highest base any participant reported this + // PREPARE round, when this rejection came from an AUTO_INCREMENT range + // claim. Zero for every other rejection. + AutoIDStoredBase uint64 + // ErrorCode is the MySQL server error code the participant chose for this + // refusal, or 0 when it named none. + ErrorCode uint16 } func (e *LocalPrepareError) Error() string { return e.Reason } +// Unwrap re-types the rejection as the coded error the participant raised, so +// the protocol layer maps it to the participant's own MySQL error code rather +// than flattening every deterministic refusal to ER_UNKNOWN_ERROR. It returns +// nil when the participant named no code, which leaves the existing +// message-based classification in charge. +func (e *LocalPrepareError) Unwrap() error { + if e.ErrorCode == 0 { + return nil + } + return &transform.CodedError{Code: e.ErrorCode, Message: e.Reason} +} + // RemotePrepareRejectedError indicates prepare quorum was not achieved and at // least one remote participant explicitly rejected the transaction during // PREPARE (a deterministic "this can never apply here" verdict, for example @@ -50,6 +77,13 @@ func (e *LocalPrepareError) Error() string { type RemotePrepareRejectedError struct { NodeID uint64 Reason string + // AutoIDStoredBase is the highest base any participant reported this + // PREPARE round, when this rejection came from an AUTO_INCREMENT range + // claim. Zero for every other rejection. + AutoIDStoredBase uint64 + // ErrorCode is the MySQL server error code the rejecting participant chose, + // or 0 when it named none. + ErrorCode uint16 } // Error reports the rejecting participant's own reason so clients receive the @@ -58,6 +92,15 @@ func (e *RemotePrepareRejectedError) Error() string { return e.Reason } +// Unwrap re-types the rejection as the coded error the remote participant +// raised; see LocalPrepareError.Unwrap. +func (e *RemotePrepareRejectedError) Unwrap() error { + if e.ErrorCode == 0 { + return nil + } + return &transform.CodedError{Code: e.ErrorCode, Message: e.Reason} +} + // CoordinatorNotParticipatedError indicates the coordinator failed to participate in the prepare phase type CoordinatorNotParticipatedError struct { TxnID uint64 diff --git a/coordinator/errors_test.go b/coordinator/errors_test.go index ed36eba9..77f891ff 100644 --- a/coordinator/errors_test.go +++ b/coordinator/errors_test.go @@ -3,6 +3,10 @@ package coordinator import ( "errors" "testing" + + "github.com/mattn/go-sqlite3" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" ) func TestPrepareConflictError(t *testing.T) { @@ -255,3 +259,47 @@ func TestPartialCommitError(t *testing.T) { }) } } + +// TestPartialCommitRefusedLocallyIsNotRetryable: a local commit the write +// gate refused after the remote quorum committed is a partial commit, not a +// rolled-back transaction. Telling the client to restart it (1213) would apply +// its writes twice, so the refusal must not show through PartialCommitError. +// +// Mutation: give PartialCommitError an Unwrap returning LocalError. "a partial +// commit was reported as retryable" fires. +func TestPartialCommitRefusedLocallyIsNotRetryable(t *testing.T) { + refused := sqlite3.Error{Code: sqlite3.ErrConstraint, ExtendedCode: sqlite3.ErrConstraintCommitHook} + require.Equal(t, protocol.ErrCodeDeadlock, protocol.ConvertToMySQLError(refused).Code) + got := protocol.ConvertToMySQLError(&PartialCommitError{IsLocal: true, LocalError: refused}) + require.NotEqual(t, protocol.ErrCodeDeadlock, got.Code, "a partial commit was reported as retryable") + require.NotEqual(t, protocol.SQLStateDeadlock, got.SQLState, "a partial commit was reported as retryable") +} + +// TestPrepareRejectionErrorsCarryTheParticipantsCode pins the last hop of the +// same plumbing: the coordinator turns a participant's refusal into an error +// whose only typed content is its own struct, so unless that struct re-types +// itself as the coded error the participant raised, every deterministic +// refusal reaches the client as ER_UNKNOWN_ERROR (1105, HY000). +// +// Mutation: delete either Unwrap, or make it return the error unconditionally +// instead of nil at code 0. The matching assertion below fires. +func TestPrepareRejectionErrorsCarryTheParticipantsCode(t *testing.T) { + t.Parallel() + + local := &LocalPrepareError{Reason: "existing max value 200 exceeds width ceiling 127", ErrorCode: 1264} + if got := protocol.ConvertToMySQLError(local); got.Code != 1264 { + t.Errorf("local rejection reached the client as %d, want 1264 (the participant's own code)", got.Code) + } + + remote := &RemotePrepareRejectedError{NodeID: 2, Reason: "same refusal on a remote participant", ErrorCode: 1264} + if got := protocol.ConvertToMySQLError(remote); got.Code != 1264 { + t.Errorf("remote rejection reached the client as %d, want 1264", got.Code) + } + + // A refusal that named no code must NOT be dressed up as one: the + // message-based classification stays in charge. + plain := &LocalPrepareError{Reason: "local prepare rejected the transaction"} + if got := protocol.ConvertToMySQLError(plain); got.Code == 1264 { + t.Error("a rejection with no code was given one") + } +} diff --git a/coordinator/write_coordinator.go b/coordinator/write_coordinator.go index ac1168c8..25765027 100644 --- a/coordinator/write_coordinator.go +++ b/coordinator/write_coordinator.go @@ -107,6 +107,15 @@ type ReplicationResponse struct { // DDL SQLite cannot apply). Timeouts and storage failures leave it false so they // stay retryable missing ACKs rather than a final verdict. Rejected bool + // AutoIDStoredBase is this participant's own committed base for the table a + // rejected AUTO_INCREMENT range claim named, so the claimant can retry above + // it rather than spin. Zero on every response that is not such a rejection. + AutoIDStoredBase uint64 + // ErrorCode is the MySQL server error code the rejecting participant chose + // for this refusal, or 0 when it named none. Error carries the message; + // this carries the code, because a typed error cannot cross either the + // local or the remote participant boundary. + ErrorCode uint16 } // NewWriteCoordinator creates a new write coordinator for full database replication @@ -197,7 +206,14 @@ func (wc *WriteCoordinator) cleanupStagedPayloads(txnID uint64) { func (wc *WriteCoordinator) validateStatements(txn *Transaction) error { for i, stmt := range txn.Statements { isDML := isDMLStatement(stmt.Type) - if isDML && len(stmt.EncodedRow) == 0 { + // An AUTO_INCREMENT range claim rides an existing DML StatementType but + // carries no row image by design: its payload is AutoIDClaimPayload, + // applied from the PREPARE-time intent at COMMIT, never from EncodedRow + // (protocol/transaction.go). Exempting it here is what lets an old + // binary's own DML gate reject the same wire statement when it does not + // understand the claim flag, while this coordinator's own validation of + // its own claim does not. + if isDML && len(stmt.EncodedRow) == 0 && !stmt.AutoIDClaim { return fmt.Errorf("DML statement missing encoded CDC row (stmt %d)", i) } if isDML && stmt.TableName == "" { @@ -227,6 +243,12 @@ func (wc *WriteCoordinator) buildPrepareRequest(txn *Transaction) *ReplicationRe Operation: stmt.Operation, EncodedRow: stmt.EncodedRow, EncodedCodec: stmt.EncodedCodec, + // The claim flag AND its payload must reach PREPARE: this is where + // prepareAutoIncClaim evaluates the payload against the participant's + // own stored base and schema, before either the empty-IntentKey gate + // or the ordinary DML row-image check would otherwise apply to it. + AutoIDClaim: stmt.AutoIDClaim, + AutoIDClaimPayload: stmt.AutoIDClaimPayload, } if !isDML { prepareStmts[i].SQL = stmt.SQL @@ -279,6 +301,13 @@ func (wc *WriteCoordinator) buildCommitRequest(txn *Transaction) *ReplicationReq TableName: stmt.TableName, Database: stmt.Database, IntentKey: stmt.IntentKey, + // The AUTO_INCREMENT claim FLAG is decision metadata and must reach + // COMMIT: it is what tells a participant to apply the claim it + // prepared. The PAYLOAD beside it is deliberately not copied. A + // participant reads the claim's values from the intent it wrote at + // PREPARE, when it checked them against its own schema and its own + // stored base, so nothing it writes comes off the COMMIT wire. + AutoIDClaim: stmt.AutoIDClaim, } if !isDMLStatement(stmt.Type) { commitStmts[i].SQL = stmt.SQL @@ -343,7 +372,7 @@ func (wc *WriteCoordinator) runPreparePhase(ctx context.Context, txn *Transactio // Execute prepare phase on all nodes (including self) - single attempt, no retry // All nodes now participate uniformly - no skipLocalReplication - prepResponses, remoteRejection, prepErr := wc.executePreparePhase(ctx, txn, prepReq, otherNodes, false) + prepResponses, remoteRejection, declined, prepErr := wc.executePreparePhase(ctx, txn, prepReq, otherNodes, false) telemetry.TwoPhaseQuorumAcks.With("prepare").Observe(float64(len(prepResponses))) // A local rejection (invalid DDL, missing table, ...) is final - surface the @@ -404,6 +433,7 @@ func (wc *WriteCoordinator) runPreparePhase(ctx context.Context, txn *Transactio return nil, &QuorumNotAchievedError{ Phase: "prepare", AcksReceived: totalAcks, + Declined: declined, QuorumRequired: cluster.RequiredQuorum, TotalMembership: cluster.TotalMembership, AliveNodes: len(cluster.AliveNodes), @@ -704,10 +734,11 @@ type response struct { // executePreparePhase broadcasts prepare requests to all nodes and collects responses. // Returns successful responses, the first remote (non-local) explicit rejection seen (if // any - the caller decides whether it matters, since a remote rejection alone must not -// veto a transaction that otherwise reaches quorum), and a conflict error if any node -// reports a conflict. +// veto a transaction that otherwise reaches quorum), the number of participants that +// answered but declined without a verdict (for example a lost race for a row lock), +// and a conflict error if any node reports a conflict. func (wc *WriteCoordinator) executePreparePhase(ctx context.Context, txn *Transaction, prepReq *ReplicationRequest, - otherNodes []uint64, skipLocalReplication bool) (map[uint64]*ReplicationResponse, *RemotePrepareRejectedError, error) { + otherNodes []uint64, skipLocalReplication bool) (map[uint64]*ReplicationResponse, *RemotePrepareRejectedError, int, error) { // Calculate total nodes to contact totalNodes := len(otherNodes) @@ -718,7 +749,7 @@ func (wc *WriteCoordinator) executePreparePhase(ctx context.Context, txn *Transa if totalNodes == 0 { // No nodes to contact - this is OK if local execution was already done // Return empty map, caller will add self if skipLocalReplication was true - return make(map[uint64]*ReplicationResponse), nil, nil + return make(map[uint64]*ReplicationResponse), nil, 0, nil } // Channel for collecting responses @@ -759,8 +790,16 @@ func (wc *WriteCoordinator) executePreparePhase(ctx context.Context, txn *Transa // Collect responses prepResponses := make(map[uint64]*ReplicationResponse) var conflictErr error - var localErr error + var localErr *LocalPrepareError var remoteRejection *RemotePrepareRejectedError + // maxAutoIDStoredBase is the highest base any participant reported this + // round, tracked over every response regardless of outcome (zero on + // anything that is not an AUTO_INCREMENT claim rejection). A claimant + // retries above this, not above the first rejection it happens to see, so + // two participants rejecting with different bases converge to the + // participant that is furthest ahead. + var maxAutoIDStoredBase uint64 + declined := 0 for i := 0; i < totalNodes; i++ { select { @@ -776,6 +815,9 @@ func (wc *WriteCoordinator) executePreparePhase(ctx context.Context, txn *Transa if r.resp == nil { continue } + if r.resp.AutoIDStoredBase > maxAutoIDStoredBase { + maxAutoIDStoredBase = r.resp.AutoIDStoredBase + } if r.resp.ConflictDetected { // Any conflict -> return error, client will retry conflictErr = fmt.Errorf("conflict on node %d: %s", r.nodeID, r.resp.ConflictDetails) @@ -797,13 +839,16 @@ func (wc *WriteCoordinator) executePreparePhase(ctx context.Context, txn *Transa // Only an explicit rejection is a final answer. Anything else - a // timeout, a storage error - must fall through to the quorum check so // it stays retryable, exactly as before DDL was validated here. + if !r.resp.Rejected { + declined++ + } if r.resp.Rejected { reason := r.resp.Error if r.nodeID == wc.nodeID { if reason == "" { reason = "local prepare rejected the transaction" } - localErr = &LocalPrepareError{Reason: reason} + localErr = &LocalPrepareError{Reason: reason, ErrorCode: r.resp.ErrorCode} } else if remoteRejection == nil { // A remote rejection does not by itself abort the transaction - // it has no veto over quorum. It is only surfaced by the caller @@ -811,7 +856,8 @@ func (wc *WriteCoordinator) executePreparePhase(ctx context.Context, txn *Transa if reason == "" { reason = "remote prepare rejected the transaction" } - remoteRejection = &RemotePrepareRejectedError{NodeID: r.nodeID, Reason: reason} + remoteRejection = &RemotePrepareRejectedError{NodeID: r.nodeID, Reason: reason, + ErrorCode: r.resp.ErrorCode} } } } @@ -822,18 +868,28 @@ func (wc *WriteCoordinator) executePreparePhase(ctx context.Context, txn *Transa } } + // Attach the round's maximum reported base to whichever rejection is on + // record. Both are nil on every transaction that is not a claim, so this + // changes nothing else. + if localErr != nil { + localErr.AutoIDStoredBase = maxAutoIDStoredBase + } + if remoteRejection != nil { + remoteRejection.AutoIDStoredBase = maxAutoIDStoredBase + } + // An explicit local rejection is deterministic, so it takes precedence over a // peer conflict: retrying the transaction cannot make it succeed. Transport // failures and missing responses are not rejections - they fall through to the // quorum check. if localErr != nil { - return prepResponses, remoteRejection, localErr + return prepResponses, remoteRejection, declined, localErr } // If any conflict was detected, return the error if conflictErr != nil { - return prepResponses, remoteRejection, conflictErr + return prepResponses, remoteRejection, declined, conflictErr } - return prepResponses, remoteRejection, nil + return prepResponses, remoteRejection, declined, nil } diff --git a/coordinator/write_coordinator_benchmarks_test.go b/coordinator/write_coordinator_benchmarks_test.go index a0ed8303..81d4fedc 100644 --- a/coordinator/write_coordinator_benchmarks_test.go +++ b/coordinator/write_coordinator_benchmarks_test.go @@ -66,7 +66,7 @@ func BenchmarkExecutePreparePhase(b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - _, _, _ = wc.executePreparePhase(ctx, txn, req, otherNodes, false) + _, _, _, _ = wc.executePreparePhase(ctx, txn, req, otherNodes, false) } } diff --git a/coordinator/write_coordinator_executeprepare_test.go b/coordinator/write_coordinator_executeprepare_test.go index 01e24af8..27930098 100644 --- a/coordinator/write_coordinator_executeprepare_test.go +++ b/coordinator/write_coordinator_executeprepare_test.go @@ -47,7 +47,7 @@ func TestExecutePreparePhase_BroadcastToAllNodes(t *testing.T) { ctx, cancel := WithTimeout(200) defer cancel() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) AssertNoError(t, err) AssertResponseMapSize(t, responses, 5) // coordinator + 4 others @@ -90,7 +90,7 @@ func TestExecutePreparePhase_SkipLocalReplication_False(t *testing.T) { ctx, cancel := WithTimeout(200) defer cancel() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) AssertNoError(t, err) AssertResponseMapSize(t, responses, 3) // coordinator + 2 others @@ -137,7 +137,7 @@ func TestExecutePreparePhase_SkipLocalReplication_True(t *testing.T) { ctx, cancel := WithTimeout(200) defer cancel() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, true) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, true) AssertNoError(t, err) AssertResponseMapSize(t, responses, 3) // only other nodes @@ -187,7 +187,7 @@ func TestExecutePreparePhase_ConcurrentExecution(t *testing.T) { defer cancel() start := time.Now() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) elapsed := time.Since(start) AssertNoError(t, err) @@ -233,7 +233,7 @@ func TestExecutePreparePhase_ResponseTimeout(t *testing.T) { ctx, cancel := WithTimeout(300) defer cancel() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) AssertNoError(t, err) @@ -284,7 +284,7 @@ func TestExecutePreparePhase_MixedResponses(t *testing.T) { ctx, cancel := WithTimeout(200) defer cancel() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) // Conflict should be returned as error if err == nil { @@ -332,7 +332,7 @@ func TestExecutePreparePhase_AllNodesTimeout(t *testing.T) { ctx, cancel := WithTimeout(300) defer cancel() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) AssertNoError(t, err) @@ -373,7 +373,7 @@ func TestExecutePreparePhase_ContextCancelled(t *testing.T) { CancelAfter(ctx, cancel, 20*time.Millisecond) start := time.Now() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) elapsed := time.Since(start) // Should return quickly after context cancellation @@ -419,7 +419,7 @@ func TestExecutePreparePhase_NilResponse(t *testing.T) { ctx, cancel := WithTimeout(200) defer cancel() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) AssertNoError(t, err) @@ -461,7 +461,7 @@ func TestExecutePreparePhase_ZeroTotalNodes(t *testing.T) { ctx, cancel := WithTimeout(200) defer cancel() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, true) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, true) AssertNoError(t, err) AssertResponseMapSize(t, responses, 0) @@ -506,7 +506,7 @@ func TestExecutePreparePhase_ChannelBufferOverflow(t *testing.T) { defer cancel() start := time.Now() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) elapsed := time.Since(start) AssertNoError(t, err) @@ -552,7 +552,7 @@ func TestExecutePreparePhase_GoroutineLeakPrevention(t *testing.T) { beforeGoroutines := CountGoroutines() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) AssertNoError(t, err) @@ -601,7 +601,7 @@ func TestExecutePreparePhase_ConflictStopsCollection(t *testing.T) { ctx, cancel := WithTimeout(200) defer cancel() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) // Should return conflict error if err == nil { @@ -649,7 +649,7 @@ func TestExecutePreparePhase_ErrorResponsesNotAdded(t *testing.T) { ctx, cancel := WithTimeout(200) defer cancel() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) AssertNoError(t, err) @@ -705,7 +705,7 @@ func TestExecutePreparePhase_CallsAreDeadlineBounded(t *testing.T) { // Parent context has no deadline of its own - the bound must come from wc.timeout start := time.Now() - responses, _, err := wc.executePreparePhase(context.Background(), txn, req, []uint64{2, 3}, false) + responses, _, _, err := wc.executePreparePhase(context.Background(), txn, req, []uint64{2, 3}, false) AssertNoError(t, err) AssertResponseMapSize(t, responses, 3) @@ -747,7 +747,7 @@ func TestExecutePreparePhase_DDLCallsUseDDLValidationTimeout(t *testing.T) { req := &ReplicationRequest{TxnID: txn.ID, NodeID: wc.nodeID, Phase: PhasePrep, Database: txn.Database} start := time.Now() - responses, _, err := wc.executePreparePhase(context.Background(), txn, req, []uint64{2, 3}, false) + responses, _, _, err := wc.executePreparePhase(context.Background(), txn, req, []uint64{2, 3}, false) AssertNoError(t, err) AssertResponseMapSize(t, responses, 3) @@ -796,7 +796,7 @@ func TestExecutePreparePhase_DMLCallsKeepRegularWriteTimeout(t *testing.T) { req := &ReplicationRequest{TxnID: txn.ID, NodeID: wc.nodeID, Phase: PhasePrep, Database: txn.Database} start := time.Now() - responses, _, err := wc.executePreparePhase(context.Background(), txn, req, []uint64{2, 3}, false) + responses, _, _, err := wc.executePreparePhase(context.Background(), txn, req, []uint64{2, 3}, false) AssertNoError(t, err) AssertResponseMapSize(t, responses, 3) @@ -843,7 +843,7 @@ func TestExecutePreparePhase_DDLTransactionSurvivesDelayBeyondWriteTimeout(t *te ctx, cancel := WithTimeout(3000) defer cancel() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) AssertNoError(t, err) AssertResponseSuccess(t, responses, 2) @@ -877,7 +877,7 @@ func TestExecutePreparePhase_DMLTransactionTimesOutAtWriteTimeout(t *testing.T) ctx, cancel := WithTimeout(3000) defer cancel() - responses, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, _, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) AssertNoError(t, err) AssertNodeNotInResponses(t, responses, 2) diff --git a/coordinator/write_coordinator_remote_rejection_test.go b/coordinator/write_coordinator_remote_rejection_test.go index eea7d8c0..bb482e67 100644 --- a/coordinator/write_coordinator_remote_rejection_test.go +++ b/coordinator/write_coordinator_remote_rejection_test.go @@ -49,7 +49,7 @@ func TestExecutePreparePhase_RemoteRejectionTracked(t *testing.T) { ctx, cancel := WithTimeout(200) defer cancel() - responses, remoteRejection, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) + responses, remoteRejection, _, err := wc.executePreparePhase(ctx, txn, req, otherNodes, false) AssertNoError(t, err) if remoteRejection == nil { diff --git a/db/autoinc_claim.go b/db/autoinc_claim.go new file mode 100644 index 00000000..cabe8f1d --- /dev/null +++ b/db/autoinc_claim.go @@ -0,0 +1,363 @@ +package db + +import ( + "database/sql" + "errors" + "fmt" + "math" + "os" + "strings" + "time" + + "github.com/maxpert/marmot/common" + "github.com/maxpert/marmot/protocol" + "github.com/maxpert/marmot/protocol/query/transform/intmarker" +) + +// AutoIncClaimTable is the hidden table holding one row per narrow +// AUTO_INCREMENT table across every user database: the current allocation +// base, keyed by (database, table). +// +// The table lives in the SYSTEM database (db.SystemDatabaseName), not in the +// user database the claim is about. A claim applied inside a pinned session +// on the user database (db/db_integration.go BeginPinnedSession -> +// db/preupdate_hook.go conn.BeginTx BEGIN IMMEDIATE) would try to take that +// same database's single SQLite writer a second time and block on its own +// BEGIN; the system database is a separate SQLite file with its own writer, +// so a claim never contends with a user database's pinned session. Two +// properties this table used to get "for free" by living inside each user +// database now come from that placement instead: DatabaseManager.ListDatabases +// never returns SystemDatabaseName, so the table is never client-visible +// through SHOW DATABASES-style listings, and DatabaseManager.TakeSnapshot / +// TakeSnapshotToDir checkpoint and ship the system database file explicitly +// on every snapshot, so the base travels with a restore without needing to be +// named per user database. The base deliberately does NOT live in Pebble: a +// restored node has no Pebble state, so a full-cluster restore would make +// every voter a yes-man at once. +// +// The "__marmot__" prefix still buys two properties unrelated to placement: +// the preupdate hook only ever captures CDC for tables present in the schema +// cache, and this table is created by raw DDL outside that cache, so it never +// enters CDC; and the prefix backs the parser guard +// (protocol/query/rules/claim_table_guard.go), which refuses client SQL that +// would write to, or change the schema of, any table under this prefix. +// +// Note for anyone tempted to use the name as a filter: SQLite's LIKE treats "_" +// as a single-character wildcard, so '__marmot__%' is broader than it looks and +// is not an exact-name test. +// +// The literal itself lives in common.AutoIncClaimTableName - the one place in +// the repo that spells it out - so protocol/query/rules can refuse client SQL +// naming this table without db importing protocol (which would cycle back +// through protocol -> protocol/query/rules -> db). +const AutoIncClaimTable = common.AutoIncClaimTableName + +// autoIncClaimDDL creates the table. WITHOUT ROWID because the row is +// addressed only by (db, tbl) and a rowid would be dead weight. The +// composite primary key is what lets one system-database table serve every +// user database at once. +var autoIncClaimDDL = `CREATE TABLE IF NOT EXISTS ` + AutoIncClaimTable + ` ( + db TEXT NOT NULL, + tbl TEXT NOT NULL, + base INTEGER NOT NULL, + owner INTEGER NOT NULL, + granted_at INTEGER NOT NULL, + PRIMARY KEY (db, tbl) +) WITHOUT ROWID` + +// AutoIncClaim is the payload a claim statement carries in its intent's +// DataSnapshot. It is defined in protocol (protocol/autoinc_claim.go), which +// coordinator also depends on, because db already imports coordinator (for +// coordinator.Replicator) and coordinator importing db back would cycle. This +// alias lets db's own claim-handling code keep using the unqualified name. +type AutoIncClaim = protocol.AutoIncClaim + +// ErrAutoIncBaseAbsent reports that a table has no claim row. +// +// It is an error and never a zero, and the distinction is the protocol's whole +// safety argument. The natural implementation is "absent means 0 means yes", +// and an engineer will write that unless told not to: a cache that defaults to +// 0 and accepts is precisely the thing that cannot cast the rejection that +// would repair it. +var ErrAutoIncBaseAbsent = errors.New("no auto-increment claim row for table") + +// ErrAutoIncClaimNotApplicable reports a COMMIT whose claim this node cannot +// apply: the claim intent written at PREPARE is gone, or the stored base has +// moved past the claim's newBase since PREPARE. Either way this node did not +// hold the claim continuously from its vote to its commit, and ACKing would +// let it count toward a quorum for a range it may also have granted to +// another claimant. +var ErrAutoIncClaimNotApplicable = errors.New("auto-increment claim cannot be applied") + +// AutoIncClaimStore is the node's AUTO_INCREMENT base store: one row per +// (database, table). The receiver is ALWAYS the SYSTEM database, never a user +// database, and every method therefore names its database explicitly. +type AutoIncClaimStore struct { + sys *ReplicatedDatabase +} + +// NewAutoIncClaimStore wraps the system database as a claim store. sys must +// be the system database (db.SystemDatabaseName), never a user database. +func NewAutoIncClaimStore(sys *ReplicatedDatabase) *AutoIncClaimStore { + return &AutoIncClaimStore{sys: sys} +} + +// ReadBase returns a table's committed base. +// +// It reads through the READ pool, not the write handle: writeDB is capped at a +// single connection, so reading a base through it would put every claim behind +// the one SQLite writer. +// +// An absent row returns ErrAutoIncBaseAbsent. Callers must reject on it, and +// on every other error too. The claim table is created with the system +// database (DatabaseManager's system-database initialisation), so a missing +// table is not an expected state and surfaces as the read error it is. +func (s *AutoIncClaimStore) ReadBase(database, table string) (uint64, error) { + var base int64 + err := s.sys.GetReadDB().QueryRow( + "SELECT base FROM "+AutoIncClaimTable+" WHERE db = ? AND tbl = ?", database, table).Scan(&base) + switch { + case err == sql.ErrNoRows: + return 0, ErrAutoIncBaseAbsent + case err != nil: + return 0, fmt.Errorf("read auto-increment base for %s.%s: %w", database, table, err) + } + if base < 0 { + return 0, fmt.Errorf("auto-increment base for %s.%s is negative (%d)", database, table, base) + } + return uint64(base), nil +} + +// Seed creates or RAISES a table's base at DDL time. Never lowers. +// +// Tagging an existing table holding ids 1..1000 with an absent row would leave +// the base at 0, so the first range would be [0,R) and the allocator would +// mint 1, 2, 3 over live rows: a UNIQUE failure on a node that holds them and +// a silent overwrite on one that does not. Each node seeds from its own +// MAX(id) when it applies the replicated DDL, so seeds can differ; they +// converge upward on the first claim, because the PREPARE condition rejects +// any proposal at or below a participant's own stored base and returns that +// base for the retry. +func (s *AutoIncClaimStore) Seed(database, table string, floor, owner uint64) error { + writeDB := s.sys.GetWriteDB() + if _, err := writeDB.Exec(autoIncClaimDDL); err != nil { + return fmt.Errorf("create %s: %w", AutoIncClaimTable, err) + } + _, err := writeDB.Exec( + "INSERT INTO "+AutoIncClaimTable+" (db, tbl, base, owner, granted_at) VALUES (?, ?, ?, ?, ?) "+ + "ON CONFLICT(db, tbl) DO UPDATE SET base = MAX(base, excluded.base), owner = excluded.owner, granted_at = excluded.granted_at", + database, table, int64(floor), int64(owner), time.Now().UnixNano()) + if err != nil { + return fmt.Errorf("seed auto-increment base for %s.%s: %w", database, table, err) + } + return nil +} + +// RaiseAutoIncBasesFrom raises every claim base in the system database file at +// incomingPath to at least the base the system database file at localPath +// holds for the same (database, table), and copies in any row only localPath +// has. It is run on a peer's system database before a restore installs it in +// place of this node's own. +// +// A restore must never lower a base this node committed. The peer may have +// missed a claim this node was in the majority for; installing the peer's +// lower base would let this node accept that range again, and majority +// intersection - the protocol's whole safety argument - would be gone. A +// missing localPath (a node with no prior state) leaves incomingPath as it is. +func RaiseAutoIncBasesFrom(incomingPath, localPath string) (err error) { + if _, statErr := os.Stat(localPath); errors.Is(statErr, os.ErrNotExist) { + return nil + } else if statErr != nil { + return fmt.Errorf("stat local system database: %w", statErr) + } + + // The merged bases must survive a power loss once the restorer moves this + // file into place, as every other system-database commit does + // (WithDurableCommits): the durable driver, with synchronous=FULL for the + // checkpoint that lands them in the main file. + conn, err := sql.Open(SQLiteDurableDriverName, incomingPath+"?_sync=FULL") + if err != nil { + return fmt.Errorf("open incoming system database: %w", err) + } + defer func() { + if closeErr := conn.Close(); err == nil && closeErr != nil { + err = fmt.Errorf("close incoming system database: %w", closeErr) + } + }() + // ATTACH is per connection, so every statement below must run on one. + conn.SetMaxOpenConns(1) + + // A peer running a binary older than the claim table ships a system + // database without it; the local rows still have to land. + if _, err := conn.Exec(autoIncClaimDDL); err != nil { + return fmt.Errorf("create %s in incoming system database: %w", AutoIncClaimTable, err) + } + if _, err := conn.Exec("ATTACH DATABASE ? AS local", localPath); err != nil { + return fmt.Errorf("attach local system database: %w", err) + } + var localHasTable int + if err := conn.QueryRow("SELECT COUNT(*) FROM local.sqlite_master WHERE type = 'table' AND name = ?", + AutoIncClaimTable).Scan(&localHasTable); err != nil { + return fmt.Errorf("inspect local system database: %w", err) + } + if localHasTable > 0 { + // "WHERE true" disambiguates the upsert's ON CONFLICT from a join + // constraint, as SQLite requires for INSERT ... SELECT. + if _, err := conn.Exec("INSERT INTO main." + AutoIncClaimTable + " (db, tbl, base, owner, granted_at) " + + "SELECT db, tbl, base, owner, granted_at FROM local." + AutoIncClaimTable + " WHERE true " + + "ON CONFLICT(db, tbl) DO UPDATE SET base = excluded.base, owner = excluded.owner, granted_at = excluded.granted_at " + + "WHERE excluded.base > base"); err != nil { + return fmt.Errorf("raise incoming auto-increment bases: %w", err) + } + } + if _, err := conn.Exec("DETACH DATABASE local"); err != nil { + return fmt.Errorf("detach local system database: %w", err) + } + // The restorer moves only the main database file into place. Closing the + // last connection checkpoints the write-ahead log into it; a failed close + // is returned by the deferred Close above. + return nil +} + +// AutoIncWidthMax returns the largest id the table's AUTO_INCREMENT column can +// hold, derived from THIS node's own schema. +// +// It is never taken from a claimant. The width marker rides in the replicated +// DDL text, so every node derives the same ceiling from the same source; +// trusting the proposer would let a node whose schema is mid-DDL rubber-stamp a +// range its column cannot hold. +// +// A table with no width marker has no narrow ceiling, and a claim against it is +// a caller error rather than a width question: the narrow allocator is only +// ever engaged for a marked column. +// +// This stays a method on the USER database: widthMax is derived from the user +// database's own schema, a rule the relocation of the claim store to the +// system database does not touch. +func (mdb *ReplicatedDatabase) AutoIncWidthMax(table string) (uint64, error) { + schema, err := mdb.GetCachedTableSchema(table) + if err != nil { + return 0, fmt.Errorf("no cached schema for %s: %w", table, err) + } + autoIncCol := schema.GetAutoIncrementCol() + if autoIncCol == "" { + return 0, fmt.Errorf("table %s has no auto-increment column", table) + } + for _, col := range schema.FullColumns { + if !strings.EqualFold(col.Name, autoIncCol) { + continue + } + attrs := intmarker.Attributes{Bits: col.DeclaredWidth, Unsigned: col.Unsigned} + if !attrs.Marked() { + return 0, fmt.Errorf("table %s auto-increment column %s carries no width marker", table, autoIncCol) + } + return attrs.WidthMax(), nil + } + return 0, fmt.Errorf("table %s auto-increment column %s is absent from its schema", table, autoIncCol) +} + +// ApplyClaims writes the committed base for every AUTO_INCREMENT range claim +// prepared under txnID against database. It is called by the COMMIT handler, +// on the user-database path, only for a COMMIT that carries the claim flag, +// and before the transaction is committed. +// +// meta is the USER database's meta store: the transaction whose intents are +// being read is against the user database, even though the write this method +// performs lands in the system database. +// +// The stored base becomes newBase + size. The claimant owns +// newBase+1 .. newBase+size, so newBase+size is both the +// last id handed out and the prevBase the next claimant proposes. That sum is +// the quantity the protocol advances monotonically. +// +// The write is conditional on the stored base still being at or below the +// claim's newBase, where its range starts, and ApplyClaims fails with ErrAutoIncClaimNotApplicable +// when it is not, or when the transaction holds no claim intent at all. This +// is what makes a node's COMMIT ACKs safe without trusting the claim key's +// row lock to have survived from PREPARE to COMMIT: the system database has +// one writer, so the conditional writes a node applies are serialised, and +// each applies only if no range it already committed reaches past the new +// claim's newBase. The condition is on newBase rather than prevBase so that it +// holds on its own, not only because PREPARE admitted newBase == prevBase. The ranges one node ACKs are therefore pairwise disjoint, +// and two overlapping ranges cannot both gather a commit majority, because +// any two majorities share a node. The same condition keeps a base that a DDL +// seed raised between PREPARE and COMMIT from being lowered, and refuses a +// range that would sit below that raised floor. +// +// Every claim in the transaction lands in ONE SQLite transaction against the +// system database. A partially applied set would leave this node holding a +// base below a range it had already voted to give away, and the PREPARE condition +// would then let it accept that same range again. +// +// The values come only from the intents. A claim's DataSnapshot was fixed at +// PREPARE, when this node checked it against its own schema and its own stored +// base; the COMMIT message is decision metadata and carries no payload to read. +// Owner and grant time are taken from the intent record's own NodeID and TSWall +// rather than from the payload, so a claimant cannot misattribute a range to +// another node. +// +// The row is updated in place and never deleted: an absent row is a hard rejection at PREPARE, so any code that +// removed rows would create a window in which a live table re-mints ids it has +// already used. +func (s *AutoIncClaimStore) ApplyClaims(database string, txnID uint64, meta MetaStore) error { + intents, err := meta.GetIntentsByTxn(txnID) + if err != nil { + return fmt.Errorf("read intents for txn %d: %w", txnID, err) + } + + tx, err := s.sys.GetWriteDB().Begin() + if err != nil { + return fmt.Errorf("begin auto-increment claim apply: %w", err) + } + applied := 0 + for _, intent := range intents { + if intent.IntentType != IntentTypeAutoIDClaim { + continue + } + if aErr := applyClaimTx(tx, database, intent); aErr != nil { + _ = tx.Rollback() + return fmt.Errorf("txn %d: %w", txnID, aErr) + } + applied++ + } + if applied == 0 { + _ = tx.Rollback() + return fmt.Errorf("txn %d: %w: no claim intent is held for this transaction", txnID, ErrAutoIncClaimNotApplicable) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit auto-increment claim apply for txn %d: %w", txnID, err) + } + return nil +} + +// applyClaimTx applies one claim intent inside tx, conditional on the stored +// base not having passed the claim's newBase (see ApplyClaims). +func applyClaimTx(tx *sql.Tx, database string, intent *WriteIntentRecord) error { + claim, err := protocol.DecodeAutoIncClaim(intent.DataSnapshot) + if err != nil { + return err + } + // Re-checked here, not assumed from PREPARE: this arithmetic decides what + // is written, and a wrapped sum would store a base below the ids the range + // just handed out. The base column is a signed SQLite INTEGER, so the sum + // must also fit in an int64. + if claim.Size == 0 || claim.NewBase > math.MaxInt64 || claim.Size > math.MaxInt64-claim.NewBase { + return fmt.Errorf("auto-increment claim for %s has an unusable size %d at base %d", + claim.Table, claim.Size, claim.NewBase) + } + res, err := tx.Exec( + "UPDATE "+AutoIncClaimTable+" SET base = ?, owner = ?, granted_at = ? WHERE db = ? AND tbl = ? AND base <= ?", + int64(claim.NewBase+claim.Size), int64(intent.NodeID), intent.TSWall, database, claim.Table, int64(claim.NewBase)) + if err != nil { + return fmt.Errorf("write auto-increment base for %s.%s: %w", database, claim.Table, err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("write auto-increment base for %s.%s: %w", database, claim.Table, err) + } + if n != 1 { + return fmt.Errorf("%w: the stored base for %s.%s is absent or has passed the claim's newBase %d", + ErrAutoIncClaimNotApplicable, database, claim.Table, claim.NewBase) + } + return nil +} diff --git a/db/autoinc_claim_bench_test.go b/db/autoinc_claim_bench_test.go new file mode 100644 index 00000000..74689e05 --- /dev/null +++ b/db/autoinc_claim_bench_test.go @@ -0,0 +1,85 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "testing" + + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// benchClaimFixture builds one database holding a marked AUTO_INCREMENT table +// and a seeded claim row, and hands back the system-database claim store and +// the user database's meta store - the two halves a claim touches at COMMIT. +func benchClaimFixture(b *testing.B) (*AutoIncClaimStore, MetaStore, func()) { + b.Helper() + + dm, err := NewDatabaseManager(b.TempDir(), 1, hlc.NewClock(1)) + require.NoError(b, err) + require.NoError(b, dm.CreateDatabase("testdb")) + + mdb, err := dm.GetDatabase("testdb") + require.NoError(b, err) + _, err = mdb.GetWriteDB().Exec("CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + require.NoError(b, err) + require.NoError(b, mdb.ReloadSchema()) + + store := NewAutoIncClaimStore(dm.GetSystemDatabase()) + require.NoError(b, store.Seed("testdb", "users", 0, 1)) + + return store, mdb.GetMetaStore(), func() { dm.Close() } +} + +// BenchmarkReadBase measures the read a participant does on every PREPARE of a +// claim. It runs through the read pool, so it must not be behind the single +// SQLite writer. +func BenchmarkReadBase(b *testing.B) { + store, _, cleanup := benchClaimFixture(b) + defer cleanup() + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := store.ReadBase("testdb", "users"); err != nil { + b.Fatalf("ReadBase: %v", err) + } + } +} + +// BenchmarkApplyAutoIncClaims measures the write a participant does on every +// COMMIT of a claim: read the transaction's claim intents and write the new +// base into the system database. Only the apply is timed; writing the intent +// that stands in for PREPARE is setup and is excluded. +func BenchmarkApplyAutoIncClaims(b *testing.B) { + store, meta, cleanup := benchClaimFixture(b) + defer cleanup() + + const size = 64 + key := protocol.AutoIncClaimKey("testdb", "users") + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + txnID := uint64(i + 1) + // Each claim proposes the base the previous one committed, as a + // claimant with an up-to-date view does. + base := uint64(i) * size + payload, err := protocol.EncodeAutoIncClaim(AutoIncClaim{Table: "users", PrevBase: base, NewBase: base, Size: size}) + require.NoError(b, err) + require.NoError(b, meta.WriteIntent(txnID, IntentTypeAutoIDClaim, AutoIncClaimTable, key, + OpTypeInsert, "", payload, hlc.Timestamp{WallTime: int64(i + 1)}, 1)) + b.StartTimer() + + if err := store.ApplyClaims("testdb", txnID, meta); err != nil { + b.Fatalf("ApplyClaims: %v", err) + } + + b.StopTimer() + require.NoError(b, meta.DeleteIntentsByTxn(txnID)) + b.StartTimer() + } +} diff --git a/db/autoinc_claim_test.go b/db/autoinc_claim_test.go new file mode 100644 index 00000000..d4b98192 --- /dev/null +++ b/db/autoinc_claim_test.go @@ -0,0 +1,269 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "runtime" + "testing" + + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// TestClaimIntentReachesPebbleWithItsPayload is the storage-selection property +// the whole claim protocol rests on. +// +// WriteIntent short-circuits IntentTypeDML into storeDMLIntent, whose record +// omits DataSnapshot and which writes only to an in-memory map, while +// GetIntentsByTxn iterates the Pebble prefix. A claim stored on the DML branch +// would therefore never be returned and its payload would never have existed - +// so COMMIT would apply nothing, every participant's base would stay put, and +// the next claimant would compute the same range and mint the same ids. +func TestClaimIntentReachesPebbleWithItsPayload(t *testing.T) { + source := newRowidTestDatabase(t, 1) + store := source.GetMetaStore() + + payload, err := protocol.EncodeAutoIncClaim(AutoIncClaim{Table: "users", PrevBase: 100, NewBase: 100, Size: 64}) + if err != nil { + t.Fatalf("EncodeAutoIncClaim: %v", err) + } + + const txnID = uint64(9001) + key := protocol.AutoIncClaimKey("testdb", "users") + if err := store.WriteIntent(txnID, IntentTypeAutoIDClaim, AutoIncClaimTable, key, + OpTypeInsert, "", payload, hlc.Timestamp{WallTime: 1}, 1); err != nil { + t.Fatalf("WriteIntent: %v", err) + } + + // Mutation: use IntentTypeDML for the claim. GetIntentsByTxn returns + // nothing and this fires - which is exactly the silent-duplicate-ids + // failure, reduced to a test. + intents, err := store.GetIntentsByTxn(txnID) + if err != nil { + t.Fatalf("GetIntentsByTxn: %v", err) + } + if len(intents) != 1 { + t.Fatalf("GetIntentsByTxn returned %d intents, want 1", len(intents)) + } + if intents[0].IntentType != IntentTypeAutoIDClaim { + t.Errorf("intent type = %v, want AUTO_ID_CLAIM", intents[0].IntentType) + } + // The payload is the thing COMMIT reads; an intent without it is useless. + // Mutation: drop DataSnapshot from the non-DML record literal. + if len(intents[0].DataSnapshot) == 0 { + t.Fatal("claim intent carries no DataSnapshot; the COMMIT handler would apply nothing") + } + got, err := protocol.DecodeAutoIncClaim(intents[0].DataSnapshot) + if err != nil { + t.Fatalf("DecodeAutoIncClaim: %v", err) + } + if got != (AutoIncClaim{Table: "users", PrevBase: 100, NewBase: 100, Size: 64}) { + t.Errorf("payload round-tripped as %+v", got) + } +} + +// TestAutoIncBaseAbsentIsAnErrorNotZero pins the fail-closed rule. The natural +// implementation is "absent means 0 means yes", and that is precisely the thing +// that cannot cast the rejection that would repair it. +// +// The claim table now lives in the system database, keyed by (database, +// table): AutoIncClaimStore's receiver stands in for that system database, and +// ReadBase names both the database and the table it is asking about. +// +// Mutation: return (0, nil) for a missing row or a missing table. +func TestAutoIncBaseAbsentIsAnErrorNotZero(t *testing.T) { + sys := newRowidTestDatabase(t, 1) + store := NewAutoIncClaimStore(sys) + + // No claim table at all: not an expected state (the system database + // creates it), but it must still be an error and never a base. + if base, err := store.ReadBase("testdb", "users"); err == nil { + t.Errorf("with no claim table, ReadBase returned base %d and no error", base) + } + + // Seed a DIFFERENT table to bring the claim table into existence without + // creating a row for "users", so the next read exercises "table exists, + // row does not" rather than "table absent" again. + if err := store.Seed("testdb", "other", 1, 1); err != nil { + t.Fatalf("Seed: %v", err) + } + + // Table exists, row for "users" does not. + if _, err := store.ReadBase("testdb", "users"); !errors.Is(err, ErrAutoIncBaseAbsent) { + t.Errorf("with no row, ReadBase returned %v, want ErrAutoIncBaseAbsent", err) + } +} + +// TestSeedAutoIncBaseOnlyRaises pins the DDL-time floor. Tagging a table that +// already holds ids 1..1000 must not leave the base at 0, or the first range is +// [0,R) and the allocator mints over live rows. +func TestSeedAutoIncBaseOnlyRaises(t *testing.T) { + sys := newRowidTestDatabase(t, 1) + store := NewAutoIncClaimStore(sys) + + if err := store.Seed("testdb", "users", 1000, 1); err != nil { + t.Fatalf("Seed: %v", err) + } + base, err := store.ReadBase("testdb", "users") + if err != nil { + t.Fatalf("ReadBase: %v", err) + } + if base != 1000 { + t.Fatalf("base = %d after seeding 1000, want 1000", base) + } + + // A later seed with a LOWER floor must not lower the base: re-applying a + // replicated DDL on a node that has since committed claims would otherwise + // wind the base backwards and remint over ids already handed out. + // Mutation: use excluded.base instead of MAX(base, excluded.base). + if err := store.Seed("testdb", "users", 5, 1); err != nil { + t.Fatalf("Seed (lower): %v", err) + } + base, err = store.ReadBase("testdb", "users") + if err != nil { + t.Fatalf("ReadBase: %v", err) + } + if base != 1000 { + t.Errorf("base = %d after seeding a lower floor, want it to stay 1000", base) + } + + // A higher floor does raise it. + if err := store.Seed("testdb", "users", 4999, 1); err != nil { + t.Fatalf("Seed (higher): %v", err) + } + base, err = store.ReadBase("testdb", "users") + if err != nil { + t.Fatalf("ReadBase: %v", err) + } + if base != 4999 { + t.Errorf("base = %d after seeding 4999, want 4999", base) + } +} + +// TestClaimTableIsCDCSkipped pins the property the __marmot__ prefix buys: a +// write to the claim table must produce no CDC entry, or the claim would enter +// the blind-apply path and be applied twice by two different mechanisms. This +// holds regardless of which database the claim table sits in, so it is +// exercised directly against source's own SQLite rather than through the +// store (source stands in for the system database here). +// +// Mutation: add an exception at the preupdate hook's prefix skip so the table +// is captured. The hook then looks the table up in the schema cache, which +// never holds it, and the capture fails with "schema cache miss" at +// captureEntries' require.NoError; either that or a non-zero entry count +// fails the test. +func TestClaimTableIsCDCSkipped(t *testing.T) { + source := newRowidTestDatabase(t, 1) + if _, err := source.GetWriteDB().Exec(autoIncClaimDDL); err != nil { + t.Fatalf("create %s: %v", AutoIncClaimTable, err) + } + if err := source.ReloadSchema(); err != nil { + t.Fatalf("ReloadSchema: %v", err) + } + + entries := captureEntries(t, source, 7001, + "INSERT INTO "+AutoIncClaimTable+" (db, tbl, base, owner, granted_at) VALUES ('testdb', 'users', 64, 1, 0)") + if len(entries) != 0 { + t.Errorf("writing the claim table produced %d CDC entries, want 0", len(entries)) + } +} + +// TestDropDatabaseRemovesItsClaimRowsAndLeavesOthersIntact pins the DROP +// DATABASE side of the relocation: removing a database must take its claim +// rows with it (db/database_manager.go DropDatabase), but must not touch +// another database's rows in the same shared system-database table. This is +// the opposite of DROP TABLE (TestDroppedTableKeepsItsClaimRow, +// db/autoinc_commit_test.go), which deliberately leaves the row behind. +// +// Mutation: scope the DELETE without a WHERE db = ? clause, or drop it +// entirely. Either the row survives its own database's removal, or a +// surviving database's row is deleted too, and the assertions below fire. +func TestDropDatabaseRemovesItsClaimRowsAndLeavesOthersIntact(t *testing.T) { + _, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + + require.NoError(t, dm.CreateDatabase("dropme")) + require.NoError(t, dm.CreateDatabase("keepme")) + + store := NewAutoIncClaimStore(dm.GetSystemDatabase()) + require.NoError(t, store.Seed("dropme", "users", 1000, 1)) + require.NoError(t, store.Seed("keepme", "users", 2000, 1)) + + require.NoError(t, dm.DropDatabase("dropme")) + + if _, err := store.ReadBase("dropme", "users"); !errors.Is(err, ErrAutoIncBaseAbsent) { + t.Errorf("dropme.users claim row survived DROP DATABASE: ReadBase returned %v, want ErrAutoIncBaseAbsent", err) + } + + base, err := store.ReadBase("keepme", "users") + require.NoError(t, err, "DROP DATABASE of one database must not disturb another's claim rows") + if base != 2000 { + t.Errorf("keepme.users base = %d after dropping a different database, want 2000", base) + } +} + +// TestSystemDatabaseCommitsAreDurable: a claim COMMIT this node ACKs counts +// toward a majority, so the base it wrote must survive an OS crash or power +// loss. Every connection the system database opens runs synchronous=FULL - +// including one that replaces a connection database/sql discarded - and on +// darwin fullfsync, without which FULL does not flush the drive's cache. User +// databases keep NORMAL. +// +// Mutation: drop _sync from the DSN (the driver then opens every connection +// at NORMAL). "a new system-database connection does not sync commits" +// fires. On darwin, dropping the durable driver's fullfsync pragma fires +// "a new system-database connection does not flush the drive cache". +func TestSystemDatabaseCommitsAreDurable(t *testing.T) { + dm, err := NewDatabaseManager(t.TempDir(), 1, hlc.NewClock(1)) + require.NoError(t, err) + defer dm.Close() + require.NoError(t, dm.CreateDatabase("testdb")) + user, err := dm.GetDatabase("testdb") + require.NoError(t, err) + sys := dm.GetSystemDatabase() + ctx := context.Background() + + const normal, full = 1, 2 + wantFullfsync := 0 + if runtime.GOOS == "darwin" { + wantFullfsync = 1 + } + assertDurable := func(conn *sql.Conn, which string) { + t.Helper() + var sync, fullfsync int + require.NoError(t, conn.QueryRowContext(ctx, "PRAGMA synchronous").Scan(&sync)) + require.Equal(t, full, sync, "a new system-database connection does not sync commits (%s)", which) + require.NoError(t, conn.QueryRowContext(ctx, "PRAGMA fullfsync").Scan(&fullfsync)) + require.Equal(t, wantFullfsync, fullfsync, "a new system-database connection does not flush the drive cache (%s)", which) + } + + // Replace the single write connection the way database/sql does after a + // driver.ErrBadConn, then check the connection that replaces it. + old, err := sys.GetWriteDB().Conn(ctx) + require.NoError(t, err) + require.ErrorIs(t, old.Raw(func(any) error { return driver.ErrBadConn }), driver.ErrBadConn) + require.ErrorIs(t, old.Close(), sql.ErrConnDone, "database/sql kept a connection that reported ErrBadConn") + replacement, err := sys.GetWriteDB().Conn(ctx) + require.NoError(t, err) + assertDurable(replacement, "replaced write connection") + require.NoError(t, replacement.Close()) + + // Holding one read connection forces the pool to open another. + first, err := sys.GetReadDB().Conn(ctx) + require.NoError(t, err) + second, err := sys.GetReadDB().Conn(ctx) + require.NoError(t, err) + assertDurable(second, "second read connection") + require.NoError(t, second.Close()) + require.NoError(t, first.Close()) + + var sync int + require.NoError(t, user.GetWriteDB().QueryRow("PRAGMA synchronous").Scan(&sync)) + require.Equal(t, normal, sync, "user databases keep NORMAL") +} diff --git a/db/autoinc_commit_test.go b/db/autoinc_commit_test.go new file mode 100644 index 00000000..13a04f27 --- /dev/null +++ b/db/autoinc_commit_test.go @@ -0,0 +1,454 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "context" + "strings" + "testing" + + "github.com/maxpert/marmot/cfg" + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// commitStatement builds what a claim looks like on the COMMIT wire: the flag, +// the intent key, and no payload. buildCommitRequest strips the payload on +// purpose, so a COMMIT handler that reads its values from the wire instead of +// from the intent it prepared will fail against this shape rather than in +// production. +func commitStatement(database, table string) protocol.Statement { + return protocol.Statement{ + Type: protocol.StatementInsert, + Database: database, + TableName: table, + IntentKey: []byte(protocol.AutoIncClaimKey(database, table)), + AutoIDClaim: true, + } +} + +// TestCommitAppliesClaimAndAdvancesBase is the COMMIT half of the protocol: the +// row only materialises here, so the whole uniqueness argument +// rests on this write happening and on it storing newBase+size. +// +// Mutation: store claim.NewBase instead of claim.NewBase+claim.Size. The base +// never advances, the next claimant proposes the same range, and the assertion +// on 1064 fires. +func TestCommitAppliesClaimAndAdvancesBase(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 1000, 1) + + const txnID = 7100 + prep := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, + NodeID: 7, + StartTS: hlc.Timestamp{WallTime: 4242}, + Database: "testdb", + Statements: []protocol.Statement{claimStatement(t, "testdb", "users", 1000, 1000, 64)}, + }) + require.True(t, prep.Success, "PREPARE refused a well-formed claim: %s", prep.Error) + + res := engine.Commit(context.Background(), &CommitRequest{ + TxnID: txnID, + Database: "testdb", + Statements: []protocol.Statement{commitStatement("testdb", "users")}, + }) + require.True(t, res.Success, "COMMIT failed: %s", res.Error) + + base := readClaimBase(t, dm, "testdb", "users") + if base != 1064 { + t.Fatalf("stored base is %d, want 1064: the claimant owns 1001..1064, so the next prevBase is 1064", base) + } + + // Owner and grant time come from the intent record, never from the payload, + // so a claimant cannot attribute a range to a node that did not ask for it. + // Mutation: write re.nodeID (1 here) as the owner instead of intent.NodeID. + // Keyed by BOTH database and table: the system database's claim table now + // serves every user database, so a query on table name alone would match + // another database's row of the same name. + var owner, grantedAt int64 + require.NoError(t, dm.GetSystemDatabase().GetReadDB().QueryRow( + "SELECT owner, granted_at FROM "+AutoIncClaimTable+" WHERE db = ? AND tbl = ?", "testdb", "users"). + Scan(&owner, &grantedAt)) + if owner != 7 { + t.Errorf("range owner recorded as %d, want the claimant 7", owner) + } + if grantedAt != 4242 { + t.Errorf("granted_at recorded as %d, want the claim's own timestamp 4242", grantedAt) + } +} + +// TestCommitWithoutClaimWritesNothing pins the gate. An ordinary transaction +// carries no claim flag, so the commit path must not touch the claim table at +// all -- this is what keeps a Pebble intent scan off every write. +// +// Mutation: drop the statementsCarryAutoIDClaim guard and always call +// AutoIncClaimStore.ApplyClaims. The call finds no claim intent and writes nothing, so +// this test still passes; it is the benchmark, not this test, that holds that +// line. What this test does hold is that a claim-free commit still succeeds. +func TestCommitWithoutClaimWritesNothing(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 1000, 1) + + const txnID = 7200 + prep := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, + NodeID: 7, + StartTS: hlc.Timestamp{WallTime: 10}, + Database: "testdb", + Statements: []protocol.Statement{{ + Type: protocol.StatementDDL, Database: "testdb", TableName: "other", + SQL: "CREATE TABLE other (id INTEGER PRIMARY KEY)", + }}, + }) + require.True(t, prep.Success, "PREPARE failed: %s", prep.Error) + + res := engine.Commit(context.Background(), &CommitRequest{ + TxnID: txnID, + Database: "testdb", + Statements: []protocol.Statement{{ + Type: protocol.StatementDDL, Database: "testdb", TableName: "other", + SQL: "CREATE TABLE other (id INTEGER PRIMARY KEY)", + }}, + }) + require.True(t, res.Success, "COMMIT failed: %s", res.Error) + + base := readClaimBase(t, dm, "testdb", "users") + require.Equal(t, uint64(1000), base, "a claim-free commit moved the base") +} + +// TestCommitRefusesWhenClaimCannotBeApplied is the claim's first invariant: +// a participant must never ACK COMMIT unless the claim row is durably in its +// own SQLite. It runs under both batch-commit settings. A claim-only +// transaction has no CDC entries, so CommitTransaction would take its +// statement branch under either setting, and the refusal happens before +// CommitTransaction is reached; the batch committer's blocking wait is pinned +// by TestParticipantWithholdsAckWhenTransactionCommitFailsAfterClaimApplies. +// +// The failure is injected with a BEFORE UPDATE trigger on the claim table, so +// the seeded row PREPARE reads is intact and only the apply's upsert aborts. +// +// Mutation: apply the claim AFTER txnMgr.CommitTransaction, or swallow the +// apply error and return Success: true. Either way the node reports a commit it +// did not durably make and both assertions below fire. +func TestCommitRefusesWhenClaimCannotBeApplied(t *testing.T) { + for _, batch := range []bool{false, true} { + name := "batch_commit_disabled" + if batch { + name = "batch_commit_enabled" + } + t.Run(name, func(t *testing.T) { + restore := cfg.Config.BatchCommit.Enabled + cfg.Config.BatchCommit.Enabled = batch + defer func() { cfg.Config.BatchCommit.Enabled = restore }() + + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 1000, 1) + + const txnID = 7300 + prep := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, + NodeID: 7, + StartTS: hlc.Timestamp{WallTime: 11}, + Database: "testdb", + Statements: []protocol.Statement{claimStatement(t, "testdb", "users", 1000, 1000, 64)}, + }) + require.True(t, prep.Success, "PREPARE refused a well-formed claim: %s", prep.Error) + + _, err := dm.GetSystemDatabase().GetWriteDB().Exec( + "CREATE TRIGGER refuse_base_write BEFORE UPDATE ON " + AutoIncClaimTable + + " BEGIN SELECT RAISE(ABORT, 'claim write refused'); END") + require.NoError(t, err) + + res := engine.Commit(context.Background(), &CommitRequest{ + TxnID: txnID, + Database: "testdb", + Statements: []protocol.Statement{commitStatement("testdb", "users")}, + }) + if res.Success { + t.Fatal("participant ACKed COMMIT although the claim row was not written") + } + // Pin the REASON, not just the refusal: a commit that failed + // because the transaction was missing would satisfy every other + // assertion here while proving nothing about the claim write. + if !strings.Contains(res.Error, "claim write refused") { + t.Fatalf("COMMIT failed for the wrong reason: %s", res.Error) + } + + base := readClaimBase(t, dm, "testdb", "users") + require.Equal(t, uint64(1000), base, "the base moved despite the refused write") + + // The transaction stays PENDING: a participant that cannot complete + // withholds its ACK and leaves recovery to resolve the transaction, + // rather than unilaterally discarding a transaction the rest of the + // cluster may be committing. + if txn := mdb.GetTransactionManager().GetTransaction(txnID); txn == nil { + t.Error("the transaction is no longer PENDING, so the node decided it alone") + } + }) + } +} + +// TestDroppedTableKeepsItsClaimRow holds the claim's second invariant, "the +// row is never deleted", against the one operation that most looks like it +// should delete it. +// +// This is a deliberate, named divergence from MySQL, where DROP TABLE discards +// the counter and a table recreated under the same name restarts at 1. Here the +// row outlives the table, so a recreated table continues above the dropped +// one's last id. The divergence is the safe direction: an absent row is a hard +// rejection at PREPARE, so any code path that removes rows opens a +// window in which a table re-mints ids it has already used -- recovery's +// pruning hazard. Ids restarting at 1 is cosmetic; ids repeating is corruption. +// +// The table is dropped through the replicated DDL path - PREPARE and COMMIT +// of a DDL statement - which is where DDL-time seeding runs +// (seedAutoIncBasesForDDL) and so the only place a DROP TABLE could reach the +// claim row. +// +// Mutation: in seedAutoIncBasesForDDL, delete the claim row of a DDL-touched +// table that no longer exists. "a DROP TABLE deleted the claim row" fires. +func TestDroppedTableKeepsItsClaimRow(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 1000, 1) + + drop := protocol.Statement{Type: protocol.StatementDDL, Database: "testdb", TableName: "users", SQL: "DROP TABLE users"} + prep := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: 7400, NodeID: 1, StartTS: hlc.Timestamp{WallTime: 13}, Database: "testdb", + Statements: []protocol.Statement{drop}, + }) + require.True(t, prep.Success, "PREPARE of DROP TABLE: %s", prep.Error) + res := engine.Commit(context.Background(), &CommitRequest{ + TxnID: 7400, Database: "testdb", Statements: []protocol.Statement{drop}, + }) + require.True(t, res.Success, "COMMIT of DROP TABLE: %s", res.Error) + var tables int + require.NoError(t, mdb.GetReadDB().QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE name = 'users'").Scan(&tables)) + require.Zero(t, tables, "the DDL did not drop the table, so this test proves nothing") + + base, err := autoIncClaimStoreForTest(dm).ReadBase("testdb", "users") + require.NoError(t, err, "a DROP TABLE deleted the claim row") + require.Equal(t, uint64(1000), base) +} + +// dmlLogInsertStatement builds a real client DML statement (not a claim) that +// carries a CDC row image, exactly as PREPARE requires for the DML path +// (createDMLIntent, db/replication_engine.go). Riding alongside a claim +// statement in the same PREPARE request is what makes cdcEntries > 0 at +// COMMIT (db/transaction.go:269,277), which is the only thing that routes +// CommitTransaction into the batch committer at all. +func dmlLogInsertStatement(database, intentKey string, id int64, v string) protocol.Statement { + return testProtocolDMLStatement(protocol.StatementInsert, database, "log", []byte(intentKey), nil, + encodeTestValues(map[string]interface{}{"id": id, "v": v})) +} + +// dmlLogCommitStatement is the COMMIT-side decision metadata for the same +// insert: type, table, and intent key only, no row image (COMMIT never +// carries a payload -- CommitTransaction rebuilds txn.Statements from the CDC +// entries persisted at PREPARE). +func dmlLogCommitStatement(database, intentKey string) protocol.Statement { + return protocol.Statement{ + Type: protocol.StatementInsert, + Database: database, + TableName: "log", + IntentKey: []byte(intentKey), + } +} + +// TestParticipantWithholdsAckWhenClaimRowIsNotDurable is the claim's first +// invariant, proved with a transaction that genuinely reaches CommitTransaction's +// DML branch: the PREPARE carries a claim statement AND an ordinary DML insert, +// so GetIntentEntries returns a non-empty slice and the len(cdcEntries) > 0 +// branch at db/transaction.go:277 is the one taken, not the DDL/statement +// branch at :294 a claim-only transaction would take. +// +// The claim apply itself still fails and refuses the commit before +// CommitTransaction is ever called (db/replication_engine.go: the +// AutoIncClaimStore.ApplyClaims check at the statementsCarryAutoIDClaim gate returns +// early, above txnMgr.CommitTransaction). That ordering is itself part of the +// invariant: a claim that cannot be written must never let the surrounding +// transaction's DML commit either. Both assertions here hold in both batch +// configurations, and the log table row count pins that the DML never landed. +// +// Mutation: apply the claim AFTER txnMgr.CommitTransaction, or swallow the +// apply error and return Success: true. Either way the node reports a commit +// it did not durably make and both assertions below fire. +func TestParticipantWithholdsAckWhenClaimRowIsNotDurable(t *testing.T) { + for _, batch := range []bool{false, true} { + name := "batch_commit_disabled" + if batch { + name = "batch_commit_enabled" + } + t.Run(name, func(t *testing.T) { + restore := cfg.Config.BatchCommit.Enabled + cfg.Config.BatchCommit.Enabled = batch + defer func() { cfg.Config.BatchCommit.Enabled = restore }() + + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 1000, 1) + _, err := mdb.GetWriteDB().Exec("CREATE TABLE log (id INTEGER PRIMARY KEY, v TEXT)") + require.NoError(t, err) + + const txnID = 7500 + prep := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, + NodeID: 7, + StartTS: hlc.Timestamp{WallTime: 12}, + Database: "testdb", + Statements: []protocol.Statement{ + claimStatement(t, "testdb", "users", 1000, 1000, 64), + dmlLogInsertStatement("testdb", "log:1", 1, "x"), + }, + }) + require.True(t, prep.Success, "PREPARE refused claim+DML: %s", prep.Error) + + // Proves the precondition the whole test rests on: this is a real + // DML transaction, not a claim-only one, so toggling + // BatchCommit.Enabled actually selects between CommitTransaction's + // two branches (db/transaction.go:277 vs :294) rather than being a + // no-op for this commit. + entries, err := mdb.GetMetaStore().GetIntentEntries(txnID) + require.NoError(t, err) + require.Len(t, entries, 1, "DML must produce a CDC entry or this test does not exercise the DML/batch branch") + + _, err = dm.GetSystemDatabase().GetWriteDB().Exec( + "CREATE TRIGGER refuse_base_write BEFORE UPDATE ON " + AutoIncClaimTable + + " BEGIN SELECT RAISE(ABORT, 'claim write refused'); END") + require.NoError(t, err) + + res := engine.Commit(context.Background(), &CommitRequest{ + TxnID: txnID, + Database: "testdb", + Statements: []protocol.Statement{ + commitStatement("testdb", "users"), + dmlLogCommitStatement("testdb", "log:1"), + }, + }) + if res.Success { + t.Fatal("participant ACKed COMMIT although the claim row was not written") + } + if !strings.Contains(res.Error, "claim write refused") { + t.Fatalf("COMMIT failed for the wrong reason: %s", res.Error) + } + + base := readClaimBase(t, dm, "testdb", "users") + require.Equal(t, uint64(1000), base, "the base moved despite the refused write") + + var logCount int + require.NoError(t, mdb.GetReadDB().QueryRow("SELECT COUNT(*) FROM log").Scan(&logCount)) + require.Zero(t, logCount, "the DML riding with the failed claim must not land either") + + if txn := mdb.GetTransactionManager().GetTransaction(txnID); txn == nil { + t.Error("the transaction is no longer PENDING, so the node decided it alone") + } + }) + } +} + +// TestParticipantWithholdsAckWhenTransactionCommitFailsAfterClaimApplies is the +// complementary case: the claim itself commits durably (AutoIncClaimStore.ApplyClaims +// succeeds, so the base row is written and durable independent of anything +// that follows), but the rest of the transaction's own DML +// commit then fails. This is where the batch committer's blocking fut.Get() +// (db/transaction.go:282) is the only thing standing between a failed DML +// apply and a false ACK: Enqueue hands the work to the flush loop +// asynchronously, and only a synchronous wait on the future keeps Commit() +// from returning Success: true before the flush loop's outcome is known. +// +// The base having already moved while Success is false is not a bug this test +// hunts for -- it is the documented design (db/replication_engine.go's comment +// on the AutoIncClaimStore.ApplyClaims call): a participant that cannot complete +// withholds its ACK and leaves the transaction PENDING for recovery, rather +// than silently discarding a claim the rest of the cluster may already be +// counting as committed. The invariant under test is narrower and absolute: +// Success must be false whenever the surrounding commit did not complete, in +// both batch configurations. +// +// Mutation: return Success: true whenever AutoIncClaimStore.ApplyClaims succeeds, without +// checking txnMgr.CommitTransaction's error (or drop the batch committer's +// fut.Get() and treat Enqueue as fire-and-forget). Either way res.Success is +// true while the log table holds no row, and the assertions below fire. +func TestParticipantWithholdsAckWhenTransactionCommitFailsAfterClaimApplies(t *testing.T) { + for _, batch := range []bool{false, true} { + name := "batch_commit_disabled" + if batch { + name = "batch_commit_enabled" + } + t.Run(name, func(t *testing.T) { + restore := cfg.Config.BatchCommit.Enabled + cfg.Config.BatchCommit.Enabled = batch + defer func() { cfg.Config.BatchCommit.Enabled = restore }() + + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 1000, 1) + _, err := mdb.GetWriteDB().Exec("CREATE TABLE log (id INTEGER PRIMARY KEY, v TEXT)") + require.NoError(t, err) + + const txnID = 7600 + prep := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, + NodeID: 7, + StartTS: hlc.Timestamp{WallTime: 13}, + Database: "testdb", + Statements: []protocol.Statement{ + claimStatement(t, "testdb", "users", 1000, 1000, 64), + dmlLogInsertStatement("testdb", "log:1", 1, "x"), + }, + }) + require.True(t, prep.Success, "PREPARE refused claim+DML: %s", prep.Error) + + entries, err := mdb.GetMetaStore().GetIntentEntries(txnID) + require.NoError(t, err) + require.Len(t, entries, 1, "DML must produce a CDC entry or this test does not exercise the DML/batch branch") + + // The claim table is untouched: only the DML's own apply fails, so + // AutoIncClaimStore.ApplyClaims succeeds and CommitTransaction is reached. + _, err = mdb.GetWriteDB().Exec( + "CREATE TRIGGER refuse_log_insert BEFORE INSERT ON log" + + " BEGIN SELECT RAISE(ABORT, 'dml commit refused'); END") + require.NoError(t, err) + + res := engine.Commit(context.Background(), &CommitRequest{ + TxnID: txnID, + Database: "testdb", + Statements: []protocol.Statement{ + commitStatement("testdb", "users"), + dmlLogCommitStatement("testdb", "log:1"), + }, + }) + if res.Success { + t.Fatal("participant ACKed COMMIT although the transaction's own DML commit failed") + } + if !strings.Contains(res.Error, "dml commit refused") { + t.Fatalf("COMMIT failed for the wrong reason: %s", res.Error) + } + + base := readClaimBase(t, dm, "testdb", "users") + require.Equal(t, uint64(1064), base, + "the claim apply is a separate, already-committed write; "+ + "a later failure in the same transaction must not un-commit it") + + var logCount int + require.NoError(t, mdb.GetReadDB().QueryRow("SELECT COUNT(*) FROM log").Scan(&logCount)) + require.Zero(t, logCount, "the failed DML must not land despite the claim having committed") + + if txn := mdb.GetTransactionManager().GetTransaction(txnID); txn == nil { + t.Error("the transaction is no longer PENDING, so the node decided it alone") + } + }) + } +} diff --git a/db/autoinc_hookdb_test.go b/db/autoinc_hookdb_test.go new file mode 100644 index 00000000..c0f6b04f --- /dev/null +++ b/db/autoinc_hookdb_test.go @@ -0,0 +1,131 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// TestClaimCompletesWhilePinnedSessionOpen is the LLDAP-shape property, and it +// is the reason the claim row lives in the system database rather than in the +// user database it is about. +// +// The sea-orm/LLDAP shape is: BEGIN; INSERT INTO users ...; INSERT INTO +// groups ...; COMMIT. Marmot serves that shape with a pinned session +// (db/db_integration.go BeginPinnedSession), which takes hookDB's single +// connection (hookDB.SetMaxOpenConns(1)) and opens a SQLite transaction on it +// that stays open until Release. Both pools are built with _txlock=immediate +// (db/db_integration.go writeDSN, hookDSN), so that open transaction holds the +// USER database's one SQLite writer for its whole lifetime. +// +// The second INSERT in that shape is what needs a range, so its claim runs +// while the first INSERT's pinned session is still open. Two distinct ways to +// get this wrong; both block, and the bounded timeout below turns either into +// a failure (the second is the one run as a mutation): +// +// - The claim takes a connection from hookDB (directly, or via +// StartEphemeralSession). hookDB has one slot and the pinned session holds +// it, so the claim blocks behind the transaction its own request is part +// of: a self-deadlock, not a cross-request one. +// - The claim writes the claim row into the USER database (the pre-relocation +// design). Its BEGIN IMMEDIATE on GetWriteDB() waits on the same writer the +// pinned session holds, and blocks just as hard. +// +// The claim instead writes the SYSTEM database (AutoIncClaimStore, a separate +// SQLite file with its own writer), so it completes promptly. The claim below +// is driven through the real PREPARE and COMMIT handlers on the SAME +// ReplicatedDatabase whose writer the pinned session holds, under a bounded +// timeout far below pinnedSessionTimeout's default of 50s +// (coordinator/handler.go), so a regression fails by name instead of hanging +// the suite. +// +// Mutation: point AutoIncClaimStore at the user database instead of the system +// database (NewAutoIncClaimStore(replicatedDB) in +// ReplicationEngine.autoIncClaimStore). The COMMIT apply's BEGIN blocks on the +// pinned session's writer and the select's timeout branch fires the t.Fatal +// below. +func TestClaimCompletesWhilePinnedSessionOpen(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + _, err := mdb.GetWriteDB().Exec("CREATE TABLE groups (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + require.NoError(t, err) + require.NoError(t, mdb.ReloadSchema()) + seedClaimBase(t, dm, "testdb", "groups", 3000, 1) + + // Statement one of the shape: a real pinned session on the user database, + // with a real INSERT already executed on it, held open for the rest of the + // test. Per BeginPinnedSession's doc comment pinnedCtx must stay alive + // until Release is called. + pinnedCtx, pinnedCancel := context.WithCancel(context.Background()) + defer pinnedCancel() + pinned, err := mdb.BeginPinnedSession(pinnedCtx, 9500) + require.NoError(t, err, "failed to open pinned session") + defer func() { _ = pinned.Release() }() + + _, _, err = pinned.ExecuteStatement(pinnedCtx, + "INSERT INTO users (id, v) VALUES (?, ?)", []interface{}{int64(1), "ada"}) + require.NoError(t, err, "the pinned session's first INSERT failed") + + // Build the claim statements up front so the goroutine below touches no + // *testing.T: calling require/t.Fatal from a non-test goroutine is unsafe, + // and a failure inside the goroutine must instead surface as a value on + // the result channel that the test goroutine asserts on. + claimStmt := claimStatement(t, "testdb", "groups", 3000, 3000, 16) + commitStmt := commitStatement("testdb", "groups") + + type claimOutcome struct{ err error } + done := make(chan claimOutcome, 1) + go func() { + const txnID = 9501 + prep := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, + NodeID: 9, + StartTS: hlc.Timestamp{WallTime: 30}, + Database: "testdb", + Statements: []protocol.Statement{claimStmt}, + }) + if !prep.Success { + done <- claimOutcome{err: fmt.Errorf("PREPARE refused a well-formed claim: %s", prep.Error)} + return + } + + res := engine.Commit(context.Background(), &CommitRequest{ + TxnID: txnID, + Database: "testdb", + Statements: []protocol.Statement{commitStmt}, + }) + if !res.Success { + done <- claimOutcome{err: fmt.Errorf("COMMIT failed: %s", res.Error)} + return + } + done <- claimOutcome{} + }() + + select { + case outcome := <-done: + require.NoError(t, outcome.err, "claim through PREPARE/COMMIT did not complete cleanly") + case <-time.After(5 * time.Second): + t.Fatal("claim through PREPARE/COMMIT blocked while a pinned session held the user " + + "database's writer and hookDB's single connection: the claim must write the system " + + "database and must never contend for either") + } + + // The claim committed durably, not merely promptly. + require.Equal(t, uint64(3016), readClaimBase(t, dm, "testdb", "groups"), + "the claim did not durably commit") + + // The pinned session is still usable, so the claim neither stole its + // connection nor rolled its transaction back underneath it. + _, _, err = pinned.ExecuteStatement(pinnedCtx, + "INSERT INTO groups (id, v) VALUES (?, ?)", []interface{}{int64(3001), "admins"}) + require.NoError(t, err, "the pinned session was disturbed by the concurrent claim") +} diff --git a/db/autoinc_prepare_test.go b/db/autoinc_prepare_test.go new file mode 100644 index 00000000..81d21b8f --- /dev/null +++ b/db/autoinc_prepare_test.go @@ -0,0 +1,368 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "context" + "strings" + "testing" + + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// claimStatement builds the wire shape a range claim takes: an existing DML +// statement type, a non-empty intent key, the payload, and no row image. +func claimStatement(t *testing.T, database, table string, prevBase, newBase, size uint64) protocol.Statement { + t.Helper() + payload, err := protocol.EncodeAutoIncClaim(AutoIncClaim{ + Table: table, PrevBase: prevBase, NewBase: newBase, Size: size, + }) + require.NoError(t, err) + return protocol.Statement{ + Type: protocol.StatementInsert, + Database: database, + TableName: table, + IntentKey: []byte(protocol.AutoIncClaimKey(database, table)), + AutoIDClaim: true, + AutoIDClaimPayload: payload, + } +} + +// markedTableDB sets up a database holding one marked AUTO_INCREMENT table. +func markedTableDB(t *testing.T, engine *ReplicationEngine, dm *DatabaseManager, ddl string) *ReplicatedDatabase { + t.Helper() + require.NoError(t, dm.CreateDatabase("testdb")) + mdb, err := dm.GetDatabase("testdb") + require.NoError(t, err) + _, err = mdb.GetWriteDB().Exec(ddl) + require.NoError(t, err) + require.NoError(t, mdb.ReloadSchema()) + return mdb +} + +// autoIncClaimStoreForTest wraps dm's system database as an AutoIncClaimStore, +// the same store DatabaseManager.wireGCCoordination injects into every +// TransactionManager in production (db/database_manager.go). Tests drive +// claim seeding and readback through this instead of a method on the user +// database, since the claim table lives in the system database now. +func autoIncClaimStoreForTest(dm *DatabaseManager) *AutoIncClaimStore { + return NewAutoIncClaimStore(dm.GetSystemDatabase()) +} + +// seedClaimBase seeds a (database, table) claim row directly through the +// system-database store, standing in for what seedAutoIncBasesForDDL does at +// DDL time. +func seedClaimBase(t *testing.T, dm *DatabaseManager, database, table string, base, owner uint64) { + t.Helper() + require.NoError(t, autoIncClaimStoreForTest(dm).Seed(database, table, base, owner)) +} + +// readClaimBase reads a (database, table) claim row's base directly through +// the system-database store. +func readClaimBase(t *testing.T, dm *DatabaseManager, database, table string) uint64 { + t.Helper() + base, err := autoIncClaimStoreForTest(dm).ReadBase(database, table) + require.NoError(t, err) + return base +} + +// TestPrepareClaimRejectsAbsentBase is the fail-closed clause, and it is the one +// an implementer gets wrong: the natural implementation is "absent means 0 +// means yes", and a cache that defaults to 0 and accepts is precisely the thing +// that cannot cast the rejection that would repair it. +// +// Mutation: treat ErrAutoIncBaseAbsent as base 0 and fall through to the +// comparison. The claim is accepted and this fires. +func TestPrepareClaimRejectsAbsentBase(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + + // No claim row has ever been written for this table. + result := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: 5001, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: 1}, + Database: "testdb", + Statements: []protocol.Statement{claimStatement(t, "testdb", "users", 0, 0, 64)}, + }) + + if result.Success { + t.Fatal("a participant with no stored base ACKed a range claim; absent must reject") + } + if !result.Rejected { + t.Error("the refusal was not marked Rejected, so the coordinator would retry it as a timeout") + } + if !strings.Contains(result.Error, "users") { + t.Errorf("rejection %q does not name the table", result.Error) + } +} + +// TestPrepareClaimCondition walks the three comparisons the condition makes. +func TestPrepareClaimCondition(t *testing.T) { + cases := []struct { + name string + storedBase uint64 + prevBase uint64 + newBase uint64 + size uint64 + wantAccept bool + wantReturnedBase uint64 + }{ + { + name: "fresh claim on a seeded table is accepted", + storedBase: 0, prevBase: 0, newBase: 0, size: 64, wantAccept: true, + }, + { + name: "a claim proposing this node's own base is accepted", + storedBase: 1000, prevBase: 1000, newBase: 1000, size: 64, wantAccept: true, + }, + { + // A participant that missed an earlier commit round holds a base + // BELOW the claimant's. It must still accept: the commit will carry + // it forward. Mutation: tighten storedBase > prevBase to !=. + name: "a claim from ahead of this node's base is accepted", + storedBase: 500, prevBase: 1000, newBase: 1000, size: 64, wantAccept: true, + }, + { + // The soundness case: this participant already committed a higher + // claim, so the proposer lost the race and must be told the real base. + // Mutation: drop the storedBase > prevBase comparison. + name: "stale claim is rejected and returns this node's base", + storedBase: 5000, prevBase: 1000, newBase: 1000, size: 64, + wantAccept: false, wantReturnedBase: 5000, + }, + { + // Allocation is lowest-free: a claimant proposes its OWN view of + // the base and nothing else. Both directions are refused, so the + // term is == and not >=. + // Mutation: relax newBase != prevBase to newBase < prevBase. + name: "a claim below its own view of the base is rejected", + storedBase: 1000, prevBase: 1000, newBase: 999, size: 64, + wantAccept: false, wantReturnedBase: 1000, + }, + { + // Mutation: relax newBase != prevBase to newBase > prevBase. + name: "a claim above its own view of the base is rejected", + storedBase: 1000, prevBase: 1000, newBase: 1500, size: 64, + wantAccept: false, wantReturnedBase: 1000, + }, + { + // A zero-size claim would leave the committed base where it is, so + // the same range could be handed out twice. + // Mutation: drop the size == 0 comparison. + name: "a zero-size claim is rejected", + storedBase: 1000, prevBase: 1000, newBase: 1000, size: 0, + wantAccept: false, wantReturnedBase: 1000, + }, + { + // Mutation: drop the widthMax term. The column is INT, so its + // ceiling is 2147483647; a range ending above it would mint ids the + // column cannot hold. + name: "a claim past the column's ceiling is rejected", + storedBase: 2147483000, prevBase: 2147483000, newBase: 2147483000, size: 10000, + wantAccept: false, wantReturnedBase: 2147483000, + }, + { + // 2^64 - 2147483000: the additive form newBase+size wraps to exactly + // 0, which compares as comfortably inside a signed INT's ceiling, so + // a claim for the whole key space would be accepted. + // Mutation: write the ceiling arm as claim.NewBase+claim.Size > widthMax. + name: "a claim whose size overflows uint64 is rejected", + storedBase: 2147483000, prevBase: 2147483000, newBase: 2147483000, + size: 18446744071562068616, + wantAccept: false, wantReturnedBase: 2147483000, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", tc.storedBase, 1) + + result := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: 6001, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: 1}, + Database: "testdb", + Statements: []protocol.Statement{claimStatement(t, "testdb", "users", tc.prevBase, tc.newBase, tc.size)}, + }) + + if tc.wantAccept { + if !result.Success { + t.Fatalf("claim rejected, want accepted: %s", result.Error) + } + return + } + if result.Success { + t.Fatal("claim accepted, want rejected") + } + // The claimant retries with the maximum base returned, so a + // rejection that carries nothing makes it spin. + // Mutation: leave AutoIDStoredBase unset on the rejection paths. + if result.AutoIDStoredBase != tc.wantReturnedBase { + t.Errorf("rejection returned base %d, want %d", result.AutoIDStoredBase, tc.wantReturnedBase) + } + }) + } +} + +// TestPrepareClaimDerivesWidthFromItsOwnSchema pins that the ceiling comes from +// this node's own sqlite_master and not from the claimant. A claim carries no +// width at all, so a node whose column is narrower rejects a range a wider +// node would have accepted. +// +// Mutation: take a width from the claim payload. The SMALLINT node accepts a +// range only an INT column could hold. +func TestPrepareClaimDerivesWidthFromItsOwnSchema(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + // SMALLINT: ceiling 32767. + markedTableDB(t, engine, dm, "CREATE TABLE small (id INTEGER /*M:16a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "small", 32000, 1) + + result := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: 7001, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: 1}, + Database: "testdb", + Statements: []protocol.Statement{claimStatement(t, "testdb", "small", 32000, 32000, 1000)}, + }) + + if result.Success { + t.Fatal("a SMALLINT column accepted a range ending at 33000, past its 32767 ceiling") + } + if !strings.Contains(result.Error, "exhausts") { + t.Errorf("rejection %q does not name exhaustion", result.Error) + } +} + +// TestPrepareClaimTakesIntentLockBeforeReadingBase pins the ORDER of the two +// halves of a participant's vote, which is the whole of the protocol's +// soundness argument under concurrency. +// +// A participant's vote is "my stored base is not above your prevBase". Reading +// the base and writing the intent are each individually synchronised, so -race +// sees nothing wrong; what matters is that they cannot straddle another +// claimant's commit. If the base is read BEFORE the intent's row lock is +// taken, this interleaving grants the same ids twice: +// +// B reads base 0 -> A commits, base becomes 100, A's lock is released -> +// B's WriteIntent now succeeds -> B is accepted for [0,100) as well. +// +// The fix is to take the lock first, so the read happens inside it. The +// observable consequence, and what this test asserts, is that a claim whose +// lock is already held by a live transaction fails WITHOUT CASTING A VOTE: it +// comes back as a plain missing ACK (Rejected false, no base attached), not as +// a stale-base rejection, even though its base genuinely is stale. A node that +// answers "your base is stale, mine is 500" has by definition read the base +// without the lock. +// +// Mutation: move the AddStatement/WriteIntent pair back below the condition +// switch in prepareAutoIncClaim (db/replication_engine.go). The stale claim is +// then judged before the lock is attempted, the result comes back +// Rejected=true with AutoIDStoredBase=500, and the three assertions below fire +// by name. +func TestPrepareClaimTakesIntentLockBeforeReadingBase(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 500, 1) + + // Claimant A prepares and does not commit, so its intent - and the claim + // key's row lock - stays held for the rest of the test. + prepA := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: 8100, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: 100}, + Database: "testdb", + Statements: []protocol.Statement{claimStatement(t, "testdb", "users", 500, 500, 10)}, + }) + require.True(t, prepA.Success, "PREPARE refused a well-formed claim: %s", prepA.Error) + + // Claimant B is genuinely stale: this node holds 500 and B proposes 0. + prepB := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: 8101, + NodeID: 2, + StartTS: hlc.Timestamp{WallTime: 101}, + Database: "testdb", + Statements: []protocol.Statement{claimStatement(t, "testdb", "users", 0, 0, 10)}, + }) + require.False(t, prepB.Success, "a second claim was accepted while another holds the claim key's lock") + require.False(t, prepB.Rejected, + "the node cast a verdict on a claim whose lock it never held: the base was read outside the lock") + require.Zero(t, prepB.AutoIDStoredBase, + "the node returned its stored base for a claim it never got the lock to judge") + require.Contains(t, prepB.Error, "write-write conflict", + "expected the lost race for the claim key's row lock, got: %s", prepB.Error) + + // The base is untouched by either attempt: only COMMIT advances it. + require.Equal(t, uint64(500), readClaimBase(t, dm, "testdb", "users")) +} + +// TestPrepareClaimRejectsOnceTheWinnerHasCommitted is the other side of +// TestPrepareClaimTakesIntentLockBeforeReadingBase: once the lock IS free, the +// base the loser reads is the winner's committed one, so it rejects with a +// base its coordinator can retry above instead of being granted the same ids. +// +// The winner commits before the loser prepares, so this does not exercise +// the read-under-lock ordering (TestPrepareClaimTakesIntentLockBeforeReadingBase +// does); it pins the verdict and the retry base once the lock is free. +// +// Mutation: drop the "storedBase > prevBase" rejection from +// prepareAutoIncClaim and "the winner's committed range was handed out a +// second time" fires; or omit AutoIDStoredBase from that rejection and "the +// rejection must carry this node's base" fires. +func TestPrepareClaimRejectsOnceTheWinnerHasCommitted(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 500, 1) + + const winner = 8200 + prep := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: winner, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: 200}, + Database: "testdb", + Statements: []protocol.Statement{claimStatement(t, "testdb", "users", 500, 500, 10)}, + }) + require.True(t, prep.Success, "PREPARE refused a well-formed claim: %s", prep.Error) + res := engine.Commit(context.Background(), &CommitRequest{ + TxnID: winner, + Database: "testdb", + Statements: []protocol.Statement{commitStatement("testdb", "users")}, + }) + require.True(t, res.Success, "COMMIT failed: %s", res.Error) + require.Equal(t, uint64(510), readClaimBase(t, dm, "testdb", "users")) + + loser := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: 8201, + NodeID: 2, + StartTS: hlc.Timestamp{WallTime: 201}, + Database: "testdb", + Statements: []protocol.Statement{claimStatement(t, "testdb", "users", 500, 500, 10)}, + }) + require.False(t, loser.Success, "the winner's committed range was handed out a second time") + require.True(t, loser.Rejected, "a stale claim on a free lock must be a verdict, not a missing ACK") + require.Equal(t, uint64(510), loser.AutoIDStoredBase, + "the rejection must carry this node's base so the claimant retries above it") +} + +// TestPrepareResultConversionKeepsTheStoredBase: the coordinator's own node +// answers PREPARE through LocalReplicator, which returns +// PrepareResult.ToCoordinatorResponse. A rejected claim's stored base must +// survive that conversion, or the coordinator never learns the base to retry +// above from its own vote. +// +// Mutation: drop AutoIDStoredBase from PrepareResult.ToCoordinatorResponse. +// "the local PREPARE path dropped a rejected claim's stored base" fires. +func TestPrepareResultConversionKeepsTheStoredBase(t *testing.T) { + res := (&PrepareResult{Rejected: true, AutoIDStoredBase: 510}).ToCoordinatorResponse() + require.Equal(t, uint64(510), res.AutoIDStoredBase, "the local PREPARE path dropped a rejected claim's stored base") +} diff --git a/db/autoinc_restore_merge_test.go b/db/autoinc_restore_merge_test.go new file mode 100644 index 00000000..a6dfb99d --- /dev/null +++ b/db/autoinc_restore_merge_test.go @@ -0,0 +1,164 @@ +package db + +import ( + "database/sql" + "database/sql/driver" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/require" +) + +// newClaimFile creates a system-database file holding the given claim rows +// for database "d", in WAL mode as the system database runs. +func newClaimFile(t *testing.T, path string, withTable bool, bases map[string]int64) { + t.Helper() + conn, err := sql.Open(SQLiteDriverName, path) + require.NoError(t, err) + defer func() { require.NoError(t, conn.Close()) }() + _, err = conn.Exec("PRAGMA journal_mode=WAL") + require.NoError(t, err) + _, err = conn.Exec("CREATE TABLE registry (name TEXT)") + require.NoError(t, err) + if !withTable { + return + } + _, err = conn.Exec(autoIncClaimDDL) + require.NoError(t, err) + for tbl, base := range bases { + _, err = conn.Exec("INSERT INTO "+AutoIncClaimTable+" (db, tbl, base, owner, granted_at) VALUES ('d', ?, ?, 1, 1)", tbl, base) + require.NoError(t, err) + } +} + +// claimBasesIn reads every claim base for database "d" from a lone .db file, +// copied away from any -wal/-shm beside it, as the restorer moves it. +func claimBasesIn(t *testing.T, path string) map[string]int64 { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + lone := filepath.Join(t.TempDir(), "lone.db") + require.NoError(t, os.WriteFile(lone, data, 0o644)) + + conn, err := sql.Open(SQLiteDriverName, lone) + require.NoError(t, err) + defer func() { require.NoError(t, conn.Close()) }() + rows, err := conn.Query("SELECT tbl, base FROM " + AutoIncClaimTable + " WHERE db = 'd'") + require.NoError(t, err) + defer rows.Close() + bases := map[string]int64{} + for rows.Next() { + var tbl string + var base int64 + require.NoError(t, rows.Scan(&tbl, &base)) + bases[tbl] = base + } + require.NoError(t, rows.Err()) + return bases +} + +// TestRaiseAutoIncBasesFromNeverLowersABase: a restore installs a peer's system +// database. Every base must end at max(local, incoming): the peer may have +// missed a claim this node committed as part of a majority, and a lowered base +// would let this node accept that range again. +// +// Mutation: drop "WHERE excluded.base > base" from the merge's upsert. The +// local copy then overwrites a higher incoming base and "a restore lowered a +// base" fires. +func TestRaiseAutoIncBasesFromNeverLowersABase(t *testing.T) { + dir := t.TempDir() + incoming := filepath.Join(dir, "incoming.db") + local := filepath.Join(dir, "local.db") + newClaimFile(t, incoming, true, map[string]int64{"stale_on_peer": 500, "ahead_on_peer": 700}) + newClaimFile(t, local, true, map[string]int64{"stale_on_peer": 900, "ahead_on_peer": 100, "only_local": 50}) + + require.NoError(t, RaiseAutoIncBasesFrom(incoming, local)) + + got := claimBasesIn(t, incoming) + require.Equal(t, int64(900), got["stale_on_peer"], "the peer's lower base replaced this node's committed one") + require.Equal(t, int64(700), got["ahead_on_peer"], "a restore lowered a base") + require.Equal(t, int64(50), got["only_local"], "a row only this node held was lost") +} + +// TestRaiseAutoIncBasesFromAPeerWithoutTheClaimTable: a peer running a binary +// that predates the claim table ships a system database without it; this +// node's rows must still land. +func TestRaiseAutoIncBasesFromAPeerWithoutTheClaimTable(t *testing.T) { + dir := t.TempDir() + incoming := filepath.Join(dir, "incoming.db") + local := filepath.Join(dir, "local.db") + newClaimFile(t, incoming, false, nil) + newClaimFile(t, local, true, map[string]int64{"users": 42}) + + require.NoError(t, RaiseAutoIncBasesFrom(incoming, local)) + require.Equal(t, map[string]int64{"users": 42}, claimBasesIn(t, incoming)) +} + +// TestRaiseAutoIncBasesFromNoLocalState: a node with no system database of +// its own takes the peer's as it is. +func TestRaiseAutoIncBasesFromNoLocalState(t *testing.T) { + dir := t.TempDir() + incoming := filepath.Join(dir, "incoming.db") + newClaimFile(t, incoming, true, map[string]int64{"users": 7}) + + require.NoError(t, RaiseAutoIncBasesFrom(incoming, filepath.Join(dir, "absent.db"))) + require.Equal(t, map[string]int64{"users": 7}, claimBasesIn(t, incoming)) +} + +// TestRaiseAutoIncBasesFromMergesDurably: the restorer moves the incoming +// system database into place after the merge, so the connection that commits +// the merged bases must be as durable as every other system-database +// connection (WithDurableCommits): synchronous=FULL and, on darwin, +// fullfsync. +// +// Mutation: open the merge connection with SQLiteDriverName, or drop its +// "_sync=FULL". "the claim-base merge opened no durable connection" or "the +// claim-base merge ran on a connection that does not sync commits" fires. +func TestRaiseAutoIncBasesFromMergesDurably(t *testing.T) { + dir := t.TempDir() + incoming := filepath.Join(dir, "incoming.db") + local := filepath.Join(dir, "local.db") + newClaimFile(t, incoming, true, map[string]int64{"users": 7}) + newClaimFile(t, local, true, map[string]int64{"users": 9}) + + durable := sqliteDrivers[SQLiteDurableDriverName] + connectHook := durable.ConnectHook + t.Cleanup(func() { durable.ConnectHook = connectHook }) + type settings struct{ sync, fullfsync int64 } + var opened []settings + durable.ConnectHook = func(conn *sqlite3.SQLiteConn) error { + if err := connectHook(conn); err != nil { + return err + } + opened = append(opened, settings{sync: pragmaInt(t, conn, "synchronous"), fullfsync: pragmaInt(t, conn, "fullfsync")}) + return nil + } + + require.NoError(t, RaiseAutoIncBasesFrom(incoming, local)) + require.NotEmpty(t, opened, "the claim-base merge opened no durable connection") + wantFullfsync := int64(0) + if runtime.GOOS == "darwin" { + wantFullfsync = 1 + } + for _, s := range opened { + require.Equal(t, int64(2), s.sync, "the claim-base merge ran on a connection that does not sync commits") + require.Equal(t, wantFullfsync, s.fullfsync, "the claim-base merge ran on a connection that does not flush the drive cache") + } + require.Equal(t, map[string]int64{"users": 9}, claimBasesIn(t, incoming)) +} + +// pragmaInt reads an integer PRAGMA on a raw driver connection. +func pragmaInt(t *testing.T, conn *sqlite3.SQLiteConn, pragma string) int64 { + t.Helper() + rows, err := conn.Query("PRAGMA "+pragma, nil) + require.NoError(t, err) + defer rows.Close() + dest := make([]driver.Value, 1) + require.NoError(t, rows.Next(dest)) + v, ok := dest[0].(int64) + require.True(t, ok, "PRAGMA %s returned %T", pragma, dest[0]) + return v +} diff --git a/db/autoinc_seed.go b/db/autoinc_seed.go new file mode 100644 index 00000000..0892891a --- /dev/null +++ b/db/autoinc_seed.go @@ -0,0 +1,124 @@ +package db + +import ( + "database/sql" + "fmt" + + "github.com/maxpert/marmot/protocol/query/transform/intmarker" +) + +// rowQuerier is the read surface autoIncMarkedColumn needs. Both *sql.DB (the +// seed-at-commit path below, which runs after the DDL has already committed +// to this node's schema) and *sql.Tx (the widthMax check at PREPARE, in +// db/ddl_prepare_validate.go, which must see the validation transaction's own +// in-flight schema before it rolls back) satisfy it. +type rowQuerier interface { + QueryRow(query string, args ...interface{}) *sql.Row +} + +// autoIncMarkedColumn derives, from a table's OWN sqlite_master row read +// through q, the column carrying an explicitly declared AUTO_INCREMENT +// marker - the same derivation AutoIncWidthMax trusts, generalised to read +// from raw DDL text instead of the schema cache so it also works inside a +// not-yet-committed validation transaction. +// +// ok is false, with no error, when the table does not exist or carries no +// explicitly declared AUTO_INCREMENT marker: neither is a failure, both mean +// "this rule does not apply to this table." +func autoIncMarkedColumn(q rowQuerier, table string) (col string, attrs intmarker.Attributes, ok bool, err error) { + var createSQL string + scanErr := q.QueryRow("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", table).Scan(&createSQL) + switch { + case scanErr == sql.ErrNoRows: + return "", intmarker.Attributes{}, false, nil + case scanErr != nil: + return "", intmarker.Attributes{}, false, fmt.Errorf("read sqlite_master for %s: %w", table, scanErr) + } + + for name, a := range intmarker.Decode(createSQL) { + if a.Marked() && a.ExplicitAutoInc { + return name, a, true, nil + } + } + return "", intmarker.Attributes{}, false, nil +} + +// autoIncSeedFloor computes the base floor a table's DDL-declared +// AUTO_INCREMENT column implies right now: max(MAX() over the table's +// existing rows, the marker's own declared floor). ok is false when the table +// carries no explicitly declared AUTO_INCREMENT marker. +func autoIncSeedFloor(q rowQuerier, table string) (floor uint64, ok bool, err error) { + col, attrs, marked, err := autoIncMarkedColumn(q, table) + if err != nil || !marked { + return 0, false, err + } + + var max sql.NullInt64 + query := fmt.Sprintf("SELECT MAX(%s) FROM %s", quoteIdent(col), quoteIdent(table)) + if scanErr := q.QueryRow(query).Scan(&max); scanErr != nil { + return 0, false, fmt.Errorf("compute existing max for %s.%s: %w", table, col, scanErr) + } + + // A MAX at or below zero raises nothing: ids are issued upward from a + // non-negative base, so non-positive rows can never collide with one, and + // MySQL likewise starts such a table's counter at 1. + floor = attrs.AutoIncFloor + if max.Valid && max.Int64 > 0 && uint64(max.Int64) > floor { + floor = uint64(max.Int64) + } + return floor, true, nil +} + +// ddlTableOwner names one table a DDL intent in the current transaction +// touched, paired with the node that authored that intent. The owner comes +// from the intent record's own NodeID - identical on every node applying the +// same replicated intent - never from local state, exactly as +// AutoIncClaimStore.ApplyClaims records intent.NodeID as a claim's owner rather than +// trusting anything computed locally. +type ddlTableOwner struct { + table string + owner uint64 +} + +// seedAutoIncBasesForDDL seeds __marmot__autoinc, in the system database, for +// every table named by a DDL intent in this transaction that carries an +// explicitly declared AUTO_INCREMENT marker. It is a +// no-op for a table with no such marker. +// +// The seed floor is derived from THIS node's own sqlite_master and MAX(id) on +// the USER database (tm.db), read after the DDL has executed - not by parsing +// the DDL statement text - so the same logic uniformly covers CREATE TABLE and +// every ALTER shape that can leave a marker in sqlite_master. That derivation +// is unchanged by the claim table's relocation to the system database; only +// where the floor is written has moved. +// +// The write itself goes through tm.autoIncClaimStore, injected by +// DatabaseManager.wireGCCoordination (db/database_manager.go) and always +// backed by the system database (db/autoinc_claim.go). A marked table found +// with no store wired is a configuration error, not a "nothing to do" case, +// so it fails fast rather than silently skipping the seed. +func (tm *TransactionManager) seedAutoIncBasesForDDL(tables []ddlTableOwner) error { + seen := make(map[string]bool, len(tables)) + claimStore, dbName := tm.autoIncClaimStoreAndDatabaseName() + for _, t := range tables { + if t.table == "" || seen[t.table] { + continue + } + seen[t.table] = true + + floor, ok, err := autoIncSeedFloor(tm.db, t.table) + if err != nil { + return fmt.Errorf("derive auto-increment seed for %s: %w", t.table, err) + } + if !ok { + continue + } + if claimStore == nil { + return fmt.Errorf("seed auto-increment base for %s.%s: no auto-increment claim store wired", dbName, t.table) + } + if err := claimStore.Seed(dbName, t.table, floor, t.owner); err != nil { + return err + } + } + return nil +} diff --git a/db/autoinc_seed_test.go b/db/autoinc_seed_test.go new file mode 100644 index 00000000..403b8e42 --- /dev/null +++ b/db/autoinc_seed_test.go @@ -0,0 +1,334 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "database/sql" + "errors" + "testing" + + "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" +) + +func newSeedTestDB(t *testing.T) *sql.DB { + t.Helper() + db, err := sql.Open(SQLiteDriverName, ":memory:") + require.NoError(t, err) + db.SetMaxOpenConns(1) + t.Cleanup(func() { db.Close() }) + return db +} + +// newSeedTestStore builds an AutoIncClaimStore backed by its own standalone +// ReplicatedDatabase, standing in for the system database the claim table now +// lives in. It is deliberately a separate database from newSeedTestDB's +// in-memory *sql.DB: seedAutoIncBasesForDDL derives its floor from the USER +// database (tm.db) but writes through the injected store, which is always +// backed by the system database - the same split that +// TransactionManager.autoIncClaimStoreAndDatabaseName exists to serve. +func newSeedTestStore(t *testing.T) *AutoIncClaimStore { + t.Helper() + sys := newRowidTestDatabase(t, 1) + // The system database's initialisation creates the claim table + // (DatabaseManager's system-database setup); this fixture stands in for it. + _, err := sys.GetWriteDB().Exec(autoIncClaimDDL) + require.NoError(t, err) + return NewAutoIncClaimStore(sys) +} + +// readClaimRow reads a (database, table) claim row directly through the +// store's own system database, keyed by both database and table - the claim +// table now serves every user database from one system-database table, so a +// lookup by table name alone could match another database's row. +func readClaimRow(t *testing.T, store *AutoIncClaimStore, database, table string) (base, owner int64, ok bool) { + t.Helper() + base64, err := store.ReadBase(database, table) + if errors.Is(err, ErrAutoIncBaseAbsent) { + return 0, 0, false + } + require.NoError(t, err) + + err = store.sys.GetReadDB().QueryRow( + `SELECT owner FROM `+AutoIncClaimTable+` WHERE db = ? AND tbl = ?`, database, table).Scan(&owner) + require.NoError(t, err, "base was readable but owner was not for %s.%s", database, table) + return int64(base64), owner, true +} + +// TestSeedAutoIncBasesForDDL_NewMarkedTableSeedsZero pins the empty-table +// case: a brand-new table with a marked, explicitly declared AUTO_INCREMENT +// column and no rows seeds base 0. +func TestSeedAutoIncBasesForDDL_NewMarkedTableSeedsZero(t *testing.T) { + db := newSeedTestDB(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + + store := newSeedTestStore(t) + tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) + tm.SetAutoIncClaimStore(store) + tm.SetDatabaseName("testdb") + require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 7}})) + + base, owner, ok := readClaimRow(t, store, "testdb", "t") + require.True(t, ok, "expected a claim row to be created") + require.Equal(t, int64(0), base) + require.Equal(t, int64(7), owner) +} + +// TestSeedAutoIncBasesForDDL_ExistingRowsRaiseTheFloor pins the tagged-table +// case: a table already holding ids 1..1000 must seed a base at or above +// 1000, or the allocator would hand out ids that collide with live rows. +func TestSeedAutoIncBasesForDDL_ExistingRowsRaiseTheFloor(t *testing.T) { + db := newSeedTestDB(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + for i := 1; i <= 1000; i++ { + _, err := db.Exec(`INSERT INTO t (id, v) VALUES (?, ?)`, i, "x") + require.NoError(t, err) + } + + store := newSeedTestStore(t) + tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) + tm.SetAutoIncClaimStore(store) + tm.SetDatabaseName("testdb") + require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}})) + + base, _, ok := readClaimRow(t, store, "testdb", "t") + require.True(t, ok) + require.GreaterOrEqual(t, base, int64(1000)) +} + +// TestSeedAutoIncBasesForDDL_DeclaredFloorWins pins the other input to the +// floor: max(MAX(existing rows), the marker's own AutoIncFloor). A client +// that declares AUTO_INCREMENT=5000 on a table holding only a handful of low +// ids must still seed at the declared floor (4999), not at the smaller +// MAX(id). +func TestSeedAutoIncBasesForDDL_DeclaredFloorWins(t *testing.T) { + db := newSeedTestDB(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER /*M:32a:4999*/ PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + for i := 1; i <= 10; i++ { + _, err := db.Exec(`INSERT INTO t (id, v) VALUES (?, ?)`, i, "x") + require.NoError(t, err) + } + + store := newSeedTestStore(t) + tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) + tm.SetAutoIncClaimStore(store) + tm.SetDatabaseName("testdb") + require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}})) + + base, _, ok := readClaimRow(t, store, "testdb", "t") + require.True(t, ok) + require.Equal(t, int64(4999), base) +} + +// TestSeedAutoIncBasesForDDL_OnlyRaises pins the raise-only invariant end to +// end through the DDL-seeding path: seeding again with a lower current +// max/floor must never lower an already-stored base. +func TestSeedAutoIncBasesForDDL_OnlyRaises(t *testing.T) { + db := newSeedTestDB(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + for i := 1; i <= 1000; i++ { + _, err := db.Exec(`INSERT INTO t (id, v) VALUES (?, ?)`, i, "x") + require.NoError(t, err) + } + + store := newSeedTestStore(t) + tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) + tm.SetAutoIncClaimStore(store) + tm.SetDatabaseName("testdb") + require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}})) + base1, _, ok := readClaimRow(t, store, "testdb", "t") + require.True(t, ok) + require.GreaterOrEqual(t, base1, int64(1000)) + + // Second DDL touching the same table sees a much smaller MAX(id) (e.g. most + // rows were since deleted) and no declared floor; the stored base must not + // move down to it. + _, err = db.Exec(`DELETE FROM t WHERE id > 5`) + require.NoError(t, err) + require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}})) + + base2, _, ok := readClaimRow(t, store, "testdb", "t") + require.True(t, ok) + require.Equal(t, base1, base2, "seeding must only ever raise the base") +} + +// TestSeedAutoIncBasesForDDL_NoMarkerSeedsNothing pins the "this rule does +// not apply" path: a table with no explicitly declared AUTO_INCREMENT marker +// must get no claim row at all. +func TestSeedAutoIncBasesForDDL_NoMarkerSeedsNothing(t *testing.T) { + db := newSeedTestDB(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + + // A store IS wired here, deliberately: the point is that a table with no + // marker never reaches the store gate at all, not merely that a missing + // store happens not to matter. + store := newSeedTestStore(t) + tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) + tm.SetAutoIncClaimStore(store) + tm.SetDatabaseName("testdb") + require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}})) + + _, _, ok := readClaimRow(t, store, "testdb", "t") + require.False(t, ok, "a table with no AUTO_INCREMENT marker must get no claim row") +} + +// TestSeedAutoIncBasesForDDL_MarkedButNotExplicitAutoIncSeedsNothing pins the +// narrower condition: a merely-narrow column (e.g. TINYINT with no +// AUTO_INCREMENT) carries a width marker but is not the seeding target. +func TestSeedAutoIncBasesForDDL_MarkedButNotExplicitAutoIncSeedsNothing(t *testing.T) { + db := newSeedTestDB(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, flag INTEGER /*M:8*/)`) + require.NoError(t, err) + + store := newSeedTestStore(t) + tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) + tm.SetAutoIncClaimStore(store) + tm.SetDatabaseName("testdb") + require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}})) + + _, _, ok := readClaimRow(t, store, "testdb", "t") + require.False(t, ok) +} + +// TestSeedAutoIncBasesForDDL_MissingTableIsANoOp covers a DROP TABLE intent +// naming a table that no longer exists by the time seeding runs: absent is +// not an error. +func TestSeedAutoIncBasesForDDL_MissingTableIsANoOp(t *testing.T) { + db := newSeedTestDB(t) + tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) + require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "gone", owner: 1}})) +} + +// TestSeedAutoIncBasesForDDL_DedupesRepeatedTable pins that a table named by +// more than one DDL intent in the same transaction is only seeded once, using +// the first intent's owner. +func TestSeedAutoIncBasesForDDL_DedupesRepeatedTable(t *testing.T) { + db := newSeedTestDB(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER /*M:32a*/ PRIMARY KEY)`) + require.NoError(t, err) + + store := newSeedTestStore(t) + tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) + tm.SetAutoIncClaimStore(store) + tm.SetDatabaseName("testdb") + require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{ + {table: "t", owner: 42}, + {table: "t", owner: 99}, + })) + + _, owner, ok := readClaimRow(t, store, "testdb", "t") + require.True(t, ok) + require.Equal(t, int64(42), owner) +} + +// TestApplyNonDMLIntents_SeedsAutoIncBaseAfterCreateTable is the wiring +// test: a CREATE TABLE DDL intent applied through the real applyNonDMLIntents +// path (DDL exec, schema cache reload, then seeding) leaves a claim row for +// its marked column, owned by the intent's own NodeID. +func TestApplyNonDMLIntents_SeedsAutoIncBaseAfterCreateTable(t *testing.T) { + db := newSeedTestDB(t) + schemaCache := NewSchemaCache() + store := newSeedTestStore(t) + tm := NewTransactionManager(db, nil, hlc.NewClock(1), schemaCache) + tm.SetAutoIncClaimStore(store) + tm.SetDatabaseName("testdb") + + intents := []*WriteIntentRecord{ + { + IntentType: IntentTypeDDL, + TableName: "orders", + NodeID: 3, + SQLStatement: `CREATE TABLE orders (id INTEGER /*M:16a*/ PRIMARY KEY, total INTEGER)`, + }, + } + require.NoError(t, tm.applyNonDMLIntents(1, intents)) + + base, owner, ok := readClaimRow(t, store, "testdb", "orders") + require.True(t, ok, "expected applyNonDMLIntents to seed a claim row for the tagged table") + require.Equal(t, int64(0), base) + require.Equal(t, int64(3), owner) +} + +// TestApplyNonDMLIntents_UnmarkedDDLSeedsNothing is the negative wiring case: +// ordinary DDL with no AUTO_INCREMENT marker must not create a claim row. +func TestApplyNonDMLIntents_UnmarkedDDLSeedsNothing(t *testing.T) { + db := newSeedTestDB(t) + schemaCache := NewSchemaCache() + store := newSeedTestStore(t) + tm := NewTransactionManager(db, nil, hlc.NewClock(1), schemaCache) + tm.SetAutoIncClaimStore(store) + tm.SetDatabaseName("testdb") + + intents := []*WriteIntentRecord{ + { + IntentType: IntentTypeDDL, + TableName: "plain", + NodeID: 3, + SQLStatement: `CREATE TABLE plain (id INTEGER PRIMARY KEY, name TEXT)`, + }, + } + require.NoError(t, tm.applyNonDMLIntents(1, intents)) + + _, _, ok := readClaimRow(t, store, "testdb", "plain") + require.False(t, ok) +} + +// TestSeedAutoIncBasesForDDL_NoStoreWiredFailsFast pins the fail-fast +// contract: a marked table found with no claim store wired is a +// configuration error, not a "nothing to do" case. The natural mistake is to +// treat a nil store like an absent marker and silently skip the seed; that +// would leave a table live under AUTO_INCREMENT with no claim row, and the +// very next INSERT's PREPARE would then reject on ErrAutoIncBaseAbsent with no +// record of why. +// +// Mutation: fall through when claimStore == nil instead of returning the +// "no auto-increment claim store wired" error. +func TestSeedAutoIncBasesForDDL_NoStoreWiredFailsFast(t *testing.T) { + db := newSeedTestDB(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + + tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) + tm.SetDatabaseName("testdb") + // Deliberately no SetAutoIncClaimStore call. + + err = tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}}) + require.Error(t, err, "seeding a marked table with no store wired must fail, not silently skip") + require.Contains(t, err.Error(), "no auto-increment claim store wired") + require.Contains(t, err.Error(), "t", "the error should name the table it failed to seed") +} + +// TestSeedAutoIncBasesForDDL_NegativeIdsSeedZero: a table whose ids are all +// negative is tagged. A negative MAX(id) contributes nothing to the floor, as +// in MySQL, whose counter for such a table starts at 1; the base is 0 and +// readable. +// +// Mutation: cast a negative MAX(id) to uint64 as the floor (drop the +// "max.Int64 > 0" guard in autoIncSeedFloor). The stored base is then negative, +// ReadBase refuses it, every claim on the table is rejected, and "a table with +// only negative ids must seed base 0" fires. +func TestSeedAutoIncBasesForDDL_NegativeIdsSeedZero(t *testing.T) { + db := newSeedTestDB(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + for _, id := range []int{-5, -1} { + _, err := db.Exec(`INSERT INTO t (id, v) VALUES (?, ?)`, id, "x") + require.NoError(t, err) + } + + store := newSeedTestStore(t) + tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) + tm.SetAutoIncClaimStore(store) + tm.SetDatabaseName("testdb") + require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}})) + + base, err := store.ReadBase("testdb", "t") + require.NoError(t, err, "a table with only negative ids must seed base 0") + require.Equal(t, uint64(0), base, "a table with only negative ids must seed base 0") +} diff --git a/db/database_manager.go b/db/database_manager.go index cde2959e..1f392332 100644 --- a/db/database_manager.go +++ b/db/database_manager.go @@ -1,9 +1,11 @@ package db import ( + "context" "crypto/sha256" "database/sql" "encoding/hex" + "errors" "fmt" "io" "os" @@ -29,10 +31,20 @@ type DatabaseProvider interface { GetDatabase(name string) (*ReplicatedDatabase, error) } -// DatabaseManager manages multiple MVCC databases +// DatabaseManager manages multiple MVCC databases. +// +// Locking: mu guards the maps and the wiring fields and is only ever held for +// map access and wiring, never while a database's GC is stopped, because a GC +// pass takes mu (GetMinAppliedTxnID, and anti-entropy's refresh through +// ListDatabases). lifecycleMu serialises the operations that open, close, +// register or delete a database (Create, Drop, Detach, Attach, Import, Close), +// so one of them can close a database outside mu while the others wait; no GC +// pass ever takes lifecycleMu. type DatabaseManager struct { + lifecycleMu sync.Mutex mu sync.RWMutex databases map[string]*ReplicatedDatabase + detached map[string]*detachedDatabase // user databases out of service for a snapshot restore systemDB *ReplicatedDatabase dataDir string nodeID uint64 @@ -40,6 +52,7 @@ type DatabaseManager struct { refreshReplicationStates RefreshReplicationStatesFunc // Callback to refresh peer states before GC cdcHub CDCHub // CDC notification hub, can be nil vecIndexMgr *VectorIndexManager // Optional vector index manager + autoIncClaimStore *AutoIncClaimStore // AUTO_INCREMENT claim store, backed by systemDB } // DatabaseMetadata represents database registry information @@ -53,6 +66,7 @@ type DatabaseMetadata struct { func NewDatabaseManager(dataDir string, nodeID uint64, clock *hlc.Clock) (*DatabaseManager, error) { dm := &DatabaseManager{ databases: make(map[string]*ReplicatedDatabase), + detached: make(map[string]*detachedDatabase), dataDir: dataDir, nodeID: nodeID, clock: clock, @@ -94,15 +108,20 @@ func (dm *DatabaseManager) initSystemDatabase() error { return fmt.Errorf("failed to create system meta store: %w", err) } - systemDB, err := NewReplicatedDatabase(systemDBPath, dm.nodeID, dm.clock, metaStore) + // The system database holds the AUTO_INCREMENT claim bases, and a claim + // COMMIT this node ACKs counts toward the majority the claim protocol's + // safety rests on: the base must survive an OS crash or power loss, not + // only a process crash. User databases keep synchronous=NORMAL. + systemDB, err := NewReplicatedDatabase(systemDBPath, dm.nodeID, dm.clock, metaStore, WithDurableCommits()) if err != nil { metaStore.Close() return fmt.Errorf("failed to create system database: %w", err) } - // Wire up GC coordination for system database - dm.wireGCCoordination(systemDB, SystemDatabaseName) - + // dm.systemDB must be assigned BEFORE wireGCCoordination runs: that call + // wires the AUTO_INCREMENT claim store (db/autoinc_claim.go), which is + // always backed by dm.systemDB, and wireGCCoordination is itself called + // below for this very database. dm.systemDB = systemDB // Add system database to the databases map so it can be retrieved via GetDatabase() @@ -120,6 +139,16 @@ func (dm *DatabaseManager) initSystemDatabase() error { return fmt.Errorf("failed to create database registry table: %w", err) } + // Create the AUTO_INCREMENT claim table (db/autoinc_claim.go). It lives + // here, in the system database, rather than in each user database: see + // AutoIncClaimTable's doc comment for why. + if _, err = systemDB.GetDB().Exec(autoIncClaimDDL); err != nil { + return fmt.Errorf("failed to create auto-increment claim table: %w", err) + } + + // Wire up GC coordination for system database + dm.wireGCCoordination(systemDB, SystemDatabaseName) + log.Info().Str("path", systemDBPath).Msg("System database initialized") return nil } @@ -180,7 +209,7 @@ func (dm *DatabaseManager) openDatabase(name, path string) error { // This ensures transaction logs are retained until all peers have applied them // // NOTE: This function must NOT acquire dm.mu as it is called from contexts -// that already hold the write lock (CreateDatabase, ReopenDatabase, etc.) +// that already hold the write lock (CreateDatabase, AttachDatabase, etc.) // or during single-threaded initialization. Reading refreshReplicationStates // is safe because the caller either has exclusive access via write lock // or we're in initialization before any concurrent access is possible. @@ -202,6 +231,19 @@ func (dm *DatabaseManager) wireGCCoordination(mdb *ReplicatedDatabase, dbName st if dm.vecIndexMgr != nil { txnMgr.SetVectorCDCNotifier(dm.vecIndexMgr) } + + // Wire the AUTO_INCREMENT claim store (db/autoinc_claim.go). It always + // backs onto the system database, never onto mdb itself, so this wires the + // same store instance into every TransactionManager - user databases and + // the system database alike. dm.systemDB is nil only during the brief + // window before initSystemDatabase assigns it, which precedes this + // function's own call for the system database. + if dm.systemDB != nil { + if dm.autoIncClaimStore == nil { + dm.autoIncClaimStore = NewAutoIncClaimStore(dm.systemDB) + } + txnMgr.SetAutoIncClaimStore(dm.autoIncClaimStore) + } } // SetRefreshReplicationStatesFunc sets the callback for refreshing peer replication states @@ -292,11 +334,19 @@ func (dm *DatabaseManager) CreateDatabase(name string) error { return fmt.Errorf("cannot create system database") } - dm.mu.Lock() - defer dm.mu.Unlock() + dm.lifecycleMu.Lock() + defer dm.lifecycleMu.Unlock() // Check if database already exists - return success for idempotency (IF NOT EXISTS semantics) - if _, exists := dm.databases[name]; exists { + dm.mu.RLock() + _, exists := dm.databases[name] + detached := dm.detached[name] + dm.mu.RUnlock() + if detached != nil && detached.dropped { + // Its files go when the restore that holds them ends (AttachDatabase). + return fmt.Errorf("database %s is being dropped: %w", name, ErrDatabaseDetached) + } + if exists || detached != nil { log.Debug().Str("database", name).Msg("Database already exists, returning success") return nil } @@ -318,9 +368,6 @@ func (dm *DatabaseManager) CreateDatabase(name string) error { return fmt.Errorf("failed to create database file: %w", err) } - // Wire up GC coordination for newly created database - dm.wireGCCoordination(db, name) - // Register in system database createdAt := time.Now().UnixNano() _, err = dm.systemDB.GetDB().Exec( @@ -329,19 +376,28 @@ func (dm *DatabaseManager) CreateDatabase(name string) error { ) if err != nil { db.Close() - metaStore.Close() os.Remove(fullPath) cleanupMetaStoreFiles(fullPath) return fmt.Errorf("failed to register database in system: %w", err) } + // Wire up GC coordination only once the database is registered: until + // then its GC cannot reach mu, so the failure path above can close it. + dm.mu.Lock() + dm.wireGCCoordination(db, name) dm.databases[name] = db + dm.mu.Unlock() log.Info().Str("name", name).Str("path", dbPath).Msg("Database created") return nil } // DropDatabase drops a database // Returns nil if database doesn't exist (idempotent for IF EXISTS semantics) +// +// A database detached for a snapshot restore is dropped at once in the +// registry; its files go when the restore ends (AttachDatabase), and until +// then it no longer exists for lookups. A database whose reattach failed is +// dropped outright. func (dm *DatabaseManager) DropDatabase(name string) error { if name == SystemDatabaseName { return fmt.Errorf("cannot drop system database") @@ -351,12 +407,16 @@ func (dm *DatabaseManager) DropDatabase(name string) error { return fmt.Errorf("cannot drop default database") } - dm.mu.Lock() - defer dm.mu.Unlock() + dm.lifecycleMu.Lock() + defer dm.lifecycleMu.Unlock() - // Check if database exists - if not, return success (idempotent) + dm.mu.RLock() db, exists := dm.databases[name] - if !exists { + detached := dm.detached[name] + dm.mu.RUnlock() + + // Check if database exists - if not, return success (idempotent) + if (!exists && detached == nil) || (detached != nil && detached.dropped) { log.Info().Str("name", name).Msg("Database does not exist, DROP is no-op") return nil } @@ -369,6 +429,7 @@ func (dm *DatabaseManager) DropDatabase(name string) error { if err != nil { return fmt.Errorf("failed to get database path: %w", err) } + fullPath := filepath.Join(dm.dataDir, dbPath) // Remove from registry _, err = dm.systemDB.GetDB().Exec("DELETE FROM __marmot_databases WHERE name = ?", name) @@ -376,42 +437,75 @@ func (dm *DatabaseManager) DropDatabase(name string) error { return fmt.Errorf("failed to remove database from registry: %w", err) } - // Close database connection + // Remove this database's AUTO_INCREMENT claim rows (db/autoinc_claim.go). + // DROP TABLE deliberately does NOT reach this path and leaves its row in + // place; only DROP DATABASE removes claim rows. + _, err = dm.systemDB.GetDB().Exec("DELETE FROM "+AutoIncClaimTable+" WHERE db = ?", name) + if err != nil { + return fmt.Errorf("failed to remove auto-increment claims for database %s: %w", name, err) + } + + if detached != nil { + dm.mu.Lock() + if !detached.restoreFailed { + detached.dropped = true + detached.dropPath = fullPath + dm.mu.Unlock() + log.Info().Str("name", name).Msg("Database dropped; its files go when its snapshot restore ends") + return nil + } + delete(dm.detached, name) + dm.mu.Unlock() + if err := detached.metaStore.Close(); err != nil { + log.Error().Err(err).Str("name", name).Msg("Failed to close meta store") + } + removeDatabaseFiles(fullPath) + log.Info().Str("name", name).Msg("Database dropped") + return nil + } + + // Out of the map under mu, closed outside it (see DatabaseManager). + dm.mu.Lock() + delete(dm.databases, name) + dm.mu.Unlock() if err := db.Close(); err != nil { log.Error().Err(err).Str("name", name).Msg("Failed to close database") } + removeDatabaseFiles(fullPath) - // Delete from map - delete(dm.databases, name) + log.Info().Str("name", name).Msg("Database dropped") + return nil +} - // Delete database files - fullPath := filepath.Join(dm.dataDir, dbPath) +// removeDatabaseFiles deletes a closed database's SQLite file, its WAL and SHM +// files, and its meta store directory. +func removeDatabaseFiles(fullPath string) { if err := os.Remove(fullPath); err != nil && !os.IsNotExist(err) { log.Error().Err(err).Str("path", fullPath).Msg("Failed to delete database file") } - - // Delete WAL and SHM files os.Remove(fullPath + "-wal") os.Remove(fullPath + "-shm") - - // Delete meta database directory (PebbleDB) - metaPath := strings.TrimSuffix(fullPath, ".db") + "_meta.pebble" - os.RemoveAll(metaPath) - - log.Info().Str("name", name).Msg("Database dropped") - return nil + cleanupMetaStoreFiles(fullPath) } // GetDatabase returns a database by name func (dm *DatabaseManager) GetDatabase(name string) (*ReplicatedDatabase, error) { dm.mu.RLock() defer dm.mu.RUnlock() + return dm.getDatabaseLocked(name) +} +// getDatabaseLocked returns the database in service under name, or +// ErrDatabaseDetached while it is out of service for a restore. Caller must +// hold mu. +func (dm *DatabaseManager) getDatabaseLocked(name string) (*ReplicatedDatabase, error) { db, exists := dm.databases[name] if !exists { + if detached := dm.detached[name]; detached != nil && !detached.dropped { + return nil, fmt.Errorf("database %s: %w", name, ErrDatabaseDetached) + } return nil, fmt.Errorf("database %s does not exist", name) } - return db, nil } @@ -448,7 +542,8 @@ func (dm *DatabaseManager) DatabaseExists(name string) bool { defer dm.mu.RUnlock() _, exists := dm.databases[name] - return exists + detached := dm.detached[name] + return exists || (detached != nil && !detached.dropped) } // ListDatabases returns all database names @@ -474,19 +569,41 @@ func (dm *DatabaseManager) GetSystemDatabase() *ReplicatedDatabase { // Close closes all databases func (dm *DatabaseManager) Close() error { - dm.mu.Lock() - defer dm.mu.Unlock() + dm.lifecycleMu.Lock() + defer dm.lifecycleMu.Unlock() + + // Closed outside mu (see DatabaseManager). + dm.mu.RLock() + databases := make(map[string]*ReplicatedDatabase, len(dm.databases)) + for name, db := range dm.databases { + databases[name] = db + } + detached := make(map[string]*detachedDatabase, len(dm.detached)) + for name, d := range dm.detached { + detached[name] = d + } + dm.mu.RUnlock() var lastErr error // Close all user databases - for name, db := range dm.databases { + for name, db := range databases { + if name == SystemDatabaseName { + continue + } if err := db.Close(); err != nil { log.Error().Err(err).Str("name", name).Msg("Failed to close database") lastErr = err } } + for name, d := range detached { + if err := d.metaStore.Close(); err != nil { + log.Error().Err(err).Str("name", name).Msg("Failed to close detached database's meta store") + lastErr = err + } + } + // Close system database if err := dm.systemDB.Close(); err != nil { log.Error().Err(err).Msg("Failed to close system database") @@ -497,73 +614,166 @@ func (dm *DatabaseManager) Close() error { return lastErr } -// ReopenDatabase atomically swaps in a new database connection after snapshot apply. -// Uses "create-swap-close" pattern to avoid closing connections while queries are in-flight: -// 1. Create new database connection (before closing old one) -// 2. Atomically swap in the new connection (under lock) -// 3. Close old database after a delay (in goroutine, lets in-flight queries complete) -func (dm *DatabaseManager) ReopenDatabase(name string) error { - dm.mu.Lock() +// ErrDatabaseDetached is returned for a database taken out of service while a +// snapshot restore replaces its file (DetachDatabase). +var ErrDatabaseDetached = errors.New("database is detached for a snapshot restore") - // Get the existing database - oldDB, exists := dm.databases[name] - if !exists { - dm.mu.Unlock() - return fmt.Errorf("database %s does not exist", name) +// ErrDrainIncomplete is returned by DetachDatabase when the database was +// detached but a write in flight on it did not finish in time. +var ErrDrainIncomplete = errors.New("detached, but in-flight writes did not drain") + +// detachedDatabase is a user database out of service for a snapshot restore. +type detachedDatabase struct { + metaStore MetaStore // kept open across the restore for AttachDatabase + restoreFailed bool // AttachDatabase failed; the next restore of it may take it over + dropped bool // DROP DATABASE arrived during the restore; AttachDatabase completes it + dropPath string // the dropped database's file, removed by AttachDatabase +} + +// DetachDatabase takes a user database out of service so a snapshot restore +// can replace its SQLite file. It leaves the map at once, so lookups get +// ErrDatabaseDetached, and every commit on its connections is refused from +// then on (writeGate). It then drains: it waits, bounded by ctx, for the write +// transaction in flight on the database to finish, and stops its batch +// committer, GC and pools. When it returns nil, no write on the database can +// be ACKed until AttachDatabase: a caller still holding the old +// *ReplicatedDatabase gets a refused commit or "sql: database is closed", +// never a nil pool. The meta store stays open, so prepared transactions keep +// their intents, row locks and registration, and commit records are +// untouched. The system database and every other database are left alone. +// +// A database whose reattach failed (DatabasesAwaitingRestore) is already out +// of service, and the call hands it to the new restore. +// +// If the drain does not finish within ctx, DetachDatabase returns +// ErrDrainIncomplete with the database still detached: the caller must +// AttachDatabase it without replacing its file, because a commit that passed +// the gate before it closed may still complete into that file. Any other +// error leaves the database as it was. +func (dm *DatabaseManager) DetachDatabase(ctx context.Context, name string) error { + mdb, err := dm.takeOutOfService(name) + if err != nil || mdb == nil { + return err + } + // Drained outside both locks (see DatabaseManager): the detached entry + // fences Create and Drop. + if err := mdb.drainSQLite(ctx); err != nil { + return fmt.Errorf("database %s: %w: %w", name, ErrDrainIncomplete, err) } + return nil +} - // Get the path from system database before closing - var dbPath string - err := dm.systemDB.GetDB().QueryRow( - "SELECT path FROM __marmot_databases WHERE name = ?", name, - ).Scan(&dbPath) - if err != nil { - dm.mu.Unlock() - return fmt.Errorf("failed to get database path: %w", err) +// takeOutOfService is DetachDatabase's locked step. It returns the database +// to drain, or nil when a database whose reattach failed is handed to the new +// restore with nothing left to drain. +func (dm *DatabaseManager) takeOutOfService(name string) (*ReplicatedDatabase, error) { + if name == SystemDatabaseName { + return nil, fmt.Errorf("cannot detach the system database") } + dm.lifecycleMu.Lock() + defer dm.lifecycleMu.Unlock() + dm.mu.Lock() + defer dm.mu.Unlock() - fullPath := filepath.Join(dm.dataDir, dbPath) + if detached := dm.detached[name]; detached != nil { + if detached.restoreFailed && !detached.dropped { + detached.restoreFailed = false + return nil, nil + } + return nil, fmt.Errorf("database %s: %w", name, ErrDatabaseDetached) + } + mdb, exists := dm.databases[name] + if !exists { + return nil, fmt.Errorf("database %s does not exist", name) + } + // Commits are refused before the database leaves the map, so a caller + // that sees it detached cannot still commit through a copy it holds. + mdb.gate.close() + delete(dm.databases, name) + dm.detached[name] = &detachedDatabase{metaStore: mdb.GetMetaStore()} + return mdb, nil +} - // Get old MetaStore and close it FIRST to release PebbleDB lock - // This must happen before creating the new MetaStore - oldMetaStore := oldDB.GetMetaStore() +// AttachDatabase returns a database detached by DetachDatabase to service, +// opening whatever file is now at its path over the meta store it kept. If +// that fails, the database stays out of service and is listed by +// DatabasesAwaitingRestore until a later restore of it succeeds. If the +// database was dropped during the restore, the drop is completed instead. +func (dm *DatabaseManager) AttachDatabase(name string) error { + dm.lifecycleMu.Lock() + defer dm.lifecycleMu.Unlock() - // Close old database and MetaStore synchronously (we hold the mutex, so no new queries can start) - log.Info().Str("name", name).Msg("Closing old database for snapshot reload") - if err := oldDB.Close(); err != nil { - log.Warn().Err(err).Str("name", name).Msg("Error closing old database connection") + dm.mu.RLock() + detached := dm.detached[name] + dm.mu.RUnlock() + if detached == nil { + return fmt.Errorf("database %s is not detached", name) } - if oldMetaStore != nil { - if err := oldMetaStore.Close(); err != nil { - dm.mu.Unlock() - return fmt.Errorf("failed to close old meta store for %s: %w", name, err) + if detached.dropped { + dm.mu.Lock() + delete(dm.detached, name) + dm.mu.Unlock() + if err := detached.metaStore.Close(); err != nil { + log.Error().Err(err).Str("name", name).Msg("Failed to close dropped database's meta store") } + removeDatabaseFiles(detached.dropPath) + log.Info().Str("name", name).Msg("Database dropped during its snapshot restore; drop completed") + return nil } - // Now create new MetaStore (PebbleDB lock is released) - metaStore, err := NewMetaStore(fullPath) + mdb, err := dm.openDetached(name, detached.metaStore) if err != nil { + dm.mu.Lock() + detached.restoreFailed = true dm.mu.Unlock() - return fmt.Errorf("failed to create meta store for %s: %w", name, err) + return fmt.Errorf("failed to reattach database %s: %w", name, err) } + dm.mu.Lock() + dm.wireGCCoordination(mdb, name) + delete(dm.detached, name) + dm.databases[name] = mdb + dm.mu.Unlock() + log.Info().Str("name", name).Msg("Database reattached after snapshot restore") + return nil +} - // Create new database connection - newDB, err := NewReplicatedDatabase(fullPath, dm.nodeID, dm.clock, metaStore) - if err != nil { - metaStore.Close() - dm.mu.Unlock() - return fmt.Errorf("failed to reopen database %s: %w", name, err) +// openDetached opens the file registered for a detached database over the +// meta store it kept. +func (dm *DatabaseManager) openDetached(name string, metaStore MetaStore) (*ReplicatedDatabase, error) { + var dbPath string + if err := dm.systemDB.GetDB().QueryRow( + "SELECT path FROM __marmot_databases WHERE name = ?", name, + ).Scan(&dbPath); err != nil { + return nil, fmt.Errorf("failed to get database path: %w", err) } + return NewReplicatedDatabase(filepath.Join(dm.dataDir, dbPath), dm.nodeID, dm.clock, metaStore) +} - // Wire up GC coordination for new connection - dm.wireGCCoordination(newDB, name) - - // Atomically swap in the new connection - dm.databases[name] = newDB - dm.mu.Unlock() - - log.Info().Str("name", name).Msg("Database connection swapped after snapshot") +// DatabasesAwaitingRestore lists the databases whose reattach after a +// snapshot restore failed. They stay out of service until a restore of them +// succeeds; anti-entropy retries one each round. +func (dm *DatabaseManager) DatabasesAwaitingRestore() []string { + dm.mu.RLock() + defer dm.mu.RUnlock() + var names []string + for name, detached := range dm.detached { + if detached.restoreFailed && !detached.dropped { + names = append(names, name) + } + } + return names +} +// errIfDetachedLocked refuses a snapshot of every database while one the +// registry names is detached: the snapshot would ship a registry naming a +// database it has no file for. The caller retries once it is back. Caller +// must hold mu. +func (dm *DatabaseManager) errIfDetachedLocked() error { + for name, detached := range dm.detached { + if !detached.dropped { + return fmt.Errorf("cannot snapshot while database %s is out of service: %w", name, ErrDatabaseDetached) + } + } return nil } @@ -632,8 +842,8 @@ func (dm *DatabaseManager) GetDatabasePath(name string) (string, error) { // and imports them into the database manager. This is used on first startup // of a seed node to make existing databases available. func (dm *DatabaseManager) ImportExistingDatabases(importDir string) (int, error) { - dm.mu.Lock() - defer dm.mu.Unlock() + dm.lifecycleMu.Lock() + defer dm.lifecycleMu.Unlock() if importDir == "" { return 0, nil @@ -683,7 +893,10 @@ func (dm *DatabaseManager) ImportExistingDatabases(importDir string) (int, error dbName := strings.TrimSuffix(name, ".db") // Check if database already exists - if existingDB, exists := dm.databases[dbName]; exists { + dm.mu.RLock() + existingDB, exists := dm.databases[dbName] + dm.mu.RUnlock() + if exists { // Check if existing database is empty (no user tables) // If so, we can replace it with the imported version srcPath := filepath.Join(importDir, name) @@ -691,10 +904,13 @@ func (dm *DatabaseManager) ImportExistingDatabases(importDir string) (int, error log.Debug().Str("name", dbName).Msg("Database already exists with data, skipping import") continue } - // Close existing empty database before replacing + // Close existing empty database before replacing: out of the map + // under mu, closed outside it (see DatabaseManager). log.Info().Str("name", dbName).Msg("Replacing empty database with imported version") - existingDB.Close() + dm.mu.Lock() delete(dm.databases, dbName) + dm.mu.Unlock() + existingDB.Close() } // Copy database file to databases directory @@ -732,9 +948,6 @@ func (dm *DatabaseManager) ImportExistingDatabases(importDir string) (int, error continue } - // Wire up GC coordination for imported database - dm.wireGCCoordination(db, dbName) - // Register in system database createdAt := time.Now().UnixNano() relPath := filepath.Join("databases", name) @@ -748,7 +961,12 @@ func (dm *DatabaseManager) ImportExistingDatabases(importDir string) (int, error continue } + // Wire up GC coordination only once the database is registered: until + // then its GC cannot reach mu, so the failure path above can close it. + dm.mu.Lock() + dm.wireGCCoordination(db, dbName) dm.databases[dbName] = db + dm.mu.Unlock() imported++ log.Info().Str("name", dbName).Str("src", srcPath).Msg("Imported existing database") } @@ -848,6 +1066,10 @@ func (dm *DatabaseManager) TakeSnapshot() ([]SnapshotInfo, uint64, error) { dm.mu.RLock() defer dm.mu.RUnlock() + if err := dm.errIfDetachedLocked(); err != nil { + return nil, 0, err + } + var snapshots []SnapshotInfo // Checkpoint and get info for system database @@ -886,39 +1108,17 @@ func (dm *DatabaseManager) TakeSnapshot() ([]SnapshotInfo, uint64, error) { continue } - if err := dm.checkpointDatabase(db); err != nil { - log.Warn().Err(err).Str("database", name).Msg("Failed to checkpoint database") - continue - } - - dbPath := filepath.Join(dm.dataDir, "databases", name+".db") - info, err := os.Stat(dbPath) + snapshot, err := dm.describeDatabaseFile(name, db) if err != nil { - log.Warn().Err(err).Str("database", name).Msg("Failed to stat database") + log.Warn().Err(err).Str("database", name).Msg("Failed to describe database for snapshot") continue } - - dbSHA256, err := calculateFileSHA256(dbPath) - if err != nil { - log.Warn().Err(err).Str("database", name).Msg("Failed to hash database") - continue - } - - snapshots = append(snapshots, SnapshotInfo{ - Name: name, - Filename: filepath.Join("databases", name+".db"), - FullPath: dbPath, - Size: info.Size(), - SHA256: dbSHA256, - }) + snapshots = append(snapshots, snapshot) // MetaStore (PebbleDB) is NOT included - see note above } - // Get max committed transaction ID - maxTxnID, err := dm.GetMaxCommittedTxnID() - if err != nil { - return nil, 0, fmt.Errorf("failed to get max txn id: %w", err) - } + // Get max committed transaction ID (without lock since we already hold it) + maxTxnID := dm.getMaxCommittedTxnIDLocked() log.Info(). Int("databases", len(snapshots)). @@ -928,6 +1128,54 @@ func (dm *DatabaseManager) TakeSnapshot() ([]SnapshotInfo, uint64, error) { return snapshots, maxTxnID, nil } +// TakeDatabaseSnapshotInfo checkpoints one user database and describes its +// file, with its max transaction ID, as TakeSnapshot does for every database. +// Unlike TakeSnapshot it is refused only while this database is out of +// service, never because another one is. +func (dm *DatabaseManager) TakeDatabaseSnapshotInfo(name string) (SnapshotInfo, uint64, error) { + dm.mu.RLock() + defer dm.mu.RUnlock() + + db, err := dm.getDatabaseLocked(name) + if err != nil { + return SnapshotInfo{}, 0, err + } + snapshot, err := dm.describeDatabaseFile(name, db) + if err != nil { + return SnapshotInfo{}, 0, err + } + maxTxnID, err := dm.getMaxTxnIDLocked(name) + if err != nil { + return SnapshotInfo{}, 0, fmt.Errorf("failed to get max txn id: %w", err) + } + return snapshot, maxTxnID, nil +} + +// describeDatabaseFile checkpoints user database name and describes its file +// in place. Caller must hold mu. +func (dm *DatabaseManager) describeDatabaseFile(name string, db *ReplicatedDatabase) (SnapshotInfo, error) { + if err := dm.checkpointDatabase(db); err != nil { + return SnapshotInfo{}, fmt.Errorf("failed to checkpoint database %s: %w", name, err) + } + filename := filepath.Join("databases", name+".db") + dbPath := filepath.Join(dm.dataDir, filename) + info, err := os.Stat(dbPath) + if err != nil { + return SnapshotInfo{}, fmt.Errorf("failed to stat database %s: %w", name, err) + } + dbSHA256, err := calculateFileSHA256(dbPath) + if err != nil { + return SnapshotInfo{}, fmt.Errorf("failed to hash database %s: %w", name, err) + } + return SnapshotInfo{ + Name: name, + Filename: filename, + FullPath: dbPath, + Size: info.Size(), + SHA256: dbSHA256, + }, nil +} + // checkpointDatabase forces a WAL checkpoint to ensure data is in the main database file func (dm *DatabaseManager) checkpointDatabase(db *ReplicatedDatabase) error { _, err := db.GetDB().Exec("PRAGMA wal_checkpoint(TRUNCATE)") @@ -966,6 +1214,10 @@ func (dm *DatabaseManager) TakeSnapshotToDir(targetDir string) ([]SnapshotInfo, dm.mu.Lock() defer dm.mu.Unlock() + if err := dm.errIfDetachedLocked(); err != nil { + return nil, 0, nil, err + } + var snapshots []SnapshotInfo // Checkpoint and copy system database @@ -1101,10 +1353,11 @@ func (dm *DatabaseManager) TakeSnapshotForDatabase( dm.mu.RLock() defer dm.mu.RUnlock() - // Get database handle - db, exists := dm.databases[dbName] - if !exists { - return SnapshotInfo{}, 0, fmt.Errorf("database %s not found", dbName) + // Only this database's own detach refuses its snapshot; other databases + // out of service are not part of it. + db, err := dm.getDatabaseLocked(dbName) + if err != nil { + return SnapshotInfo{}, 0, err } // Checkpoint the database (PRAGMA wal_checkpoint(TRUNCATE)) @@ -1138,8 +1391,8 @@ func (dm *DatabaseManager) TakeSnapshotForDatabase( return SnapshotInfo{}, 0, fmt.Errorf("failed to calculate checksum: %w", err) } - // Get max txn ID for this specific database - maxTxnID, err := dm.GetMaxTxnID(dbName) + // Get max txn ID for this specific database (without lock since we already hold it) + maxTxnID, err := dm.getMaxTxnIDLocked(dbName) if err != nil { return SnapshotInfo{}, 0, fmt.Errorf("failed to get max txn id: %w", err) } @@ -1326,7 +1579,12 @@ func (dm *DatabaseManager) GetMinAppliedTxnID(database string) (uint64, error) { func (dm *DatabaseManager) GetMaxTxnID(database string) (uint64, error) { dm.mu.RLock() defer dm.mu.RUnlock() + return dm.getMaxTxnIDLocked(database) +} +// getMaxTxnIDLocked is GetMaxTxnID for a caller that holds dm.mu: taking the +// read lock again would deadlock behind a writer waiting for it. +func (dm *DatabaseManager) getMaxTxnIDLocked(database string) (uint64, error) { db, ok := dm.databases[database] if !ok { return 0, fmt.Errorf("database %s not found", database) diff --git a/db/database_manager_cdc_test.go b/db/database_manager_cdc_test.go index 55cbb286..e5c987a0 100644 --- a/db/database_manager_cdc_test.go +++ b/db/database_manager_cdc_test.go @@ -1,6 +1,7 @@ package db import ( + "context" "os" "testing" @@ -131,7 +132,7 @@ func TestDatabaseManager_CDCHubNilSafe(t *testing.T) { assert.NotNil(t, db) } -func TestDatabaseManager_ReopenDatabasePreservesCDCHub(t *testing.T) { +func TestDatabaseManager_ReattachPreservesCDCHub(t *testing.T) { // Create temp directory for test tmpDir, err := os.MkdirTemp("", "marmot-test-*") require.NoError(t, err) @@ -151,9 +152,9 @@ func TestDatabaseManager_ReopenDatabasePreservesCDCHub(t *testing.T) { err = dm.CreateDatabase("test_db") require.NoError(t, err) - // Reopen the database (simulates snapshot reload) - err = dm.ReopenDatabase("test_db") - require.NoError(t, err) + // Detach and reattach the database, as a snapshot restore does + require.NoError(t, dm.DetachDatabase(context.Background(), "test_db")) + require.NoError(t, dm.AttachDatabase("test_db")) // Verify CDC hub is still set assert.Equal(t, hub, dm.GetCDCHub()) diff --git a/db/database_manager_test.go b/db/database_manager_test.go index c53d64a3..6b2def64 100644 --- a/db/database_manager_test.go +++ b/db/database_manager_test.go @@ -537,7 +537,7 @@ func TestTakeSnapshotForDatabaseNotFound(t *testing.T) { t.Fatal("Expected error for non-existent database, got nil") } - expectedErr := "database nonexistent not found" + expectedErr := "database nonexistent does not exist" if !strings.Contains(err.Error(), expectedErr) { t.Errorf("Expected error containing '%s', got '%s'", expectedErr, err.Error()) } diff --git a/db/db_integration.go b/db/db_integration.go index 2cbe5916..2d0257b7 100644 --- a/db/db_integration.go +++ b/db/db_integration.go @@ -3,6 +3,7 @@ package db import ( "context" "database/sql" + "errors" "fmt" "strings" "time" @@ -35,21 +36,55 @@ type ReplicatedDatabase struct { replicationFn ReplicationFunc batchCommitter *SQLiteBatchCommitter schemaCache *SchemaCache // Shared schema cache for preupdate hooks + gate *writeGate // refuses every commit once the database leaves service } // ReplicationFunc is called to replicate transactions to other nodes // This is injected from the coordinator layer type ReplicationFunc func(ctx context.Context, txn *Transaction) error +// ReplicatedDatabaseOption configures NewReplicatedDatabase. +type ReplicatedDatabaseOption func(*replicatedDatabaseOptions) + +type replicatedDatabaseOptions struct { + driverName string + synchronous string + batchCommit bool +} + +// WithDurableCommits makes every commit on the database survive an OS crash +// or power loss, not only a process crash. Every connection it opens runs +// synchronous=FULL, carried in the DSN because the driver resets the mode on +// each new connection, through SQLiteDurableDriverName. It gets no batch +// committer, whose connection would commit and checkpoint the same file at +// synchronous=NORMAL. +func WithDurableCommits() ReplicatedDatabaseOption { + return func(o *replicatedDatabaseOptions) { + o.driverName = SQLiteDurableDriverName + o.synchronous = "FULL" + o.batchCommit = false + } +} + // NewReplicatedDatabase creates a new transaction-enabled database // metaStore is the MetaStore for storing transaction metadata (intent entries, txn records, etc.) -func NewReplicatedDatabase(dbPath string, nodeID uint64, clock *hlc.Clock, metaStore MetaStore) (*ReplicatedDatabase, error) { +func NewReplicatedDatabase(dbPath string, nodeID uint64, clock *hlc.Clock, metaStore MetaStore, opts ...ReplicatedDatabaseOption) (*ReplicatedDatabase, error) { + o := replicatedDatabaseOptions{ + driverName: SQLiteDriverName, + synchronous: "NORMAL", + batchCommit: cfg.Config.BatchCommit.Enabled, + } + for _, opt := range opts { + opt(&o) + } + // Get timeout from config (LockWaitTimeoutSeconds is in seconds, SQLite needs milliseconds) busyTimeoutMS := cfg.Config.Transaction.LockWaitTimeoutSeconds * 1000 poolCfg := cfg.Config.ConnectionPool isMemoryDB := strings.Contains(dbPath, ":memory:") var writeDB, hookDB, readDB *sql.DB + gate := &writeGate{} // Helper to close all opened connections on error closeAll := func() { @@ -71,14 +106,14 @@ func NewReplicatedDatabase(dbPath string, nodeID uint64, clock *hlc.Clock, metaS writeDSN := dbPath if !isMemoryDB { if strings.Contains(writeDSN, "?") { - writeDSN += fmt.Sprintf("&_journal_mode=WAL&_busy_timeout=%d&_txlock=immediate&cache=shared", busyTimeoutMS) + writeDSN += fmt.Sprintf("&_journal_mode=WAL&_busy_timeout=%d&_txlock=immediate&_sync=%s&cache=shared", busyTimeoutMS, o.synchronous) } else { - writeDSN += fmt.Sprintf("?_journal_mode=WAL&_busy_timeout=%d&_txlock=immediate&cache=shared", busyTimeoutMS) + writeDSN += fmt.Sprintf("?_journal_mode=WAL&_busy_timeout=%d&_txlock=immediate&_sync=%s&cache=shared", busyTimeoutMS, o.synchronous) } } var err error - writeDB, err = sql.Open(SQLiteDriverName, writeDSN) + writeDB, err = gate.openDB(o.driverName, writeDSN) if err != nil { return nil, fmt.Errorf("failed to open write database: %w", err) } @@ -93,7 +128,7 @@ func NewReplicatedDatabase(dbPath string, nodeID uint64, clock *hlc.Clock, metaS // This connection is acquired during ExecuteLocalWithHooks, captures CDC, // then releases BEFORE 2PC broadcast - avoiding deadlock with incoming commits. hookDSN := writeDSN // Same settings as write connection - hookDB, err = sql.Open(SQLiteDriverName, hookDSN) + hookDB, err = gate.openDB(o.driverName, hookDSN) if err != nil { closeAll() return nil, fmt.Errorf("failed to open hook database: %w", err) @@ -110,13 +145,13 @@ func NewReplicatedDatabase(dbPath string, nodeID uint64, clock *hlc.Clock, metaS readDSN := dbPath if !isMemoryDB { if strings.Contains(readDSN, "?") { - readDSN += fmt.Sprintf("&_journal_mode=WAL&_busy_timeout=%d&cache=shared", busyTimeoutMS) + readDSN += fmt.Sprintf("&_journal_mode=WAL&_busy_timeout=%d&_sync=%s&cache=shared", busyTimeoutMS, o.synchronous) } else { - readDSN += fmt.Sprintf("?_journal_mode=WAL&_busy_timeout=%d&cache=shared", busyTimeoutMS) + readDSN += fmt.Sprintf("?_journal_mode=WAL&_busy_timeout=%d&_sync=%s&cache=shared", busyTimeoutMS, o.synchronous) } } - readDB, err = sql.Open(SQLiteDriverName, readDSN) + readDB, err = gate.openDB(o.driverName, readDSN) if err != nil { closeAll() return nil, fmt.Errorf("failed to open read database: %w", err) @@ -143,10 +178,6 @@ func NewReplicatedDatabase(dbPath string, nodeID uint64, clock *hlc.Clock, metaS closeAll() return nil, fmt.Errorf("failed to set busy timeout: %w", err) } - if _, err = db.Exec("PRAGMA synchronous=NORMAL"); err != nil { - closeAll() - return nil, fmt.Errorf("failed to set synchronous mode: %w", err) - } if _, err = db.Exec("PRAGMA cache_size=-64000"); err != nil { closeAll() return nil, fmt.Errorf("failed to set cache size: %w", err) @@ -184,7 +215,7 @@ func NewReplicatedDatabase(dbPath string, nodeID uint64, clock *hlc.Clock, metaS // Create batch committer for SQLite-level batching (opens its own optimized connection) var batchCommitter *SQLiteBatchCommitter - if cfg.Config.BatchCommit.Enabled { + if o.batchCommit { batchCommitter = NewSQLiteBatchCommitter( dbPath, cfg.Config.BatchCommit.MaxBatchSize, @@ -197,6 +228,7 @@ func NewReplicatedDatabase(dbPath string, nodeID uint64, clock *hlc.Clock, metaS cfg.Config.BatchCommit.IncrementalVacuumPages, cfg.Config.BatchCommit.IncrementalVacuumTimeLimitMS, ) + batchCommitter.gate = gate if err := batchCommitter.Start(); err != nil { closeAll() return nil, fmt.Errorf("failed to start batch committer: %w", err) @@ -213,6 +245,7 @@ func NewReplicatedDatabase(dbPath string, nodeID uint64, clock *hlc.Clock, metaS nodeID: nodeID, batchCommitter: batchCommitter, schemaCache: schemaCache, + gate: gate, } // Wire batch committer to transaction manager @@ -296,46 +329,83 @@ func (mdb *ReplicatedDatabase) GetClock() *hlc.Clock { return mdb.clock } -// Close closes the database connections, MetaStore, and stops GC. -// Order is important: stop GC first, then close connections. +// Close closes the database connections and the MetaStore and stops GC +// (closeSQLite). It does not wait for a write transaction in flight: that +// transaction's commit is refused (see writeGate). func (mdb *ReplicatedDatabase) Close() error { - // Stop GC goroutine first to prevent it from accessing closed connections - if mdb.txnMgr != nil { - mdb.txnMgr.StopGarbageCollection() + err := mdb.closeSQLite() + if mdb.metaStore != nil { + if msErr := mdb.metaStore.Close(); err == nil { + err = msErr + } } + return err +} - // Stop batch committer (flushes pending) +// closeSQLite closes the database's SQLite file for good, leaving the meta +// store open: the batch committer stops after committing what is queued, no +// later commit on any of the database's connections succeeds (writeGate), the +// GC stops, and the pools close. The pool fields keep their closed *sql.DB, so +// a caller still holding the database gets "sql: database is closed", never a +// nil pool. +// +// The caller must not hold DatabaseManager.mu: stopping the GC waits for a +// pass that may itself be waiting on that lock. +func (mdb *ReplicatedDatabase) closeSQLite() error { if mdb.batchCommitter != nil { mdb.batchCommitter.Stop() } - - var errs []error - - if mdb.writeDB != nil { - if err := mdb.writeDB.Close(); err != nil { - errs = append(errs, err) - } + mdb.gate.close() + return mdb.closePools() +} + +// drainSQLite takes the database's SQLite file out of service for a restore +// that will replace it, leaving the meta store open. Unlike closeSQLite it +// refuses first: the gate closes before the batch committer stops, so the +// commits it has queued are refused rather than written into the file being +// replaced (their transactions stay prepared). It then waits, bounded by ctx, +// for the write transaction in flight on writeDB to finish: when it returns +// nil, every commit that passed the gate before it closed has completed, and +// none can complete later. If ctx ends first it returns that error, still +// tearing everything down; a commit already past the gate may then complete +// later. A failure to close a pool is logged, not returned: it says nothing +// about what the database's file holds. +// +// The caller must not hold DatabaseManager.mu (see closeSQLite). +func (mdb *ReplicatedDatabase) drainSQLite(ctx context.Context) error { + mdb.gate.close() + if mdb.batchCommitter != nil { + mdb.batchCommitter.Stop() } - if mdb.hookDB != nil { - if err := mdb.hookDB.Close(); err != nil { - errs = append(errs, err) - } + // writeDB has a single connection: getting it means the transaction that + // held it has committed or rolled back. Holding it through the pool's + // Close keeps every other caller off it. + held, err := mdb.writeDB.Conn(ctx) + if closeErr := mdb.closePools(); closeErr != nil { + log.Warn().Err(closeErr).Msg("Error closing a drained database's SQLite pools") } - if mdb.readDB != nil { - if err := mdb.readDB.Close(); err != nil { - errs = append(errs, err) - } + if err != nil { + return fmt.Errorf("drain in-flight writes: %w", err) } - if mdb.metaStore != nil { - if err := mdb.metaStore.Close(); err != nil { + _ = held.Close() + return nil +} + +// closePools stops the GC and closes the SQLite pools. +func (mdb *ReplicatedDatabase) closePools() error { + if mdb.txnMgr != nil { + mdb.txnMgr.StopGarbageCollection() + } + var errs []error + for _, pool := range []*sql.DB{mdb.writeDB, mdb.hookDB, mdb.readDB} { + if pool == nil { + continue + } + if err := pool.Close(); err != nil { errs = append(errs, err) } } - - if len(errs) > 0 { - return errs[0] - } - return nil + return errors.Join(errs...) } // CloseSQLiteConnections closes all SQLite connections synchronously. @@ -359,6 +429,8 @@ func (mdb *ReplicatedDatabase) CloseSQLiteConnections() { // OpenSQLiteConnections opens new SQLite connections to the database file. // This MUST be called AFTER database files have been replaced during snapshot apply. +// Like the pools NewReplicatedDatabase opens, they are gated: once the +// database leaves service, their commits are refused. // Note: This does NOT touch MetaStore (PebbleDB) - only SQLite connections. func (mdb *ReplicatedDatabase) OpenSQLiteConnections(dbPath string) error { busyTimeoutMS := cfg.Config.Transaction.LockWaitTimeoutSeconds * 1000 @@ -366,7 +438,7 @@ func (mdb *ReplicatedDatabase) OpenSQLiteConnections(dbPath string) error { // Open write connection writeDSN := fmt.Sprintf("%s?_journal_mode=WAL&_busy_timeout=%d&_txlock=immediate&cache=shared", dbPath, busyTimeoutMS) - writeDB, err := sql.Open(SQLiteDriverName, writeDSN) + writeDB, err := mdb.gate.openDB(SQLiteDriverName, writeDSN) if err != nil { return fmt.Errorf("failed to open write connection: %w", err) } @@ -381,7 +453,7 @@ func (mdb *ReplicatedDatabase) OpenSQLiteConnections(dbPath string) error { } // Open hook connection - hookDB, err := sql.Open(SQLiteDriverName, writeDSN) + hookDB, err := mdb.gate.openDB(SQLiteDriverName, writeDSN) if err != nil { writeDB.Close() return fmt.Errorf("failed to open hook connection: %w", err) @@ -392,7 +464,7 @@ func (mdb *ReplicatedDatabase) OpenSQLiteConnections(dbPath string) error { // Open read connection pool readDSN := fmt.Sprintf("%s?_journal_mode=WAL&_busy_timeout=%d&cache=shared", dbPath, busyTimeoutMS) - readDB, err := sql.Open(SQLiteDriverName, readDSN) + readDB, err := mdb.gate.openDB(SQLiteDriverName, readDSN) if err != nil { writeDB.Close() hookDB.Close() diff --git a/db/ddl_prepare_validate.go b/db/ddl_prepare_validate.go index f240dd70..9cf1b310 100644 --- a/db/ddl_prepare_validate.go +++ b/db/ddl_prepare_validate.go @@ -7,6 +7,8 @@ import ( "fmt" "github.com/mattn/go-sqlite3" + "github.com/maxpert/marmot/protocol/mysqlcode" + "github.com/maxpert/marmot/protocol/query/transform" ) // isContextError reports whether err was caused by the context being cancelled or @@ -33,11 +35,47 @@ var transientSQLiteCodes = map[sqlite3.ErrNo]bool{ sqlite3.ErrReadonly: true, // SQLITE_READONLY: database (or this connection) is read-only right now } +// AutoIncWidthExceededError reports that MAX(id) in an existing table exceeds +// the range its explicitly declared AUTO_INCREMENT column can hold - e.g. an +// ALTER that narrows the column to TINYINT after 200 rows already exist. +// +// It is raised only after DDL validation has re-derived the column's marker +// from THIS node's own (in-transaction) sqlite_master, so - unlike a raw +// SQLite runtime error - it carries no ambiguity about transient node state: +// it is always a deterministic verdict on the statement, and isDDLRejection +// treats it as an unconditional rejection. +type AutoIncWidthExceededError struct { + Table string + Column string + Max uint64 + WidthMax uint64 +} + +func (e *AutoIncWidthExceededError) Error() string { + return fmt.Sprintf( + "table %s: existing max value %d of AUTO_INCREMENT column %s exceeds its declared width ceiling %d", + e.Table, e.Max, e.Column, e.WidthMax) +} + +// Unwrap exposes this rejection's MySQL error code (ER_WARN_DATA_OUT_OF_RANGE) +// through *transform.CodedError, the same coded-error mechanism a +// transformation rule uses to choose its own client-facing code. +// protocol.ConvertToMySQLError already recognises *transform.CodedError via +// errors.As, so this alone is enough for the width-ceiling rejection to reach +// the client with the right code - no separate mapping mechanism is needed, +// and protocol.ConvertToMySQLError never has to reference this db type by +// name, which would create an import cycle (db already imports protocol; +// protocol must not import db). +func (e *AutoIncWidthExceededError) Unwrap() error { + return &transform.CodedError{Code: mysqlcode.ErrCodeDataOutOfRange, Message: e.Error()} +} + // isDDLRejection reports whether err is a deterministic verdict on the DDL // statement itself - SQLite refuses to ever apply it, such as a constraint // violation or a schema conflict (duplicate column, missing table, syntax -// error) - rather than a transient condition on this node such as lock -// contention, a resource limit, or context cancellation. +// error), or this node's own width-ceiling check found existing rows the +// declared column cannot hold - rather than a transient condition on this +// node such as lock contention, a resource limit, or context cancellation. // // Only a rejection may be surfaced to the coordinator as a final refusal of // the transaction; everything else must stay a retryable missing ACK, exactly @@ -47,6 +85,11 @@ func isDDLRejection(ctx context.Context, err error) bool { return false } + var widthErr *AutoIncWidthExceededError + if errors.As(err, &widthErr) { + return true + } + var sqliteErr sqlite3.Error if !errors.As(err, &sqliteErr) { // Not a typed SQLite error - e.g. a wrapped BeginTx failure from @@ -58,6 +101,22 @@ func isDDLRejection(ctx context.Context, err error) bool { return !transientSQLiteCodes[sqliteErr.Code] } +// mysqlCodeForError returns the MySQL server error code an error names, or 0 +// when it names none. +// +// A rejection crosses to the coordinator as a string (PrepareResult.Error), so +// the typed error is gone by the time anything maps it to a client-visible +// code. Reading the code here, where the typed error still exists, is what +// lets a participant's deterministic refusal keep its own code instead of +// being flattened to ER_UNKNOWN_ERROR. +func mysqlCodeForError(err error) uint16 { + var coded *transform.CodedError + if errors.As(err, &coded) { + return coded.Code + } + return 0 +} + // ValidateDDLStatements verifies that DDL can be applied to this node before the // participant ACKs PREPARE. // @@ -86,6 +145,11 @@ func ValidateDDLStatements(ctx context.Context, dbConn *sql.DB, statements []str // Always discard the validation transaction - it exists only to surface errors. defer func() { _ = tx.Rollback() }() + before, err := tableSchemaSnapshot(ctx, tx) + if err != nil { + return err + } + for _, stmt := range statements { if stmt == "" { continue @@ -95,5 +159,88 @@ func ValidateDDLStatements(ctx context.Context, dbConn *sql.DB, statements []str } } + after, err := tableSchemaSnapshot(ctx, tx) + if err != nil { + return err + } + + // Refuse the DDL when it leaves an explicitly declared AUTO_INCREMENT + // column unable to hold ids the table already contains. Scoped to tables whose sqlite_master row this DDL actually changed + // - derived from the schema itself, not by parsing the statement text - so + // a CREATE TABLE for an unrelated table is never rejected over a + // pre-existing condition on some other table it never touched. + return checkAutoIncWidthCeilings(ctx, tx, changedTables(before, after)) +} + +// tableSchemaSnapshot reads every table's CREATE TABLE text from +// sqlite_master, keyed by name. +func tableSchemaSnapshot(ctx context.Context, tx *sql.Tx) (map[string]string, error) { + rows, err := tx.QueryContext(ctx, "SELECT name, sql FROM sqlite_master WHERE type = 'table'") + if err != nil { + return nil, fmt.Errorf("snapshot sqlite_master: %w", err) + } + defer rows.Close() + + snapshot := make(map[string]string) + for rows.Next() { + var name string + var createSQL sql.NullString + if err := rows.Scan(&name, &createSQL); err != nil { + return nil, fmt.Errorf("scan sqlite_master row: %w", err) + } + snapshot[name] = createSQL.String + } + return snapshot, rows.Err() +} + +// changedTables returns the names of tables in `after` whose CREATE TABLE +// text differs from (or is absent from) `before` - the set of tables this +// validation's DDL actually touched, derived from the schema rather than the +// statement text so it covers CREATE TABLE and every ALTER shape uniformly. +func changedTables(before, after map[string]string) []string { + var touched []string + for name, createSQL := range after { + if before[name] != createSQL { + touched = append(touched, name) + } + } + return touched +} + +// checkAutoIncWidthCeilings rejects the DDL when a table's explicitly +// declared AUTO_INCREMENT column can no longer hold the ids the table already +// contains. It runs inside the validation transaction, so it sees the NEW +// schema (this DDL's own effect) against the OLD rows the DDL has not +// touched, and is always rolled back with the rest of validation. +func checkAutoIncWidthCeilings(ctx context.Context, tx *sql.Tx, tables []string) error { + for _, table := range tables { + col, attrs, ok, err := autoIncMarkedColumn(tx, table) + if err != nil { + return fmt.Errorf("derive auto-increment column for %s: %w", table, err) + } + if !ok { + continue + } + + var max sql.NullInt64 + q := fmt.Sprintf("SELECT MAX(%s) FROM %s", quoteIdent(col), quoteIdent(table)) + if err := tx.QueryRowContext(ctx, q).Scan(&max); err != nil { + return fmt.Errorf("compute existing max for %s.%s: %w", table, col, err) + } + if !max.Valid || max.Int64 < 0 { + // No rows, or every id is negative: negative ids are never issued + // by the allocator (it only ever advances upward from a + // non-negative base), so they carry nothing for this ceiling to + // check against. + continue + } + + widthMax := attrs.WidthMax() + if uint64(max.Int64) > widthMax { + return &AutoIncWidthExceededError{ + Table: table, Column: col, Max: uint64(max.Int64), WidthMax: widthMax, + } + } + } return nil } diff --git a/db/ddl_prepare_validate_autoinc_code_test.go b/db/ddl_prepare_validate_autoinc_code_test.go new file mode 100644 index 00000000..abd90775 --- /dev/null +++ b/db/ddl_prepare_validate_autoinc_code_test.go @@ -0,0 +1,100 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "context" + "testing" + + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/maxpert/marmot/protocol/mysqlcode" + "github.com/stretchr/testify/require" +) + +// TestAutoIncWidthExceededError_ConvertsToDataOutOfRange pins that an +// *AutoIncWidthExceededError - the PREPARE-time refusal of DDL that would +// leave an AUTO_INCREMENT column unable to hold ids the table already +// contains - reaches a client-facing caller as MySQL's +// ER_WARN_DATA_OUT_OF_RANGE (1264, SQLSTATE 22003), through the exact same +// conversion protocol.ConvertToMySQLError performs for every other error +// surfaced to a client. +// +// Mutation: remove AutoIncWidthExceededError's coded-error exposure (Unwrap); +// ConvertToMySQLError falls through to the message-based default +// (ER_UNKNOWN_ERROR, HY000) and the Code assertion below fires. +func TestAutoIncWidthExceededError_ConvertsToDataOutOfRange(t *testing.T) { + t.Parallel() + + widthErr := &AutoIncWidthExceededError{ + Table: "narrow", + Column: "id", + Max: 200, + WidthMax: 127, + } + + mysqlErr := protocol.ConvertToMySQLError(widthErr) + require.Equal(t, mysqlcode.ErrCodeDataOutOfRange, mysqlErr.Code, + "a width-ceiling rejection must reach the client as ER_WARN_DATA_OUT_OF_RANGE") + require.Equal(t, mysqlcode.SQLStateDataOutOfRange, mysqlErr.SQLState) + + require.True(t, isDDLRejection(context.Background(), widthErr), + "a width-ceiling violation must still be classified as a deterministic rejection") +} + +// TestPrepareRejectionCarriesTheMySQLCode pins the hop the test above cannot +// see. A participant's rejection crosses to the coordinator as a STRING +// (PrepareResult.Error), so the typed error - and with it the code +// ConvertToMySQLError reads - is gone by the time anything maps the failure to +// a client error. The code therefore has to be read where the typed error +// still exists and carried alongside the message. Without this, the +// width-ceiling refusal that this file's first test proves is 1264 reached a +// real MySQL client as 1105 HY000 +// (test/autoinc_claim_cluster_test.go TestAutoIncSeed_RefusesWhenExistingMaxExceedsWidth). +// +// Mutation: return 0 from mysqlCodeForError. The first assertion fires. The +// DDL rejection site itself is pinned by +// TestPrepareOfARefusedDDLCarriesTheMySQLCode. +func TestPrepareRejectionCarriesTheMySQLCode(t *testing.T) { + t.Parallel() + + widthErr := &AutoIncWidthExceededError{Table: "narrow", Column: "id", Max: 200, WidthMax: 127} + + require.Equal(t, mysqlcode.ErrCodeDataOutOfRange, mysqlCodeForError(widthErr), + "the rejection must carry the participant's own MySQL code, not 0") + require.Zero(t, mysqlCodeForError(context.DeadlineExceeded), + "an error naming no code must stay 0 so the coordinator keeps classifying the message") + + // The value survives the conversion the local participant path uses. + // Mutation: drop ErrorCode from ToCoordinatorResponse. + result := &PrepareResult{Success: false, Rejected: true, Error: widthErr.Error(), + ErrorCode: mysqlCodeForError(widthErr)} + require.Equal(t, mysqlcode.ErrCodeDataOutOfRange, result.ToCoordinatorResponse().ErrorCode, + "the code must reach the coordinator with the response, not stop at the db boundary") +} + +// TestPrepareOfARefusedDDLCarriesTheMySQLCode drives the DDL rejection site in +// ReplicationEngine.Prepare itself: a DDL that would leave a TINYINT +// AUTO_INCREMENT column unable to hold the id 200 it already contains is +// refused, and the PrepareResult carries 1264 for the coordinator. +// +// Mutation: drop ErrorCode from the PrepareResult literal at the DDL rejection +// site in prepareRegularTransaction. "the DDL rejection lost its MySQL code" +// fires. +func TestPrepareOfARefusedDDLCarriesTheMySQLCode(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE t (id INTEGER /*M:8a*/ PRIMARY KEY, v TEXT)") + _, err := mdb.GetWriteDB().Exec("INSERT INTO t (id, v) VALUES (200, 'x')") + require.NoError(t, err) + + result := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: 7600, NodeID: 1, StartTS: hlc.Timestamp{WallTime: 14}, Database: "testdb", + Statements: []protocol.Statement{{Type: protocol.StatementDDL, Database: "testdb", TableName: "t", + SQL: "ALTER TABLE t ADD COLUMN extra INTEGER"}}, + }) + require.False(t, result.Success) + require.True(t, result.Rejected) + require.Equal(t, mysqlcode.ErrCodeDataOutOfRange, result.ErrorCode, "the DDL rejection lost its MySQL code") +} diff --git a/db/ddl_prepare_validate_test.go b/db/ddl_prepare_validate_test.go index 537ec6c7..fccd7c43 100644 --- a/db/ddl_prepare_validate_test.go +++ b/db/ddl_prepare_validate_test.go @@ -303,3 +303,109 @@ func TestIsDDLRejection_ReadonlyIsNotRejected(t *testing.T) { require.False(t, isDDLRejection(context.Background(), verr), "a read-only database is a condition on this node, not a verdict on the DDL") } + +// TestValidateDDLStatements_AutoIncWidthExceededIsRejected pins that a table whose explicitly declared AUTO_INCREMENT column +// cannot hold a row's existing value must be rejected at PREPARE, not left to +// fail later at COMMIT. +func TestValidateDDLStatements_AutoIncWidthExceededIsRejected(t *testing.T) { + t.Parallel() + db := openDDLValidationDB(t) + + err := ValidateDDLStatements(context.Background(), db, []string{ + `CREATE TABLE narrow (id INTEGER /*M:8a*/ PRIMARY KEY)`, + `INSERT INTO narrow (id) VALUES (200)`, + }) + require.Error(t, err) + + var widthErr *AutoIncWidthExceededError + require.True(t, errors.As(err, &widthErr), "expected *AutoIncWidthExceededError, got %T: %v", err, err) + require.Equal(t, "narrow", widthErr.Table) + require.Equal(t, "id", widthErr.Column) + require.Equal(t, uint64(200), widthErr.Max) + require.Equal(t, uint64(127), widthErr.WidthMax) // TINYINT signed ceiling + + require.True(t, isDDLRejection(context.Background(), err), + "a width-ceiling violation is always a deterministic rejection") + + // Validation always rolls back: the table must not exist afterwards. + var count int + require.NoError(t, db.QueryRow( + `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='narrow'`).Scan(&count)) + require.Equal(t, 0, count) +} + +// TestValidateDDLStatements_ExistingRowsExceedNarrowedWidthIsRejected covers +// the scenario 2c exists for: a table that already held rows before this +// DDL ran gets narrowed (here, via a rebuild - CREATE + copy + DROP + RENAME, +// the only way SQLite can ever change a column's declared type) to a width +// its own existing data no longer fits. +func TestValidateDDLStatements_ExistingRowsExceedNarrowedWidthIsRejected(t *testing.T) { + t.Parallel() + db := openDDLValidationDB(t) + + _, err := db.Exec(`CREATE TABLE wide (id INTEGER /*M:32a*/ PRIMARY KEY)`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO wide (id) VALUES (200)`) + require.NoError(t, err) + + verr := ValidateDDLStatements(context.Background(), db, []string{ + `CREATE TABLE wide_new (id INTEGER /*M:8a*/ PRIMARY KEY)`, + `INSERT INTO wide_new SELECT id FROM wide`, + `DROP TABLE wide`, + `ALTER TABLE wide_new RENAME TO wide`, + }) + require.Error(t, verr) + + var widthErr *AutoIncWidthExceededError + require.True(t, errors.As(verr, &widthErr), "expected *AutoIncWidthExceededError, got %T: %v", verr, verr) + require.Equal(t, "wide", widthErr.Table) + require.True(t, isDDLRejection(context.Background(), verr)) + + // Rolled back: the table's original (wide) declared type is unchanged. + var createSQL string + require.NoError(t, db.QueryRow(`SELECT sql FROM sqlite_master WHERE name='wide'`).Scan(&createSQL)) + require.Contains(t, createSQL, "/*M:32a*/") +} + +// TestValidateDDLStatements_AutoIncWidthWithinCeilingIsAccepted is the +// negative case: a row within the declared width's ceiling must not be +// rejected. +func TestValidateDDLStatements_AutoIncWidthWithinCeilingIsAccepted(t *testing.T) { + t.Parallel() + db := openDDLValidationDB(t) + + require.NoError(t, ValidateDDLStatements(context.Background(), db, []string{ + `CREATE TABLE narrow (id INTEGER /*M:8a*/ PRIMARY KEY)`, + `INSERT INTO narrow (id) VALUES (127)`, + })) +} + +// TestValidateDDLStatements_NewEmptyMarkedTableIsAccepted covers the common +// case: a brand-new marked table with no rows has nothing to violate the +// ceiling. +func TestValidateDDLStatements_NewEmptyMarkedTableIsAccepted(t *testing.T) { + t.Parallel() + db := openDDLValidationDB(t) + + require.NoError(t, ValidateDDLStatements(context.Background(), db, []string{ + `CREATE TABLE narrow (id INTEGER /*M:8a*/ PRIMARY KEY)`, + })) +} + +// TestValidateDDLStatements_UnrelatedTableIsNotChecked proves the ceiling +// check is scoped to tables this DDL actually touched: a pre-existing table +// that already violates its own declared ceiling must not fail an unrelated +// CREATE TABLE that never references it. +func TestValidateDDLStatements_UnrelatedTableIsNotChecked(t *testing.T) { + t.Parallel() + db := openDDLValidationDB(t) + + _, err := db.Exec(`CREATE TABLE already_bad (id INTEGER /*M:8a*/ PRIMARY KEY)`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO already_bad (id) VALUES (200)`) + require.NoError(t, err) + + require.NoError(t, ValidateDDLStatements(context.Background(), db, []string{ + `CREATE TABLE unrelated (id INTEGER PRIMARY KEY)`, + })) +} diff --git a/db/detach_drain_test.go b/db/detach_drain_test.go new file mode 100644 index 00000000..270c0e26 --- /dev/null +++ b/db/detach_drain_test.go @@ -0,0 +1,433 @@ +package db + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/maxpert/marmot/cfg" + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// newDetachTestManager opens a DatabaseManager with a user database "app" +// holding table t. +func newDetachTestManager(t *testing.T) *DatabaseManager { + t.Helper() + dm, err := NewDatabaseManager(t.TempDir(), 1, hlc.NewClock(1)) + require.NoError(t, err) + t.Cleanup(func() { _ = dm.Close() }) + require.NoError(t, dm.CreateDatabase("app")) + app, err := dm.GetDatabase("app") + require.NoError(t, err) + _, err = app.GetWriteDB().Exec("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)") + require.NoError(t, err) + return dm +} + +// holdGCPassBeforeTheManagerLock restarts tm's GC with a short interval and a +// refresh step that blocks the first pass until release is closed. When the +// returned channel closes, a pass is inside that step; its next call is the +// manager's GetMinAppliedTxnID, which takes DatabaseManager.mu. +func holdGCPassBeforeTheManagerLock(tm *TransactionManager, release <-chan struct{}) <-chan struct{} { + entered := make(chan struct{}) + var once sync.Once + tm.StopGarbageCollection() + tm.SetRefreshReplicationStatesFunc(func(context.Context) error { + once.Do(func() { close(entered) }) + <-release + return nil + }) + tm.gcInterval = 5 * time.Millisecond + tm.StartGarbageCollection() + return entered +} + +// gcStopRequested reports whether StopGarbageCollection has begun on tm. +func gcStopRequested(tm *TransactionManager) bool { + tm.mu.RLock() + defer tm.mu.RUnlock() + return !tm.gcRunning +} + +// TestClosingADatabaseDoesNotDeadlockWithAGCPassWaitingOnTheManager: detach, +// drop and shutdown each stop a database's GC and wait for its pass to end, +// while that pass may be about to take DatabaseManager.mu +// (GetMinAppliedTxnID). None of them may hold mu while it waits. The pass is +// held in its refresh step until the operation is waiting for it, then let go +// towards the lock. +// +// Mutation: hold dm.mu across DetachDatabase's drain (or DropDatabase's +// db.Close, or Close's database loop). The pass then waits on the lock the +// waiter holds, and "deadlocked with a GC pass waiting on the manager's lock" +// fires for that operation. +func TestClosingADatabaseDoesNotDeadlockWithAGCPassWaitingOnTheManager(t *testing.T) { + operations := map[string]func(dm *DatabaseManager) error{ + "detach": func(dm *DatabaseManager) error { return dm.DetachDatabase(context.Background(), "app") }, + "drop": func(dm *DatabaseManager) error { return dm.DropDatabase("app") }, + "close": func(dm *DatabaseManager) error { return dm.Close() }, + } + for name, operation := range operations { + t.Run(name, func(t *testing.T) { + dm := newDetachTestManager(t) + app, err := dm.GetDatabase("app") + require.NoError(t, err) + tm := app.GetTransactionManager() + release := make(chan struct{}) + entered := holdGCPassBeforeTheManagerLock(tm, release) + <-entered + + done := make(chan error, 1) + go func() { done <- operation(dm) }() + require.Eventually(t, func() bool { return gcStopRequested(tm) }, 5*time.Second, time.Millisecond, + "%s never began stopping the GC", name) + close(release) + + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatalf("%s deadlocked with a GC pass waiting on the manager's lock", name) + } + }) + } +} + +// TestDetachRefusesAWriteTransactionInFlight: a caller that began a write +// transaction on the database before the detach still holds it. The detach +// waits for it rather than returning under it, and its commit is refused, so +// the write is never ACKed into a file a restore then replaces. +// +// A MySQL client sees the refusal as retryable (1213, SQLSTATE 40001). +// +// Mutation: drop the commit hook's refusal (writeGate.commitHook returns 0). +// "an in-flight transaction committed after its database was detached" fires. +// Mutation: drop the mapper's ErrConstraintCommitHook case. "a commit refused +// at the detach does not reach a MySQL client as retryable" fires. +// Mutation: skip the drain (DetachDatabase calls closeSQLite instead of +// drainSQLite). "the detach returned while a write transaction was in flight" +// fires. +func TestDetachRefusesAWriteTransactionInFlight(t *testing.T) { + dm := newDetachTestManager(t) + held, err := dm.GetDatabase("app") + require.NoError(t, err) + tx, err := held.GetWriteDB().Begin() + require.NoError(t, err) + _, err = tx.Exec("INSERT INTO t (id, v) VALUES (500, 'in flight')") + require.NoError(t, err) + + done := make(chan error, 1) + go func() { done <- dm.DetachDatabase(context.Background(), "app") }() + require.Eventually(t, held.gate.closed.Load, 5*time.Second, time.Millisecond, "the detach never closed the write gate") + require.Never(t, func() bool { return len(done) > 0 }, 200*time.Millisecond, 10*time.Millisecond, + "the detach returned while a write transaction was in flight") + + commitErr := tx.Commit() + require.Error(t, commitErr, "an in-flight transaction committed after its database was detached") + refused := protocol.ConvertToMySQLError(commitErr) + require.Equal(t, [2]any{protocol.ErrCodeDeadlock, protocol.SQLStateDeadlock}, [2]any{refused.Code, refused.SQLState}, + "a commit refused at the detach does not reach a MySQL client as retryable: %v", commitErr) + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("the detach did not finish once the transaction in flight ended") + } + + require.NoError(t, dm.AttachDatabase("app")) + app, err := dm.GetDatabase("app") + require.NoError(t, err) + var n int + require.NoError(t, app.GetReadDB().QueryRow("SELECT COUNT(*) FROM t WHERE id = 500").Scan(&n)) + require.Zero(t, n, "a refused commit left its row behind") +} + +// TestDetachThatCannotDrainKeepsTheFile: a write transaction that outlasts the +// drain's deadline makes DetachDatabase return ErrDrainIncomplete with the +// database still detached. The transaction's commit is refused all the same, +// and the database reattaches over its own file, in service. +// +// Mutation: drop the drain's error (drainSQLite returns nil after a failed +// Conn). "a detach that did not drain reported success" fires. +func TestDetachThatCannotDrainKeepsTheFile(t *testing.T) { + dm := newDetachTestManager(t) + held, err := dm.GetDatabase("app") + require.NoError(t, err) + _, err = held.GetWriteDB().Exec("INSERT INTO t (id, v) VALUES (1, 'kept')") + require.NoError(t, err) + tx, err := held.GetWriteDB().Begin() + require.NoError(t, err) + _, err = tx.Exec("INSERT INTO t (id, v) VALUES (500, 'in flight')") + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + err = dm.DetachDatabase(ctx, "app") + require.ErrorIs(t, err, ErrDrainIncomplete, "a detach that did not drain reported success") + _, err = dm.GetDatabase("app") + require.ErrorIs(t, err, ErrDatabaseDetached) + require.Error(t, tx.Commit(), "an in-flight transaction committed after its database was detached") + + require.NoError(t, dm.AttachDatabase("app")) + app, err := dm.GetDatabase("app") + require.NoError(t, err) + var v string + require.NoError(t, app.GetReadDB().QueryRow("SELECT v FROM t WHERE id = 1").Scan(&v)) + require.Equal(t, "kept", v) + _, err = app.GetWriteDB().Exec("INSERT INTO t (id, v) VALUES (2, 'after')") + require.NoError(t, err, "the reattached database does not accept writes") +} + +// TestSnapshotsRefuseWhileADatabaseIsDetached: a snapshot of every database +// ships the registry, which still names a detached database. Rather than +// silently leave its file out, both snapshot producers refuse with +// ErrDatabaseDetached (the gRPC layer turns that into Unavailable, which the +// caller retries), and include it again once it is back. +// +// Mutation: drop the check (errIfDetachedLocked returns nil). "a snapshot +// omitted a detached database" fires for TakeSnapshot and TakeSnapshotToDir. +func TestSnapshotsRefuseWhileADatabaseIsDetached(t *testing.T) { + dm := newDetachTestManager(t) + require.NoError(t, dm.DetachDatabase(context.Background(), "app")) + + _, _, err := dm.TakeSnapshot() + require.ErrorIs(t, err, ErrDatabaseDetached, "a snapshot omitted a detached database (TakeSnapshot)") + _, _, _, err = dm.TakeSnapshotToDir(t.TempDir()) + require.ErrorIs(t, err, ErrDatabaseDetached, "a snapshot omitted a detached database (TakeSnapshotToDir)") + + require.NoError(t, dm.AttachDatabase("app")) + infos, _, err := dm.TakeSnapshot() + require.NoError(t, err) + require.Contains(t, snapshotNames(infos), "app") + infos, _, _, err = dm.TakeSnapshotToDir(t.TempDir()) + require.NoError(t, err) + require.Contains(t, snapshotNames(infos), "app") +} + +// TestPerDatabaseSnapshotsRefuseOnlyTheirOwnDetach: a snapshot of one +// database ships no other database and no registry, so another database +// being out of service does not refuse it; its own detach does, with +// ErrDatabaseDetached. +// +// Mutation: make TakeDatabaseSnapshotInfo (or TakeSnapshotForDatabase) call +// errIfDetachedLocked first. "a per-database snapshot was refused because +// another database was detached" fires. Mutation: make getDatabaseLocked +// report a detached database as not existing. "a per-database snapshot of a +// detached database was not refused as detached" fires. +func TestPerDatabaseSnapshotsRefuseOnlyTheirOwnDetach(t *testing.T) { + dm := newDetachTestManager(t) + require.NoError(t, dm.CreateDatabase("other")) + require.NoError(t, dm.DetachDatabase(context.Background(), "other")) + + info, _, err := dm.TakeDatabaseSnapshotInfo("app") + require.NoError(t, err, "a per-database snapshot was refused because another database was detached (TakeDatabaseSnapshotInfo)") + require.Equal(t, "app", info.Name) + copied, _, err := dm.TakeSnapshotForDatabase(t.TempDir(), "app") + require.NoError(t, err, "a per-database snapshot was refused because another database was detached (TakeSnapshotForDatabase)") + require.Equal(t, info.SHA256, copied.SHA256, "the per-database info does not describe the file the snapshot copies") + + _, _, err = dm.TakeDatabaseSnapshotInfo("other") + require.ErrorIs(t, err, ErrDatabaseDetached, "a per-database snapshot of a detached database was not refused as detached (TakeDatabaseSnapshotInfo)") + _, _, err = dm.TakeSnapshotForDatabase(t.TempDir(), "other") + require.ErrorIs(t, err, ErrDatabaseDetached, "a per-database snapshot of a detached database was not refused as detached (TakeSnapshotForDatabase)") +} + +func snapshotNames(infos []SnapshotInfo) []string { + names := make([]string, 0, len(infos)) + for _, info := range infos { + names = append(names, info.Name) + } + return names +} + +// strandDatabase detaches "app" and reattaches it over a file that is not a +// SQLite database, as a restore that installed an unreadable file would. It +// returns the path of the database's file and its original content. +func strandDatabase(t *testing.T, dm *DatabaseManager) (string, []byte) { + t.Helper() + path, err := dm.GetDatabasePath("app") + require.NoError(t, err) + require.NoError(t, dm.DetachDatabase(context.Background(), "app")) + good, err := os.ReadFile(path) + require.NoError(t, err) + for _, suffix := range []string{"-wal", "-shm"} { + require.NoError(t, os.RemoveAll(path+suffix)) + } + require.NoError(t, os.WriteFile(path, []byte("not a SQLite database, not at all"), 0o644)) + require.Error(t, dm.AttachDatabase("app")) + return path, good +} + +// TestFailedReattachAwaitsAnotherRestore: a reattach that cannot open the +// restored file leaves the database out of service but not lost. It still +// exists, lookups get ErrDatabaseDetached, it is listed for another restore, +// and the next restore of it (a detach that takes it over, a good file, a +// reattach) puts it back in service. +// +// Mutation: make DetachDatabase refuse a database whose reattach failed. +// "a database whose reattach failed cannot be restored again" fires. +func TestFailedReattachAwaitsAnotherRestore(t *testing.T) { + dm := newDetachTestManager(t) + app, err := dm.GetDatabase("app") + require.NoError(t, err) + _, err = app.GetWriteDB().Exec("INSERT INTO t (id, v) VALUES (1, 'restored')") + require.NoError(t, err) + path, good := strandDatabase(t, dm) + + require.True(t, dm.DatabaseExists("app")) + _, err = dm.GetDatabase("app") + require.ErrorIs(t, err, ErrDatabaseDetached) + require.Equal(t, []string{"app"}, dm.DatabasesAwaitingRestore(), "a database whose reattach failed is not listed for another restore") + require.NoError(t, dm.CreateDatabase("app"), "CREATE of an existing database must stay idempotent") + + require.NoError(t, dm.DetachDatabase(context.Background(), "app"), "a database whose reattach failed cannot be restored again") + require.Empty(t, dm.DatabasesAwaitingRestore(), "a database under restore is listed as awaiting one") + require.ErrorIs(t, dm.DetachDatabase(context.Background(), "app"), ErrDatabaseDetached, "two restores took the same database") + require.NoError(t, os.WriteFile(path, good, 0o644)) + require.NoError(t, dm.AttachDatabase("app")) + + app, err = dm.GetDatabase("app") + require.NoError(t, err) + var v string + require.NoError(t, app.GetReadDB().QueryRow("SELECT v FROM t WHERE id = 1").Scan(&v)) + require.Equal(t, "restored", v) +} + +// TestDropDatabaseDuringARestore: a replicated DROP DATABASE that arrives +// while the database is detached for a restore takes effect at once - the +// registry forgets it and it no longer exists for lookups - and the end of +// the restore removes its files instead of reattaching it. A CREATE of the +// same name is refused until then (retryable) and succeeds afterwards. +// +// Mutation: refuse DROP DATABASE while the database is detached, as fix +// round 2 did. "a DROP DATABASE during a restore was refused" fires. +func TestDropDatabaseDuringARestore(t *testing.T) { + dm := newDetachTestManager(t) + path, err := dm.GetDatabasePath("app") + require.NoError(t, err) + require.NoError(t, dm.DetachDatabase(context.Background(), "app")) + + require.NoError(t, dm.DropDatabase("app"), "a DROP DATABASE during a restore was refused") + require.False(t, dm.DatabaseExists("app"), "a dropped database still exists") + _, err = dm.GetDatabase("app") + require.Error(t, err) + require.NotErrorIs(t, err, ErrDatabaseDetached) + require.NotContains(t, registryNames(t, dm), "app", "the registry still names a dropped database") + require.ErrorIs(t, dm.CreateDatabase("app"), ErrDatabaseDetached, "a CREATE raced the files of a drop in progress") + require.NoError(t, dm.DropDatabase("app"), "a repeated DROP must stay idempotent") + + require.NoError(t, dm.AttachDatabase("app")) + require.False(t, dm.DatabaseExists("app"), "a database dropped during its restore came back") + _, err = os.Stat(path) + require.ErrorIs(t, err, os.ErrNotExist, "a database dropped during its restore kept its file") + _, err = os.Stat(filepath.Join(filepath.Dir(path), "app_meta.pebble")) + require.ErrorIs(t, err, os.ErrNotExist, "a database dropped during its restore kept its meta store") + + require.NoError(t, dm.CreateDatabase("app")) + app, err := dm.GetDatabase("app") + require.NoError(t, err) + var n int + require.NoError(t, app.GetReadDB().QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE name = 't'").Scan(&n)) + require.Zero(t, n, "the re-created database kept the dropped one's tables") +} + +// TestDropDatabaseAwaitingARestore: a database whose reattach failed is +// dropped outright. +// +// Mutation: treat it as a restore in progress (leave the drop to +// AttachDatabase). "a database awaiting a restore was not dropped" fires. +func TestDropDatabaseAwaitingARestore(t *testing.T) { + dm := newDetachTestManager(t) + path, _ := strandDatabase(t, dm) + + require.NoError(t, dm.DropDatabase("app")) + require.False(t, dm.DatabaseExists("app")) + require.Empty(t, dm.DatabasesAwaitingRestore()) + _, err := os.Stat(path) + require.ErrorIs(t, err, os.ErrNotExist, "a database awaiting a restore was not dropped") + require.NoError(t, dm.CreateDatabase("app")) +} + +func registryNames(t *testing.T, dm *DatabaseManager) []string { + t.Helper() + rows, err := dm.GetSystemDatabase().GetReadDB().Query("SELECT name FROM __marmot_databases") + require.NoError(t, err) + defer rows.Close() + var names []string + for rows.Next() { + var name string + require.NoError(t, rows.Scan(&name)) + names = append(names, name) + } + require.NoError(t, rows.Err()) + return names +} + +// TestQueuedBatchCommitsAtCloseAndAtDetach: a commit queued on the batch +// committer and not yet flushed is committed when the database closes (drop, +// shutdown), as before the write gate existed, but refused when the database +// is detached for a restore, whose file is about to be replaced. +// +// Mutation: close the gate before stopping the batch committer in +// closeSQLite. "a commit queued before Close was refused" fires. Mutation: +// stop the batch committer before the gate closes on detach (in +// takeOutOfService and drainSQLite). "a commit queued before the detach was +// written into the file being replaced" fires. +func TestQueuedBatchCommitsAtCloseAndAtDetach(t *testing.T) { + restore := cfg.Config.BatchCommit + t.Cleanup(func() { cfg.Config.BatchCommit = restore }) + cfg.Config.BatchCommit.Enabled = true + cfg.Config.BatchCommit.MaxWaitMS = int(time.Minute / time.Millisecond) // only Stop flushes + cfg.Config.BatchCommit.MaxBatchSize = 1024 + clock := hlc.NewClock(1) + + t.Run("close", func(t *testing.T) { + dm := newDetachTestManager(t) + app, err := dm.GetDatabase("app") + require.NoError(t, err) + queued := app.batchCommitter.Enqueue(9801, clock.Now(), nil, nil) + require.NoError(t, app.Close()) + _, err = queued.Get() + require.NoError(t, err, "a commit queued before Close was refused") + }) + t.Run("detach", func(t *testing.T) { + dm := newDetachTestManager(t) + app, err := dm.GetDatabase("app") + require.NoError(t, err) + queued := app.batchCommitter.Enqueue(9802, clock.Now(), nil, nil) + require.NoError(t, dm.DetachDatabase(context.Background(), "app")) + _, err = queued.Get() + require.Error(t, err, "a commit queued before the detach was written into the file being replaced") + }) +} + +// TestReplicaReopenedPoolsAreGated: a replica restore closes a database's +// pools and opens new ones over the installed file (CloseDatabaseConnections, +// OpenDatabaseConnections). The new pools carry the same write gate, so once +// the database leaves service a commit on any of them is refused, as on the +// pools it was opened with. +// +// Mutation: open any one of the pools in OpenSQLiteConnections with sql.Open. +// "a database's reopened pool committed after it left service" fires for that +// pool. +func TestReplicaReopenedPoolsAreGated(t *testing.T) { + dm := newDetachTestManager(t) + require.NoError(t, dm.CloseDatabaseConnections("app")) + require.NoError(t, dm.OpenDatabaseConnections("app")) + app, err := dm.GetDatabase("app") + require.NoError(t, err) + _, err = app.GetWriteDB().Exec("INSERT INTO t (id, v) VALUES (1, 'reopened')") + require.NoError(t, err, "a database's reopened pools do not accept writes") + + app.gate.close() + pools := map[string]*sql.DB{"write": app.writeDB, "hook": app.hookDB, "read": app.readDB} + for name, pool := range pools { + _, err := pool.Exec("INSERT INTO t (id, v) VALUES (2, ?)", name) + require.Error(t, err, "a database's reopened pool committed after it left service (%s)", name) + } +} diff --git a/db/meta_schema.go b/db/meta_schema.go index 4512f643..07efb7ee 100644 --- a/db/meta_schema.go +++ b/db/meta_schema.go @@ -13,6 +13,20 @@ const ( IntentTypeDML IntentType = 0 // Regular row operations (INSERT/UPDATE/DELETE) IntentTypeDDL IntentType = 1 // Schema operations (CREATE/ALTER/DROP TABLE) IntentTypeDatabaseOp IntentType = 2 // Database operations (CREATE/DROP DATABASE) + // IntentTypeAutoIDClaim is a narrow auto-increment range claim. + // + // It must NOT be IntentTypeDML, and that is a correctness requirement + // rather than a taxonomy preference. WriteIntent short-circuits + // IntentTypeDML into storeDMLIntent, whose record omits DataSnapshot and + // which writes only to an in-memory map; GetIntentsByTxn iterates the + // Pebble prefix only, so a claim stored that way would never be returned + // and its payload would never have existed. The COMMIT handler reads + // exactly that payload to write the new base, so a claim on the DML branch + // would pass PREPARE, apply nothing at COMMIT, leave every participant's + // base where it was, and let the next claimant anywhere compute the same + // range and mint the same ids. Selecting storage is precisely what this + // type is for. + IntentTypeAutoIDClaim IntentType = 3 ) func (t IntentType) String() string { @@ -23,6 +37,8 @@ func (t IntentType) String() string { return "DDL" case IntentTypeDatabaseOp: return "DATABASE_OP" + case IntentTypeAutoIDClaim: + return "AUTO_ID_CLAIM" default: return "UNKNOWN" } diff --git a/db/meta_store.go b/db/meta_store.go index b5fdd7b1..6dd26589 100644 --- a/db/meta_store.go +++ b/db/meta_store.go @@ -51,7 +51,6 @@ type MetaStore interface { ValidateIntent(tableName, intentKey string, expectedTxnID uint64) (bool, error) DeleteIntent(tableName, intentKey string, txnID uint64) error DeleteIntentsByTxn(txnID uint64) error - MarkIntentsForCleanup(txnID uint64) error // Fast path: mark intents as ready for overwrite GetIntentsByTxn(txnID uint64) ([]*WriteIntentRecord, error) GetIntent(tableName, intentKey string) (*WriteIntentRecord, error) @@ -114,7 +113,7 @@ type MetaStore interface { ScanTransactions(fromTxnID uint64, descending bool, callback func(*TransactionRecord) error) error // Stats methods for telemetry - GetRowLockStats() (activeLocks, activeTransactions, gcMarkers, tablesWithLocks int) + GetRowLockStats() (activeLocks, activeTransactions, tablesWithLocks int) IntentStats() (pendingIntents int, err error) // Lifecycle @@ -212,7 +211,8 @@ func NewMetaStore(basePath string) (MetaStore, error) { // Wrap with MemoryMetaStore for transitionary state optimization memStore := NewMemoryMetaStore(pebble) - // Clean up orphaned CDC data from crashed transactions + // Register recovered prepared transactions and clean up orphaned CDC data + // from crashed ones. The registration cannot fail; only the cleanup can. if err := memStore.ReconstructFromPebble(); err != nil { // Log warning but don't fail startup - orphaned data is just wasted space log.Warn().Err(err).Msg("Failed to reconstruct from Pebble at startup") diff --git a/db/meta_store_gc_intent_test.go b/db/meta_store_gc_intent_test.go index 6d86d613..6ce5dcc3 100644 --- a/db/meta_store_gc_intent_test.go +++ b/db/meta_store_gc_intent_test.go @@ -1,76 +1,15 @@ package db import ( + "fmt" "testing" "github.com/maxpert/marmot/hlc" "github.com/stretchr/testify/require" ) -// TestMarkIntentsForCleanup_CreatesGCMarkers tests that MarkIntentsForCleanup creates GC markers in RowLockStore -func TestMarkIntentsForCleanup_CreatesGCMarkers(t *testing.T) { - store := newTestPebbleStore(t) - defer cleanupTestPebbleStore(t, store) - - txnID := uint64(100) - tableName := "users" - intentKey := "pk:1" - - // Create transaction and write intent - ts := hlc.Timestamp{WallTime: 1000, Logical: 0} - err := store.BeginTransaction(txnID, 1, ts) - require.NoError(t, err) - - err = store.WriteIntent(txnID, IntentTypeDML, tableName, intentKey, OpTypeInsert, "INSERT INTO users...", []byte("data"), ts, 1) - require.NoError(t, err) - - // Mark intents for cleanup - err = store.MarkIntentsForCleanup(txnID) - require.NoError(t, err) - - // Verify GC marker exists in RowLockStore - gcExists := store.rowLocks.CheckGCMarker("", tableName, intentKey) - require.True(t, gcExists, "GC marker should exist in RowLockStore") - - // Verify intent still exists in RowLockStore - holder, exists := store.rowLocks.CheckLock("", tableName, intentKey) - require.True(t, exists, "Intent lock should still exist in RowLockStore") - require.Equal(t, txnID, holder, "Intent should belong to correct transaction") -} - -// TestMarkIntentsForCleanup_MultipleIntents tests marking multiple intents -func TestMarkIntentsForCleanup_MultipleIntents(t *testing.T) { - store := newTestPebbleStore(t) - defer cleanupTestPebbleStore(t, store) - - txnID := uint64(200) - tableName := "products" - ts := hlc.Timestamp{WallTime: 1000, Logical: 0} - - // Create transaction - err := store.BeginTransaction(txnID, 1, ts) - require.NoError(t, err) - - // Write multiple intents - intentKeys := []string{"pk:1", "pk:2", "pk:3"} - for _, intentKey := range intentKeys { - err = store.WriteIntent(txnID, IntentTypeDML, tableName, intentKey, OpTypeInsert, "INSERT INTO products...", []byte("data"), ts, 1) - require.NoError(t, err) - } - - // Mark all intents for cleanup - err = store.MarkIntentsForCleanup(txnID) - require.NoError(t, err) - - // Verify all GC markers exist in RowLockStore - for _, intentKey := range intentKeys { - gcExists := store.rowLocks.CheckGCMarker("", tableName, intentKey) - require.True(t, gcExists, "GC marker should exist for %s", intentKey) - } -} - -// TestDeleteIntentsByTxn_DeletesGCMarkers tests that DeleteIntentsByTxn removes locks from RowLockStore -func TestDeleteIntentsByTxn_DeletesGCMarkers(t *testing.T) { +// TestDeleteIntentsByTxn_ReleasesLocks tests that DeleteIntentsByTxn removes locks from RowLockStore +func TestDeleteIntentsByTxn_ReleasesLocks(t *testing.T) { store := newTestPebbleStore(t) defer cleanupTestPebbleStore(t, store) @@ -79,37 +18,19 @@ func TestDeleteIntentsByTxn_DeletesGCMarkers(t *testing.T) { intentKey := "pk:100" ts := hlc.Timestamp{WallTime: 1000, Logical: 0} - // Create transaction and write intent - err := store.BeginTransaction(txnID, 1, ts) - require.NoError(t, err) - - err = store.WriteIntent(txnID, IntentTypeDML, tableName, intentKey, OpTypeInsert, "INSERT INTO orders...", []byte("data"), ts, 1) - require.NoError(t, err) - - // Mark for cleanup - err = store.MarkIntentsForCleanup(txnID) - require.NoError(t, err) - - // Verify GC marker exists in RowLockStore - gcExists := store.rowLocks.CheckGCMarker("", tableName, intentKey) - require.True(t, gcExists, "GC marker should exist") + require.NoError(t, store.BeginTransaction(txnID, 1, ts)) + require.NoError(t, store.WriteIntent(txnID, IntentTypeDML, tableName, intentKey, OpTypeInsert, "INSERT INTO orders...", []byte("data"), ts, 1)) - // Delete intents by txn - err = store.DeleteIntentsByTxn(txnID) - require.NoError(t, err) + require.NoError(t, store.DeleteIntentsByTxn(txnID)) - // Verify lock is released from RowLockStore _, exists := store.rowLocks.CheckLock("", tableName, intentKey) require.False(t, exists, "Lock should be released from RowLockStore") - - // Note: GC markers are separate from locks and not automatically cleaned up - // They remain in memory until overwritten or the store is restarted (ephemeral) - gcExists = store.rowLocks.CheckGCMarker("", tableName, intentKey) - require.True(t, gcExists, "GC marker persists in memory even after lock release") } -// TestWriteIntent_DetectsGCMarker tests that WriteIntent detects GC marker and allows overwrite -func TestWriteIntent_DetectsGCMarker(t *testing.T) { +// TestWriteIntent_OverwritesACommittedHolder tests that an intent whose +// transaction has committed, but whose lock is still held, is overwritten by +// the next writer. +func TestWriteIntent_OverwritesACommittedHolder(t *testing.T) { store := newTestPebbleStore(t) defer cleanupTestPebbleStore(t, store) @@ -119,36 +40,49 @@ func TestWriteIntent_DetectsGCMarker(t *testing.T) { intentKey := "pk:500" ts := hlc.Timestamp{WallTime: 1000, Logical: 0} - // Create first transaction and write intent - err := store.BeginTransaction(txnID1, 1, ts) - require.NoError(t, err) + require.NoError(t, store.BeginTransaction(txnID1, 1, ts)) + require.NoError(t, store.WriteIntent(txnID1, IntentTypeDML, tableName, intentKey, OpTypeInsert, "INSERT INTO inventory...", []byte("data1"), ts, 1)) + require.NoError(t, store.CommitTransaction(txnID1, hlc.Timestamp{WallTime: 1500}, nil, "testdb", tableName, 0, 1)) - err = store.WriteIntent(txnID1, IntentTypeDML, tableName, intentKey, OpTypeInsert, "INSERT INTO inventory...", []byte("data1"), ts, 1) - require.NoError(t, err) + require.NoError(t, store.BeginTransaction(txnID2, 1, hlc.Timestamp{WallTime: 2000, Logical: 0})) + require.NoError(t, store.WriteIntent(txnID2, IntentTypeDML, tableName, intentKey, OpTypeUpdate, "UPDATE inventory...", []byte("data2"), ts, 1), + "a committed holder's intent must be overwritable") - // Mark intent for cleanup - err = store.MarkIntentsForCleanup(txnID1) - require.NoError(t, err) - - // Create second transaction - err = store.BeginTransaction(txnID2, 1, hlc.Timestamp{WallTime: 2000, Logical: 0}) - require.NoError(t, err) - - // Write intent from second transaction - should detect GC marker and overwrite - err = store.WriteIntent(txnID2, IntentTypeDML, tableName, intentKey, OpTypeUpdate, "UPDATE inventory...", []byte("data2"), ts, 1) - require.NoError(t, err, "Should be able to overwrite intent marked for cleanup") - - // Verify GC marker is deleted from RowLockStore - gcExists := store.rowLocks.CheckGCMarker("", tableName, intentKey) - require.False(t, gcExists, "GC marker should be deleted after overwrite") - - // Verify new intent exists with txnID2 intent, err := store.GetIntent(tableName, intentKey) require.NoError(t, err) require.Equal(t, txnID2, intent.TxnID, "Intent should belong to new transaction") require.Nil(t, intent.DataSnapshot, "DML row payloads live in CDC segment storage, not write intents") } +// TestCommittedTransactionsLeaveNoPerRowState: committing leaves nothing behind +// per row written. The store used to keep a GC marker for every row a +// finished transaction had locked, one entry per distinct row ever written. +// +// Mutation: in RowLockStore.ReleaseByTxn, read the reverse index with Load +// instead of LoadAndDelete. "committed transactions left reverse-index +// entries" fires. +func TestCommittedTransactionsLeaveNoPerRowState(t *testing.T) { + store := newTestPebbleStore(t) + defer cleanupTestPebbleStore(t, store) + + const commits = 2000 + ts := hlc.Timestamp{WallTime: 1000} + for i := uint64(1); i <= commits; i++ { + require.NoError(t, store.BeginTransaction(i, 1, ts)) + require.NoError(t, store.WriteIntent(i, IntentTypeDML, "rows", fmt.Sprintf("pk:%d", i), OpTypeInsert, "", nil, ts, 1)) + require.NoError(t, store.CleanupAfterCommit(i)) + } + + locks, txns, tables := store.rowLocks.Stats() + require.Zero(t, locks) + require.Zero(t, txns) + require.Zero(t, tables) + require.Zero(t, store.rowLocks.byTxn.Size(), "committed transactions left reverse-index entries") + rowMap, ok := store.rowLocks.tables.Load(makeTableKey("", "rows")) + require.True(t, ok) + require.Zero(t, rowMap.Size(), "committed transactions left row entries") +} + // TestRowLockStore_PersistenceAcrossRestart tests that RowLockStore is ephemeral and starts empty after restart func TestRowLockStore_PersistenceAcrossRestart(t *testing.T) { opts := &PebbleMetaStoreOptions{ @@ -163,7 +97,7 @@ func TestRowLockStore_PersistenceAcrossRestart(t *testing.T) { tableName := "accounts" ts := hlc.Timestamp{WallTime: 1000, Logical: 0} - // Create first store, add intents, and mark one for cleanup + // Create first store and add intents { store, err := NewPebbleMetaStore(dbPath, *opts) require.NoError(t, err) @@ -175,11 +109,7 @@ func TestRowLockStore_PersistenceAcrossRestart(t *testing.T) { err = store.WriteIntent(txnID1, IntentTypeDML, tableName, "pk:1", OpTypeInsert, "INSERT INTO accounts...", []byte("data1"), ts, 1) require.NoError(t, err) - // Mark first intent for cleanup - err = store.MarkIntentsForCleanup(txnID1) - require.NoError(t, err) - - // Create second transaction and write intent (not marked for cleanup) + // Create second transaction and write intent err = store.BeginTransaction(txnID2, 1, hlc.Timestamp{WallTime: 2000, Logical: 0}) require.NoError(t, err) @@ -215,50 +145,6 @@ func TestRowLockStore_PersistenceAcrossRestart(t *testing.T) { } } -// TestGCMarker_InMemory tests the GC marker is stored in memory -func TestGCMarker_InMemory(t *testing.T) { - store := newTestPebbleStore(t) - defer cleanupTestPebbleStore(t, store) - - tableName := "test_table" - intentKey := "test_key" - txnID := uint64(700) - - // Write intent - ts := hlc.Timestamp{WallTime: 1000, Logical: 0} - err := store.BeginTransaction(txnID, 1, ts) - require.NoError(t, err) - - err = store.WriteIntent(txnID, IntentTypeDML, tableName, intentKey, OpTypeInsert, "INSERT...", []byte("data"), ts, 1) - require.NoError(t, err) - - // Set GC marker - store.rowLocks.SetGCMarker("", tableName, intentKey) - - // Verify marker exists in memory - gcExists := store.rowLocks.CheckGCMarker("", tableName, intentKey) - require.True(t, gcExists, "GC marker should exist in RowLockStore") - - // Delete marker - store.rowLocks.DeleteGCMarker("", tableName, intentKey) - - // Verify marker is gone - gcExists = store.rowLocks.CheckGCMarker("", tableName, intentKey) - require.False(t, gcExists, "GC marker should be deleted from RowLockStore") -} - -// TestMarkIntentsForCleanup_EmptyTxn tests marking cleanup for transaction with no intents -func TestMarkIntentsForCleanup_EmptyTxn(t *testing.T) { - store := newTestPebbleStore(t) - defer cleanupTestPebbleStore(t, store) - - txnID := uint64(600) - - // Mark intents for cleanup for non-existent transaction - err := store.MarkIntentsForCleanup(txnID) - require.NoError(t, err, "Should succeed even if no intents exist") -} - // Helper functions for testing func newTestPebbleStore(t *testing.T) *PebbleMetaStore { @@ -277,3 +163,29 @@ func cleanupTestPebbleStore(t *testing.T, store *PebbleMetaStore) { err := store.Close() require.NoError(t, err) } + +// BenchmarkWriteIntentThenCleanupAfterCommit measures one DML row lock taken +// and released through the commit path: the row-lock work every replicated +// row write pays at PREPARE and COMMIT. +func BenchmarkWriteIntentThenCleanupAfterCommit(b *testing.B) { + store, err := NewPebbleMetaStore(b.TempDir()+"/bench.db", PebbleMetaStoreOptions{CacheSizeMB: 8, MemTableSizeMB: 4, MemTableCount: 2}) + require.NoError(b, err) + defer store.Close() + ts := hlc.Timestamp{WallTime: 1000} + keys := make([]string, 1024) + for i := range keys { + keys[i] = fmt.Sprintf("pk:%d", i) + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + txnID := uint64(i + 1) + if err := store.WriteIntent(txnID, IntentTypeDML, "rows", keys[i%len(keys)], OpTypeInsert, "", nil, ts, 1); err != nil { + b.Fatalf("WriteIntent: %v", err) + } + if err := store.CleanupAfterCommit(txnID); err != nil { + b.Fatalf("CleanupAfterCommit: %v", err) + } + } +} diff --git a/db/meta_store_memory.go b/db/meta_store_memory.go index 30a04adf..60c84e88 100644 --- a/db/meta_store_memory.go +++ b/db/meta_store_memory.go @@ -201,11 +201,6 @@ func (m *MemoryMetaStore) DeleteIntentsByTxn(txnID uint64) error { return m.pebble.DeleteIntentsByTxn(txnID) } -// MarkIntentsForCleanup delegates to Pebble. -func (m *MemoryMetaStore) MarkIntentsForCleanup(txnID uint64) error { - return m.pebble.MarkIntentsForCleanup(txnID) -} - // GetIntentsByTxn delegates to Pebble. func (m *MemoryMetaStore) GetIntentsByTxn(txnID uint64) ([]*WriteIntentRecord, error) { return m.pebble.GetIntentsByTxn(txnID) @@ -438,7 +433,7 @@ func (m *MemoryMetaStore) Checkpoint() error { } // GetRowLockStats returns row lock statistics (stub for memory store) -func (m *MemoryMetaStore) GetRowLockStats() (activeLocks, activeTransactions, gcMarkers, tablesWithLocks int) { +func (m *MemoryMetaStore) GetRowLockStats() (activeLocks, activeTransactions, tablesWithLocks int) { return m.pebble.GetRowLockStats() } @@ -447,14 +442,36 @@ func (m *MemoryMetaStore) IntentStats() (pendingIntents int, err error) { return m.pebble.IntentStats() } -// ReconstructFromPebble cleans up orphaned CDC data from crashed transactions. -// Called at startup to ensure consistency. -// Memory stores start empty - no pending transaction state survives crash. +// ReconstructFromPebble rebuilds the memory tier's view of a store just opened. +// +// Every prepared transaction the Pebble store recovered holds its row locks +// again. It is registered here as pending, with a heartbeat of now, so that +// the stale-transaction GC aborts it heartbeat_timeout after this restart +// unless its COMMIT or ABORT arrives first; until then its captured rows are +// kept for that COMMIT. The captured rows of any other transaction without a +// commit record are orphans of a crash and are deleted. func (m *MemoryMetaStore) ReconstructFromPebble() error { - orphanedTxnIDs, err := m.pebble.findOrphanedCDCRawTxnIDs() + now := time.Now().UnixNano() + for _, rec := range m.pebble.takeRecoveredPrepared() { + m.txnStore.Begin(rec.TxnID, &TxnState{ + NodeID: rec.NodeID, + Status: TxnStatusPending, + StartTSWall: rec.StartTSWall, + StartTSLogical: rec.StartTSLogical, + LastHeartbeat: now, + }) + } + + candidates, err := m.pebble.findOrphanedCDCRawTxnIDs() if err != nil { return fmt.Errorf("failed to find orphaned CDC txnIDs: %w", err) } + orphanedTxnIDs := candidates[:0] + for _, txnID := range candidates { + if _, prepared := m.txnStore.Get(txnID); !prepared { + orphanedTxnIDs = append(orphanedTxnIDs, txnID) + } + } if len(orphanedTxnIDs) == 0 { return nil diff --git a/db/meta_store_pebble.go b/db/meta_store_pebble.go index 5efa7c67..44183760 100644 --- a/db/meta_store_pebble.go +++ b/db/meta_store_pebble.go @@ -84,6 +84,10 @@ type PebbleMetaStore struct { // Optional transaction getter for conflict resolution (set by MemoryMetaStore wrapper) txnGetter TransactionGetter + + // Prepared transactions whose row locks recoverPreparedCDCState restored + // at open, held until the memory tier takes them (takeRecoveredPrepared). + recoveredPrepared []TxnImmutableRecord } func cdcPrepareSyncStrict() bool { @@ -265,6 +269,14 @@ func rejectLegacyCDCRawKeys(db *pebble.DB) error { return iter.Error() } +// recoverPreparedCDCState restores the row locks of every transaction that was +// prepared, and had neither committed nor aborted, when the store last closed. +// Its manifest survives in one of two places: a prepare record in the segment +// log (strict prepare sync, which outlives the loss of unsynced Pebble keys), +// or the manifest key DurablyPrepareTransaction writes. Each recovered +// transaction is listed in recoveredPrepared for the memory tier, which must +// register it so the stale-transaction GC can end it if its decision never +// arrives (see takeRecoveredPrepared). func (s *PebbleMetaStore) recoverPreparedCDCState() error { for _, txnID := range s.cdcLog.pendingTxnIDs() { commit, err := s.readCommitRecord(txnID) @@ -289,24 +301,32 @@ func (s *PebbleMetaStore) recoverPreparedCDCState() error { if manifest == nil { continue } - if err := s.ensureRecoveredPreparedTransaction(txnID, manifest, status, err); err != nil { + immutable, err := s.ensureRecoveredPreparedTransaction(txnID, manifest, status, err) + if err != nil { return err } + if immutable == nil { + continue + } if err := s.restorePreparedDMLIntents(txnID, manifest); err != nil { return err } + s.recoveredPrepared = append(s.recoveredPrepared, *immutable) } - return nil + return s.recoverSealedPreparedTransactions() } -func (s *PebbleMetaStore) ensureRecoveredPreparedTransaction(txnID uint64, manifest *cdcSegmentTxnManifest, status TxnStatus, statusErr error) error { +// ensureRecoveredPreparedTransaction rewrites the PENDING record of a +// transaction recovered from a segment prepare record, and returns its +// immutable record, or nil when the manifest cannot rebuild one. +func (s *PebbleMetaStore) ensureRecoveredPreparedTransaction(txnID uint64, manifest *cdcSegmentTxnManifest, status TxnStatus, statusErr error) (*TxnImmutableRecord, error) { immutable, err := s.readImmutableTxnRecord(txnID) if err != nil { - return err + return nil, err } if immutable == nil { if manifest.NodeID == 0 { - return nil + return nil, nil } startTS := hlc.Timestamp{ WallTime: manifest.StartTSWall, @@ -314,21 +334,135 @@ func (s *PebbleMetaStore) ensureRecoveredPreparedTransaction(txnID uint64, manif NodeID: manifest.NodeID, } if err := s.writeImmutableTxnRecord(txnID, manifest.NodeID, startTS); err != nil { - return err + return nil, err + } + immutable = &TxnImmutableRecord{ + TxnID: txnID, + NodeID: manifest.NodeID, + StartTSWall: manifest.StartTSWall, + StartTSLogical: manifest.StartTSLogical, } } if statusErr == pebble.ErrNotFound { if err := s.db.Set(pebbleTxnStatusKey(txnID), []byte{byte(TxnStatusPending)}, pebble.NoSync); err != nil { + return nil, err + } + } + if statusErr == pebble.ErrNotFound || status == TxnStatusPending { + if err := s.db.Set(pebbleTxnPendingKey(txnID), nil, pebble.NoSync); err != nil { + return nil, err + } + } + return immutable, nil +} + +// recoverSealedPreparedTransactions recovers the prepared transactions whose +// manifest is the Pebble key alone (the default, grouped prepare sync). Pebble +// applies its writes in order, so a surviving manifest key means the immutable +// record and the PENDING status written before it survived too; a transaction +// without them was aborted, and its manifest is an orphan. +// +// Under grouped sync PREPARE may ACK before its CDC rows reach the disk, so an +// OS crash can leave a manifest whose rows are gone. Such a transaction cannot +// keep its promise to commit: it is aborted here, so its COMMIT is refused +// rather than ACKed having applied nothing. +func (s *PebbleMetaStore) recoverSealedPreparedTransactions() error { + recovered := make(map[uint64]struct{}, len(s.recoveredPrepared)) + for _, rec := range s.recoveredPrepared { + recovered[rec.TxnID] = struct{}{} + } + + prefix := []byte(pebblePrefixCDCManifest) + iter, err := s.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: prefixUpperBound(prefix), + }) + if err != nil { + return err + } + manifests := make(map[uint64]*cdcSegmentTxnManifest) + for iter.SeekGE(prefix); iter.Valid(); iter.Next() { + key := iter.Key() + if len(key) < len(pebblePrefixCDCManifest)+8 { + continue + } + txnID := binary.BigEndian.Uint64(key[len(pebblePrefixCDCManifest):]) + if _, done := recovered[txnID]; done { + continue + } + val, err := iter.ValueAndErr() + if err != nil { + _ = iter.Close() return err } - return s.db.Set(pebbleTxnPendingKey(txnID), nil, pebble.NoSync) + var manifest cdcSegmentTxnManifest + if err := encoding.Unmarshal(val, &manifest); err != nil { + _ = iter.Close() + return fmt.Errorf("decode CDC manifest txn %d: %w", txnID, err) + } + manifests[txnID] = &manifest + } + if err := iter.Close(); err != nil { + return err } - if status == TxnStatusPending { - return s.db.Set(pebbleTxnPendingKey(txnID), nil, pebble.NoSync) + + for txnID, manifest := range manifests { + status, err := s.readTxnStatus(txnID) + if err == pebble.ErrNotFound { + continue + } + if err != nil { + return err + } + if status != TxnStatusPending { + continue + } + commit, err := s.readCommitRecord(txnID) + if err != nil { + return err + } + if commit != nil { + continue + } + immutable, err := s.readImmutableTxnRecord(txnID) + if err != nil { + return err + } + if immutable == nil { + continue + } + if err := s.restorePreparedDMLIntents(txnID, manifest); err != nil { + log.Warn().Err(err).Uint64("txn_id", txnID). + Msg("Prepared transaction lost its CDC rows before they were synced; aborting it") + if err := s.abortUnrecoverablePrepared(txnID); err != nil { + return err + } + continue + } + s.recoveredPrepared = append(s.recoveredPrepared, *immutable) } return nil } +// abortUnrecoverablePrepared releases whatever row locks a failed recovery +// took and aborts the transaction, so its COMMIT finds no record. +func (s *PebbleMetaStore) abortUnrecoverablePrepared(txnID uint64) error { + s.rowLocks.ReleaseByTxn(txnID) + s.dmlIntents.Delete(txnID) + if err := s.AbortTransaction(txnID); err != nil { + return err + } + return s.DeleteCapturedRows(txnID) +} + +// takeRecoveredPrepared returns the prepared transactions recovered at open +// and forgets them; the caller owns their resolution from then on. +func (s *PebbleMetaStore) takeRecoveredPrepared() []TxnImmutableRecord { + recovered := s.recoveredPrepared + s.recoveredPrepared = nil + return recovered +} + func (s *PebbleMetaStore) restorePreparedDMLIntents(txnID uint64, manifest *cdcSegmentTxnManifest) error { cursor := newCDCSegmentCursor(s.cdcLog, manifest) defer cursor.Close() @@ -674,6 +808,18 @@ func (s *PebbleMetaStore) deleteTransactionKeys(txnID uint64, isCommitted bool) // Delete commit record if exists _ = batch.Delete(pebbleTxnCommitKey(txnID), nil) + // A prepared transaction's status stays behind as ABORTED: a segment + // prepare record outlives the abort, and recovery must not take it for a + // prepare whose Pebble keys were lost (recoverPreparedCDCState). + if _, err := s.readTxnStatus(txnID); err == nil { + if err := batch.Set(pebbleTxnStatusKey(txnID), []byte{byte(TxnStatusAborted)}, nil); err != nil { + return err + } + _ = batch.Delete(pebbleTxnPendingKey(txnID), nil) + } else if err != pebble.ErrNotFound { + return err + } + // If committed, try to remove from sequence index if isCommitted { // Try to read commit record to get seqNum before deleting @@ -973,12 +1119,15 @@ func (s *PebbleMetaStore) AbortTransaction(txnID uint64) error { batch := s.db.NewBatch() defer batch.Close() - // Delete all keys: /txn/, /txn_commit/, /txn_status/ + // Delete /txn/ and /txn_commit/; the status stays, as ABORTED, for the + // reason given in deleteTransactionKeys. if err := batch.Delete(pebbleTxnKey(txnID), nil); err != nil { return err } _ = batch.Delete(pebbleTxnCommitKey(txnID), nil) - _ = batch.Delete(pebbleTxnStatusKey(txnID), nil) + if err := batch.Set(pebbleTxnStatusKey(txnID), []byte{byte(TxnStatusAborted)}, nil); err != nil { + return err + } // Remove from pending index (best-effort cleanup) _ = batch.Delete(pebbleTxnPendingKey(txnID), nil) @@ -1097,33 +1246,23 @@ func (s *PebbleMetaStore) WriteIntent(txnID uint64, intentType IntentType, table // Try to acquire row lock existingTxnID, acquired := s.rowLocks.AcquireLock("", tableName, intentKey, txnID) if !acquired { - // Lock held by different transaction - check GC marker first - if s.rowLocks.CheckGCMarker("", tableName, intentKey) { - // GC marker exists - intent is marked for cleanup, delete marker and acquire - s.rowLocks.DeleteGCMarker("", tableName, intentKey) - s.rowLocks.ReleaseLock("", tableName, intentKey) - existingTxnID, acquired = s.rowLocks.AcquireLock("", tableName, intentKey, txnID) - if !acquired { - // Race condition - another transaction acquired the lock - telemetry.WriteConflictsTotal.With("intent", "gc_race").Inc() - return fmt.Errorf("write-write conflict: row %s:%s locked by transaction %d (current txn: %d)", - tableName, intentKey, existingTxnID, txnID) - } - } else { - // No GC marker - resolve conflict - if err := s.resolveIntentConflictPebble(nil, existingTxnID, txnID, tableName, intentKey); err != nil { - telemetry.WriteConflictsTotal.With("intent", "conflict").Inc() - return err - } - // Conflict resolved - release old lock and acquire new one - s.rowLocks.ReleaseLock("", tableName, intentKey) - existingTxnID, acquired = s.rowLocks.AcquireLock("", tableName, intentKey, txnID) - if !acquired { - // Race condition - telemetry.WriteConflictsTotal.With("intent", "resolve_race").Inc() - return fmt.Errorf("write-write conflict: row %s:%s locked by transaction %d (current txn: %d)", - tableName, intentKey, existingTxnID, txnID) - } + // Lock held by a different transaction: the holder's intent may be + // overwritten only once that transaction has ended. + if err := s.resolveIntentConflictPebble(existingTxnID, txnID, tableName, intentKey); err != nil { + telemetry.WriteConflictsTotal.With("intent", "conflict").Inc() + return err + } + // Conflict resolved - release the resolved holder's lock and acquire + // it. The release is conditional on that holder still owning the row: + // resolveIntentConflictPebble judged THAT transaction, and the verdict + // does not transfer to a newer one. + s.rowLocks.ReleaseLockIfHeldBy("", tableName, intentKey, existingTxnID) + existingTxnID, acquired = s.rowLocks.AcquireLock("", tableName, intentKey, txnID) + if !acquired { + // Race condition + telemetry.WriteConflictsTotal.With("intent", "resolve_race").Inc() + return fmt.Errorf("write-write conflict: row %s:%s locked by transaction %d (current txn: %d)", + tableName, intentKey, existingTxnID, txnID) } } @@ -1175,10 +1314,21 @@ func (s *PebbleMetaStore) WriteIntent(txnID uint64, intentType IntentType, table return nil } -// resolveIntentConflictPebble handles conflict with existing intent from different transaction. -// Called after GC marker check - if GC marker exists, caller handles overwrite directly. -func (s *PebbleMetaStore) resolveIntentConflictPebble(batch *pebble.Batch, existingTxnID, txnID uint64, tableName, intentKey string) error { - // Check conflicting transaction status +// resolveIntentConflictPebble decides whether the intent existingTxnID holds on +// a row may be overwritten by txnID. Only an ended holder yields: one with no +// transaction record, or one that committed or aborted. +// +// A PENDING holder never yields, however old its heartbeat. Participants do +// not refresh heartbeats, so every prepared transaction whose coordinator is +// slower than heartbeat_timeout_seconds looks stale, and a prepared +// transaction has promised its coordinator it can commit. Overwriting its +// intent here would leave its record pending and its COMMIT still accepted, +// so two transactions would each commit a write the other's lock was meant to +// exclude - for an AUTO_INCREMENT claim, two ranges over the same ids. A +// holder whose coordinator is gone is ended by the stale-transaction GC +// (TransactionManager.cleanupStaleTransactions -> CleanupStaleTransactions), +// which aborts it; after that this function sees an aborted or missing record. +func (s *PebbleMetaStore) resolveIntentConflictPebble(existingTxnID, txnID uint64, tableName, intentKey string) error { // Use custom txnGetter if set (allows MemoryMetaStore to inject its GetTransaction) getTxn := s.GetTransaction if s.txnGetter != nil { @@ -1186,7 +1336,6 @@ func (s *PebbleMetaStore) resolveIntentConflictPebble(batch *pebble.Batch, exist } conflictTxnRec, _ := getTxn(existingTxnID) - canOverwrite := false switch { case conflictTxnRec == nil: log.Debug(). @@ -1194,10 +1343,8 @@ func (s *PebbleMetaStore) resolveIntentConflictPebble(batch *pebble.Batch, exist Str("table", tableName). Str("intent_key", intentKey). Msg("Cleaning up orphaned intent (no transaction record)") - canOverwrite = true case conflictTxnRec.Status == TxnStatusCommitted: - canOverwrite = true case conflictTxnRec.Status == TxnStatusAborted: log.Debug(). @@ -1205,39 +1352,14 @@ func (s *PebbleMetaStore) resolveIntentConflictPebble(batch *pebble.Batch, exist Str("table", tableName). Str("intent_key", intentKey). Msg("Cleaning up intent from aborted transaction") - canOverwrite = true default: - // Check heartbeat timeout - use heartbeat from txnGetter (in-memory via MemoryMetaStore) - heartbeatTimeout := int64(10 * time.Second) - if cfg.Config != nil && cfg.Config.Transaction.HeartbeatTimeoutSeconds > 0 { - heartbeatTimeout = int64(time.Duration(cfg.Config.Transaction.HeartbeatTimeoutSeconds) * time.Second) - } - - heartbeat := conflictTxnRec.LastHeartbeat - timeSinceHeartbeat := time.Now().UnixNano() - heartbeat - if timeSinceHeartbeat > heartbeatTimeout { - log.Debug(). - Uint64("stale_txn_id", existingTxnID). - Str("table", tableName). - Str("intent_key", intentKey). - Int64("heartbeat_age_ms", timeSinceHeartbeat/1e6). - Msg("Cleaning up stale intent (heartbeat timeout)") - canOverwrite = true - } - } - - if !canOverwrite { return fmt.Errorf("write-write conflict: row %s:%s locked by transaction %d (current txn: %d)", tableName, intentKey, existingTxnID, txnID) } // Delete /intent_txn/ index for the overwritten transaction - if err := s.db.Delete(pebbleIntentByTxnKey(existingTxnID, tableName, intentKey), pebble.NoSync); err != nil { - return err - } - - return nil + return s.db.Delete(pebbleIntentByTxnKey(existingTxnID, tableName, intentKey), pebble.NoSync) } // ValidateIntent checks if the intent is still held by the expected transaction @@ -1335,20 +1457,13 @@ func (s *PebbleMetaStore) deleteIntentTxnIndexByPrefix(txnID uint64) error { return nil } -// MarkIntentsForCleanup marks all intents for a transaction as ready for overwrite. -// Uses in-memory RowLockStore.byTxn index instead of Pebble iteration. -func (s *PebbleMetaStore) MarkIntentsForCleanup(txnID uint64) error { - s.rowLocks.MarkGCByTxn(txnID) - return nil -} - // CleanupAfterCommit performs cleanup operations for a committed transaction. -// Phase 1: Marks intents with GC markers (other txns can overwrite) +// Phase 1: Releases the transaction's row locks // Phase 2: Deletes intent index entries only (NOT CDC raw entries) // Note: CDC raw entries are retained for streaming replication and deleted by GC. func (s *PebbleMetaStore) CleanupAfterCommit(txnID uint64) error { - // Single pass: mark GC, release locks, get keys (no Pebble iteration) - lockKeys := s.rowLocks.MarkGCAndRelease(txnID) + // Single pass: release locks, get keys (no Pebble iteration) + lockKeys := s.rowLocks.ReleaseByTxn(txnID) _, hadDML := s.dmlIntents.LoadAndDelete(txnID) _, hasPersisted := s.persistedIntentTxns.LoadAndDelete(txnID) if hadDML && !hasPersisted { @@ -2364,7 +2479,7 @@ func (s *PebbleMetaStore) GetCDCTableDDLLock(tableName string) (uint64, error) { } // GetRowLockStats returns statistics from the in-memory row lock store -func (s *PebbleMetaStore) GetRowLockStats() (activeLocks, activeTransactions, gcMarkers, tablesWithLocks int) { +func (s *PebbleMetaStore) GetRowLockStats() (activeLocks, activeTransactions, tablesWithLocks int) { return s.rowLocks.Stats() } diff --git a/db/prepared_intent_test.go b/db/prepared_intent_test.go new file mode 100644 index 00000000..df7781c4 --- /dev/null +++ b/db/prepared_intent_test.go @@ -0,0 +1,383 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// A prepared transaction's row locks are what make a participant's vote +// binding: for an AUTO_INCREMENT claim, the claim key's lock is what makes the +// read of the stored base and the vote on it atomic. The tests below pin that +// no other writer can take a lock from a transaction that has not ended, and +// that a claim COMMIT is refused whenever this node did not hold the claim +// from its vote to its commit. + +// ageHeartbeat makes txnID look abandoned: participants never refresh a +// heartbeat, so any prepared transaction whose coordinator is slower than +// heartbeat_timeout_seconds looks like this. +func ageHeartbeat(t *testing.T, mdb *ReplicatedDatabase, txnID uint64) { + t.Helper() + mem, ok := mdb.GetMetaStore().(*MemoryMetaStore) + require.True(t, ok, "test expects the production MemoryMetaStore") + mem.txnStore.UpdateHeartbeat(txnID, time.Now().Add(-time.Hour).UnixNano()) +} + +func prepareClaim(engine *ReplicationEngine, t *testing.T, txnID, nodeID uint64, prevBase, size uint64) *PrepareResult { + return engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, + NodeID: nodeID, + StartTS: hlc.Timestamp{WallTime: int64(txnID)}, + Database: "testdb", + Statements: []protocol.Statement{claimStatement(t, "testdb", "users", prevBase, prevBase, size)}, + }) +} + +func commitClaim(engine *ReplicationEngine, txnID uint64) *CommitResult { + return engine.Commit(context.Background(), &CommitRequest{ + TxnID: txnID, + Database: "testdb", + Statements: []protocol.Statement{commitStatement("testdb", "users")}, + }) +} + +// TestPreparedClaimIsNotEvictedByAStaleHeartbeat: claimant A prepares, its +// coordinator stalls past the heartbeat timeout, and claimant B proposes the +// same range. B used to evict A's lock, pass PREPARE on the base A had also +// read, and commit; A's COMMIT then found no claim intent and ACKed anyway, so +// both were granted ids 501..510 on this node. +// +// Mutation: in resolveIntentConflictPebble, let a PENDING holder yield once its +// heartbeat is older than heartbeat_timeout_seconds. "a second claimant was +// granted the claim key while a prepared claim holds it" fires. +func TestPreparedClaimIsNotEvictedByAStaleHeartbeat(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 500, 1) + + const a, b, retry = 9100, 9101, 9102 + prepA := prepareClaim(engine, t, a, 1, 500, 10) + require.True(t, prepA.Success, "A prepare: %s", prepA.Error) + ageHeartbeat(t, mdb, a) + + prepB := prepareClaim(engine, t, b, 2, 500, 20) + require.False(t, prepB.Success, "a second claimant was granted the claim key while a prepared claim holds it") + require.False(t, prepB.Rejected, "the node cast a verdict on a claim whose lock it never held") + require.Contains(t, prepB.Error, "write-write conflict") + + commitA := commitClaim(engine, a) + require.True(t, commitA.Success, "A's COMMIT: %s", commitA.Error) + require.Equal(t, uint64(510), readClaimBase(t, dm, "testdb", "users")) + + // B's retry from its stale view is now rejected with the base to retry above. + prepRetry := prepareClaim(engine, t, retry, 2, 500, 20) + require.True(t, prepRetry.Rejected, "a stale proposal was accepted after A committed") + require.Equal(t, uint64(510), prepRetry.AutoIDStoredBase) +} + +// TestPreparedDMLIsNotEvictedByAStaleHeartbeat is the same property for an +// ordinary write. B used to evict A's row lock and both COMMITs were ACKed, so +// the write-write conflict the lock exists to report was never reported. +// +// Mutation: the one named on TestPreparedClaimIsNotEvictedByAStaleHeartbeat. +// "a second writer was granted a row a prepared transaction holds" fires. +func TestPreparedDMLIsNotEvictedByAStaleHeartbeat(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE log (id INTEGER PRIMARY KEY, v TEXT)") + + prepareRow := func(txnID uint64, v string) *PrepareResult { + return engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: int64(txnID)}, + Database: "testdb", + Statements: []protocol.Statement{dmlLogInsertStatement("testdb", "log:1", 1, v)}, + }) + } + + const a, b = 9200, 9201 + prepA := prepareRow(a, "a") + require.True(t, prepA.Success, "A prepare: %s", prepA.Error) + ageHeartbeat(t, mdb, a) + + prepB := prepareRow(b, "b") + require.False(t, prepB.Success, "a second writer was granted a row a prepared transaction holds") + require.Contains(t, prepB.Error, "write-write conflict") + + commitA := engine.Commit(context.Background(), &CommitRequest{ + TxnID: a, + Database: "testdb", + Statements: []protocol.Statement{dmlLogCommitStatement("testdb", "log:1")}, + }) + require.True(t, commitA.Success, "A's COMMIT: %s", commitA.Error) + var v string + require.NoError(t, mdb.GetReadDB().QueryRow("SELECT v FROM log WHERE id = 1").Scan(&v)) + require.Equal(t, "a", v) +} + +// TestAbortedClaimLeavesNothingThatGrantsTheKeyTwice: A claims and aborts, B +// claims and holds, and C must then lose the race for the claim key rather +// than be granted the range B holds. +// +// Mutation: in resolveIntentConflictPebble, let a PENDING holder yield. "a +// second claimant was granted a range while another holds the claim key" +// fires. +func TestAbortedClaimLeavesNothingThatGrantsTheKeyTwice(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 500, 1) + + require.True(t, prepareClaim(engine, t, 8300, 1, 500, 10).Success) + abort := engine.Abort(context.Background(), &AbortRequest{TxnID: 8300, Database: "testdb"}) + require.True(t, abort.Success, "abort failed: %s", abort.Error) + + prepB := prepareClaim(engine, t, 8301, 2, 500, 10) + require.True(t, prepB.Success, "PREPARE refused a well-formed claim: %s", prepB.Error) + + prepC := prepareClaim(engine, t, 8302, 3, 500, 10) + require.False(t, prepC.Success, "a second claimant was granted a range while another holds the claim key") + require.False(t, prepC.Rejected, "the node cast a verdict on a claim whose lock it never held") + require.Contains(t, prepC.Error, "write-write conflict") + + res := commitClaim(engine, 8301) + require.True(t, res.Success, "COMMIT failed: %s", res.Error) + require.Equal(t, uint64(510), readClaimBase(t, dm, "testdb", "users")) +} + +// TestCommitRefusesAClaimWhoseIntentIsGone: a COMMIT carrying the claim flag +// on a node that holds no claim intent for the transaction must not be ACKed. +// Nothing was applied, so an ACK would count this node toward a commit quorum +// for a range it never recorded. +// +// Mutation: make ApplyClaims return nil when the transaction holds no claim +// intent. "a claim COMMIT was ACKed with no claim intent to apply" fires. +func TestCommitRefusesAClaimWhoseIntentIsGone(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 500, 1) + + const a = 9300 + require.True(t, prepareClaim(engine, t, a, 1, 500, 10).Success) + // The intent is gone while the transaction record stays pending. + require.NoError(t, mdb.GetMetaStore().DeleteIntentsByTxn(a)) + + res := commitClaim(engine, a) + require.False(t, res.Success, "a claim COMMIT was ACKed with no claim intent to apply") + require.Contains(t, res.Error, ErrAutoIncClaimNotApplicable.Error()) + require.Equal(t, uint64(500), readClaimBase(t, dm, "testdb", "users")) +} + +// TestCommitRefusesAClaimOnceTheBaseMovedPastItsPrevBase: a DDL seed raised +// the base between the claim's PREPARE and its COMMIT. Writing the claim's +// newBase+size would lower the base, and the range it hands out sits below the +// new floor, so the COMMIT must be refused and the base left where the seed +// put it. +// +// Mutation: drop "AND base <= ?" from applyClaimTx's UPDATE. The COMMIT is +// ACKed and "the claim lowered a base a seed had raised" fires. +func TestCommitRefusesAClaimOnceTheBaseMovedPastItsPrevBase(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 500, 1) + + const a = 9400 + require.True(t, prepareClaim(engine, t, a, 1, 500, 10).Success) + seedClaimBase(t, dm, "testdb", "users", 800, 2) + + res := commitClaim(engine, a) + require.Equal(t, uint64(800), readClaimBase(t, dm, "testdb", "users"), "the claim lowered a base a seed had raised") + require.False(t, res.Success, "a claim below the raised floor was ACKed") + require.Contains(t, res.Error, ErrAutoIncClaimNotApplicable.Error()) +} + +// intentsFrom is a MetaStore whose GetIntentsByTxn replays intents captured +// earlier, standing in for a COMMIT that read its claim intent just before the +// intent was released. +type intentsFrom struct { + MetaStore + intents []*WriteIntentRecord +} + +func (m intentsFrom) GetIntentsByTxn(uint64) ([]*WriteIntentRecord, error) { + return m.intents, nil +} + +// TestApplyClaimsACKsAtMostOneOfTwoOverlappingClaims is the per-node half of +// the argument that two overlapping ranges can never both commit: A read its +// claim intent at COMMIT, then lost its lock (a stale-transaction GC abort +// racing the COMMIT), and B prepared and committed the same range. A's apply +// must now be refused, so this node ACKs one of the two and never both; two +// majorities that each ACKed one would have to share a node that ACKed both. +// +// Mutation: drop "AND base <= ?" from applyClaimTx's UPDATE. A's apply +// succeeds and "this node applied two overlapping claims" fires. +func TestApplyClaimsACKsAtMostOneOfTwoOverlappingClaims(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 500, 1) + + const a, b = 9500, 9501 + require.True(t, prepareClaim(engine, t, a, 1, 500, 10).Success) + captured, err := mdb.GetMetaStore().GetIntentsByTxn(a) + require.NoError(t, err) + require.Len(t, captured, 1) + + abort := engine.Abort(context.Background(), &AbortRequest{TxnID: a, Database: "testdb"}) + require.True(t, abort.Success, "abort failed: %s", abort.Error) + require.True(t, prepareClaim(engine, t, b, 2, 500, 20).Success) + require.True(t, commitClaim(engine, b).Success) + + err = autoIncClaimStoreForTest(dm).ApplyClaims("testdb", a, intentsFrom{MetaStore: mdb.GetMetaStore(), intents: captured}) + require.True(t, errors.Is(err, ErrAutoIncClaimNotApplicable), "this node applied two overlapping claims: err=%v", err) + require.Equal(t, uint64(520), readClaimBase(t, dm, "testdb", "users")) +} + +// TestApplyClaimsIsConditionedOnTheRangeItWrites: the COMMIT's conditional +// write guards the range it hands out, newBase+1..newBase+size, so it must not +// depend on PREPARE having admitted only newBase == prevBase. Here the intent +// the COMMIT reads carries a newBase below the stored base (500): applying it +// would lower the base to 110 and re-issue ids 101..110. +// +// Mutation: condition the UPDATE on the claim's prevBase instead of its +// newBase. The apply succeeds and "a claim whose range starts below the stored +// base was applied" fires. +func TestApplyClaimsIsConditionedOnTheRangeItWrites(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 500, 1) + + const a = 9550 + require.True(t, prepareClaim(engine, t, a, 1, 500, 10).Success) + captured, err := mdb.GetMetaStore().GetIntentsByTxn(a) + require.NoError(t, err) + require.Len(t, captured, 1) + low := *captured[0] + low.DataSnapshot, err = protocol.EncodeAutoIncClaim(protocol.AutoIncClaim{Table: "users", PrevBase: 500, NewBase: 100, Size: 10}) + require.NoError(t, err) + + err = autoIncClaimStoreForTest(dm).ApplyClaims("testdb", a, intentsFrom{MetaStore: mdb.GetMetaStore(), intents: []*WriteIntentRecord{&low}}) + require.True(t, errors.Is(err, ErrAutoIncClaimNotApplicable), "a claim whose range starts below the stored base was applied: err=%v", err) + require.Equal(t, uint64(500), readClaimBase(t, dm, "testdb", "users")) +} + +// TestDeadCoordinatorsPreparedRowIsFreedByTheStaleTransactionGC is the +// liveness side of TestPreparedDMLIsNotEvictedByAStaleHeartbeat. A second +// writer no longer evicts a prepared transaction whose coordinator has gone +// quiet; it waits for the stale-transaction GC to abort it. The GC judges +// staleness by the transaction's LastHeartbeat, which is set at BEGIN and +// never refreshed for a participant, so a prepared transaction is aborted by +// the first GC pass that runs once it is heartbeat_timeout old - at most +// heartbeat_timeout + gc_interval after its PREPARE. The pass is driven +// directly here so the test does not race the GC goroutine. +// +// Mutation: make MemoryMetaStore.CleanupStaleTransactions never treat a +// pending transaction as stale. "the GC did not abort a prepared transaction +// past heartbeat_timeout" fires. (Dropping only its DeleteIntentsByTxn does +// not strand the row: once the record is aborted, the next writer's conflict +// resolution overwrites the intent.) +func TestDeadCoordinatorsPreparedRowIsFreedByTheStaleTransactionGC(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE log (id INTEGER PRIMARY KEY, v TEXT)") + tm := mdb.GetTransactionManager() + tm.StopGarbageCollection() + const heartbeatTimeout = 200 * time.Millisecond + tm.heartbeatTimeout = heartbeatTimeout + + prepareRow := func(txnID uint64, v string) *PrepareResult { + return engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: int64(txnID)}, + Database: "testdb", + Statements: []protocol.Statement{dmlLogInsertStatement("testdb", "log:1", 1, v)}, + }) + } + + const dead, b1, b2 = 9600, 9601, 9602 + prepared := time.Now() + require.True(t, prepareRow(dead, "dead").Success) + require.False(t, prepareRow(b1, "b").Success, "a live prepared row was taken") + + // A pass before the timeout leaves the prepared transaction alone. + cleaned, err := tm.cleanupStaleTransactions() + require.NoError(t, err) + require.Zero(t, cleaned, "the GC aborted a prepared transaction younger than heartbeat_timeout") + + time.Sleep(heartbeatTimeout - time.Since(prepared) + 50*time.Millisecond) + cleaned, err = tm.cleanupStaleTransactions() + require.NoError(t, err) + require.Equal(t, 1, cleaned, "the GC did not abort a prepared transaction past heartbeat_timeout") + + prepB := prepareRow(b2, "b") + require.True(t, prepB.Success, "the GC abort did not free the row: %s", prepB.Error) + t.Logf("row freed %v after the dead coordinator's PREPARE (heartbeat_timeout %v, GC pass driven directly)", + time.Since(prepared).Round(time.Millisecond), heartbeatTimeout) + + // The aborted transaction's COMMIT, should its coordinator return, is refused. + res := engine.Commit(context.Background(), &CommitRequest{ + TxnID: dead, + Database: "testdb", + Statements: []protocol.Statement{dmlLogCommitStatement("testdb", "log:1")}, + }) + require.False(t, res.Success, "a GC-aborted transaction was committed") +} + +// TestCommitRefusesADMLTransactionWhoseRowsTheGCDeleted replays a COMMIT +// racing the stale-transaction GC on one node. The COMMIT has loaded the +// transaction while it was still pending (engine.Commit -> +// TransactionManager.GetTransaction); the GC then deletes the transaction's +// intents and captured rows (MemoryMetaStore.CleanupStaleTransactions, in its +// own order), and the COMMIT runs before the GC's abort reaches the record. +// The COMMIT used to find no rows, take the statement branch, write a commit +// record and ACK a write it never applied. +// +// Mutation: drop the statementsNamePreparedRows refusal in CommitTransaction. +// "a COMMIT whose prepared rows the GC deleted was ACKed" fires. +func TestCommitRefusesADMLTransactionWhoseRowsTheGCDeleted(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE log (id INTEGER PRIMARY KEY, v TEXT)") + tm := mdb.GetTransactionManager() + tm.StopGarbageCollection() + + const txnID = 9740 + prep := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: txnID}, + Database: "testdb", + Statements: []protocol.Statement{dmlLogInsertStatement("testdb", "log:1", 1, "gc")}, + }) + require.True(t, prep.Success, "prepare: %s", prep.Error) + + txn := tm.GetTransaction(txnID) + require.NotNil(t, txn) + txn.Statements = []protocol.Statement{dmlLogCommitStatement("testdb", "log:1")} + + ms := mdb.GetMetaStore() + require.NoError(t, ms.DeleteIntentsByTxn(txnID)) + require.NoError(t, ms.DeleteIntentEntries(txnID)) + + err := tm.CommitTransaction(txn) + require.ErrorIs(t, err, ErrPreparedRowsMissing, "a COMMIT whose prepared rows the GC deleted was ACKed") + + var n int + require.NoError(t, mdb.GetReadDB().QueryRow("SELECT COUNT(*) FROM log").Scan(&n)) + require.Zero(t, n) +} diff --git a/db/prepared_recovery_test.go b/db/prepared_recovery_test.go new file mode 100644 index 00000000..8019c601 --- /dev/null +++ b/db/prepared_recovery_test.go @@ -0,0 +1,329 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/cockroachdb/pebble" + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// A participant that restarts while it holds a prepared transaction gets the +// transaction's row locks and CDC rows back from its meta store. The tests +// below pin that such a transaction stays resolvable: its COMMIT applies the +// rows it prepared, its ABORT ends it for good, and if no decision ever +// arrives the stale-transaction GC ends it. Each runs under both prepare sync +// modes, because each recovers the transaction from a different place: strict +// sync from the segment log's prepare record, grouped sync from the Pebble +// manifest key. + +var prepareSyncModes = []struct { + name string + env string +}{ + {name: "grouped", env: ""}, + {name: "strict", env: "strict"}, +} + +func prepareLogRow(engine *ReplicationEngine, txnID uint64, v string) *PrepareResult { + return engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: int64(txnID)}, + Database: "testdb", + Statements: []protocol.Statement{dmlLogInsertStatement("testdb", "log:1", 1, v)}, + }) +} + +func commitLogRow(engine *ReplicationEngine, txnID uint64) *CommitResult { + return engine.Commit(context.Background(), &CommitRequest{ + TxnID: txnID, + Database: "testdb", + Statements: []protocol.Statement{dmlLogCommitStatement("testdb", "log:1")}, + }) +} + +// restartNode closes dm and opens a new DatabaseManager and engine on the same +// data directory, as a node restart does: every meta store is reopened through +// NewMetaStore, which runs Pebble recovery and ReconstructFromPebble. +func restartNode(t *testing.T, dm *DatabaseManager) (*ReplicationEngine, *DatabaseManager) { + t.Helper() + dir := dm.GetDataDir() + require.NoError(t, dm.Close()) + clock := hlc.NewClock(1) + restarted, err := NewDatabaseManager(dir, 1, clock) + require.NoError(t, err) + t.Cleanup(func() { _ = restarted.Close() }) + return NewReplicationEngine(1, restarted, clock), restarted +} + +// restartedLogDB prepares txnID on log:1 and then restarts the node. The +// returned database's GC goroutine is stopped so a test drives passes itself. +func restartedLogDB(t *testing.T, syncEnv string, txnID uint64, v string) (*ReplicationEngine, *DatabaseManager, *ReplicatedDatabase) { + t.Helper() + t.Setenv("MARMOT_CDC_PREPARE_SYNC", syncEnv) + engine, dm, cleanup := setupTestReplicationEngine(t) + t.Cleanup(cleanup) + markedTableDB(t, engine, dm, "CREATE TABLE log (id INTEGER PRIMARY KEY, v TEXT)") + + prep := prepareLogRow(engine, txnID, v) + require.True(t, prep.Success, "prepare: %s", prep.Error) + + engine, dm = restartNode(t, dm) + mdb, err := dm.GetDatabase("testdb") + require.NoError(t, err) + mdb.GetTransactionManager().StopGarbageCollection() + return engine, dm, mdb +} + +// TestRecoveredPreparedRowIsFreedByTheStaleTransactionGC: a node restarts +// holding a prepared DML transaction whose coordinator never returns. Recovery +// re-locks its row, so a second writer is refused until the stale-transaction +// GC aborts it - which the GC can only do if recovery registered it where the +// GC looks. The GC judges the transaction by a heartbeat of the restart time, +// so the row is freed at most heartbeat_timeout + gc_interval after the +// restart. The passes are driven directly so the test does not race the GC +// goroutine. +// +// Mutation: skip the registration in ReconstructFromPebble. "the +// stale-transaction GC never saw the recovered prepared transaction" fires. +func TestRecoveredPreparedRowIsFreedByTheStaleTransactionGC(t *testing.T) { + for _, mode := range prepareSyncModes { + t.Run(mode.name, func(t *testing.T) { + const dead, b1, b2 = 9700, 9701, 9702 + engine, _, mdb := restartedLogDB(t, mode.env, dead, "dead") + restarted := time.Now() + tm := mdb.GetTransactionManager() + const heartbeatTimeout = 200 * time.Millisecond + tm.heartbeatTimeout = heartbeatTimeout + + require.False(t, prepareLogRow(engine, b1, "b").Success, + "the restart dropped the prepared transaction's row lock") + + cleaned, err := tm.cleanupStaleTransactions() + require.NoError(t, err) + require.Zero(t, cleaned, "the GC aborted a recovered transaction younger than heartbeat_timeout") + + time.Sleep(heartbeatTimeout - time.Since(restarted) + 50*time.Millisecond) + cleaned, err = tm.cleanupStaleTransactions() + require.NoError(t, err) + require.Equal(t, 1, cleaned, "the stale-transaction GC never saw the recovered prepared transaction") + + prepB := prepareLogRow(engine, b2, "b") + require.True(t, prepB.Success, "the GC abort did not free the recovered row: %s", prepB.Error) + t.Logf("row freed %v after the restart (heartbeat_timeout %v, GC pass driven directly)", + time.Since(restarted).Round(time.Millisecond), heartbeatTimeout) + + require.False(t, commitLogRow(engine, dead).Success, "a GC-aborted recovered transaction was committed") + }) + } +} + +// TestRecoveredPreparedRowIsFreedWithinTheGCBound measures the liveness bound +// with the GC goroutine itself rather than driven passes: a writer blocked by +// a recovered transaction whose decision never arrives gets the row no later +// than heartbeat_timeout + gc_interval after the restart (plus the time a pass +// and a PREPARE take), because the GC's first tick past heartbeat_timeout +// aborts it. +func TestRecoveredPreparedRowIsFreedWithinTheGCBound(t *testing.T) { + const dead = 9705 + engine, _, mdb := restartedLogDB(t, "", dead, "dead") + restarted := time.Now() + tm := mdb.GetTransactionManager() + const heartbeatTimeout, gcInterval, slack = 200 * time.Millisecond, 100 * time.Millisecond, 500 * time.Millisecond + tm.heartbeatTimeout = heartbeatTimeout + tm.gcInterval = gcInterval + tm.StartGarbageCollection() + defer tm.StopGarbageCollection() + + var freedAfter time.Duration + for txnID := uint64(dead + 1); time.Since(restarted) < 10*time.Second; txnID++ { + if prepareLogRow(engine, txnID, "b").Success { + freedAfter = time.Since(restarted) + break + } + time.Sleep(10 * time.Millisecond) + } + require.NotZero(t, freedAfter, "the recovered row was never freed") + t.Logf("row freed %v after the restart by the GC goroutine (heartbeat_timeout %v, gc_interval %v)", + freedAfter.Round(time.Millisecond), heartbeatTimeout, gcInterval) + require.Less(t, freedAfter, heartbeatTimeout+gcInterval+slack, "the recovered row outlived heartbeat_timeout + gc_interval") +} + +// TestRecoveredPreparedTransactionCommitsItsRows: the COMMIT of a transaction +// prepared before a restart applies the row it prepared. ReconstructFromPebble +// used to treat every transaction without a commit record as an orphan and +// delete its CDC rows, so this COMMIT found nothing, committed nothing, and +// ACKed. +// +// Mutation: in ReconstructFromPebble, delete the rows of recovered +// transactions too. "the COMMIT of a recovered transaction was refused" +// fires, because COMMIT refuses a DML transaction whose rows are gone; with +// that refusal also removed, "the recovered transaction's row was not +// applied" fires. +func TestRecoveredPreparedTransactionCommitsItsRows(t *testing.T) { + for _, mode := range prepareSyncModes { + t.Run(mode.name, func(t *testing.T) { + const txnID = 9710 + engine, _, mdb := restartedLogDB(t, mode.env, txnID, "prepared") + + res := commitLogRow(engine, txnID) + require.True(t, res.Success, "the COMMIT of a recovered transaction was refused: %s", res.Error) + + var v string + err := mdb.GetReadDB().QueryRow("SELECT v FROM log WHERE id = 1").Scan(&v) + require.NoError(t, err, "the recovered transaction's row was not applied") + require.Equal(t, "prepared", v) + + require.True(t, prepareLogRow(engine, txnID+1, "next").Success, + "the committed recovered transaction kept its row lock") + }) + } +} + +// TestRecoveredPreparedTransactionEndsOnItsAbort: the coordinator's ABORT +// for a transaction recovered at restart finds it pending, and ending it frees +// its row at once, without waiting for the stale-transaction GC. +// +// Mutation: make ReplicationEngine.Abort skip TransactionManager.AbortTransaction +// for a user database. The recovered transaction stays pending and "the ABORT +// of a recovered transaction left its row locked" fires. (Dropping only the +// abort's DeleteIntentsByTxn does not strand the row: once the record is +// gone, the next writer's conflict resolution overwrites the intent.) +func TestRecoveredPreparedTransactionEndsOnItsAbort(t *testing.T) { + for _, mode := range prepareSyncModes { + t.Run(mode.name, func(t *testing.T) { + const txnID = 9715 + engine, _, _ := restartedLogDB(t, mode.env, txnID, "aborted") + + require.True(t, engine.Abort(context.Background(), &AbortRequest{TxnID: txnID, Database: "testdb"}).Success) + prep := prepareLogRow(engine, txnID+1, "next") + require.True(t, prep.Success, "the ABORT of a recovered transaction left its row locked: %s", prep.Error) + require.False(t, commitLogRow(engine, txnID).Success, "an aborted recovered transaction was committed") + }) + } +} + +// TestPreparedTransactionSurvivesADetachedRestore: a snapshot restore detaches +// the database while a transaction is prepared on it. A COMMIT that arrives +// in the window is refused, never ACKed against the file being replaced; the +// meta store stays open across the window, so after the reattach the +// transaction is still prepared and its COMMIT applies its row to the file +// now in place. +// +// Mutation: make DetachDatabase leave the database in service. "a COMMIT was +// ACKed while its database was detached" fires. +func TestPreparedTransactionSurvivesADetachedRestore(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE log (id INTEGER PRIMARY KEY, v TEXT)") + + const txnID = 9725 + require.True(t, prepareLogRow(engine, txnID, "kept").Success) + require.NoError(t, dm.DetachDatabase(context.Background(), "testdb")) + require.False(t, commitLogRow(engine, txnID).Success, "a COMMIT was ACKed while its database was detached") + require.NoError(t, dm.AttachDatabase("testdb")) + + res := commitLogRow(engine, txnID) + require.True(t, res.Success, "the COMMIT after the reattach was refused: %s", res.Error) + mdb, err := dm.GetDatabase("testdb") + require.NoError(t, err) + var v string + require.NoError(t, mdb.GetReadDB().QueryRow("SELECT v FROM log WHERE id = 1").Scan(&v)) + require.Equal(t, "kept", v) +} + +// TestAbortedPreparedTransactionStaysAbortedAcrossARestart: under strict sync +// the segment prepare record outlives an ABORT, and recovery reads a prepare +// record whose Pebble status is gone as a prepare whose unsynced keys were +// lost. The abort therefore leaves an ABORTED status behind; without it, every +// restart would bring the aborted transaction and its row locks back. +// +// Mutation: delete the status key on abort instead of writing ABORTED. "an +// aborted transaction's row lock came back after the restart" fires (strict). +func TestAbortedPreparedTransactionStaysAbortedAcrossARestart(t *testing.T) { + for _, mode := range prepareSyncModes { + t.Run(mode.name, func(t *testing.T) { + t.Setenv("MARMOT_CDC_PREPARE_SYNC", mode.env) + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE log (id INTEGER PRIMARY KEY, v TEXT)") + + const aborted, next = 9720, 9721 + require.True(t, prepareLogRow(engine, aborted, "aborted").Success) + require.True(t, engine.Abort(context.Background(), &AbortRequest{TxnID: aborted, Database: "testdb"}).Success) + + engine, dm = restartNode(t, dm) + mdb, err := dm.GetDatabase("testdb") + require.NoError(t, err) + mdb.GetTransactionManager().StopGarbageCollection() + + prep := prepareLogRow(engine, next, "next") + require.True(t, prep.Success, "an aborted transaction's row lock came back after the restart: %s", prep.Error) + require.False(t, commitLogRow(engine, aborted).Success, "an aborted transaction was committed after a restart") + }) + } +} + +// TestPreparedTransactionWhoseRowsWereLostIsAbortedAtRecovery: under grouped +// sync, PREPARE can ACK before its CDC rows are on disk, so an OS crash can +// leave the Pebble manifest without the rows it points at. Recovery cannot +// keep that transaction's promise; it aborts it and releases its locks, so a +// COMMIT is refused instead of ACKed having applied nothing, and the node +// still opens. +// +// Mutation: in recoverSealedPreparedTransactions, skip the transaction on a +// row read error instead of aborting it. "a transaction whose prepared rows +// were lost is still pending" fires. +func TestPreparedTransactionWhoseRowsWereLostIsAbortedAtRecovery(t *testing.T) { + t.Setenv("MARMOT_CDC_PREPARE_SYNC", "") + dir := t.TempDir() + opts := PebbleMetaStoreOptions{CacheSizeMB: 8, MemTableSizeMB: 4, MemTableCount: 2} + store, err := NewPebbleMetaStore(dir, opts) + require.NoError(t, err) + + const txnID uint64 = 9730 + require.NoError(t, store.BeginTransaction(txnID, 1, hlc.Timestamp{WallTime: 1, NodeID: 1})) + row, err := EncodeRow(&EncodedCapturedRow{ + Table: "log", + Op: uint8(OpTypeInsert), + IntentKey: []byte("log:1"), + NewValues: map[string][]byte{"id": []byte("1")}, + }) + require.NoError(t, err) + require.NoError(t, store.WriteCapturedRow(txnID, 1, row)) + require.NoError(t, store.DurablyPrepareTransaction(txnID)) + require.NoError(t, store.Close()) + + // The OS crash: the segment's rows never reached the disk. + segments, err := filepath.Glob(filepath.Join(dir, cdcSegmentDirName, "seg-*.log")) + require.NoError(t, err) + require.NotEmpty(t, segments) + for _, seg := range segments { + require.NoError(t, os.Truncate(seg, 0)) + } + + recovered, err := NewPebbleMetaStore(dir, opts) + require.NoError(t, err, "a lost prepared row must not stop the node from opening") + defer recovered.Close() + + rec, err := recovered.GetTransaction(txnID) + require.NoError(t, err) + require.Nil(t, rec, "a transaction whose prepared rows were lost is still pending") + require.Empty(t, recovered.takeRecoveredPrepared()) + _, locked := recovered.rowLocks.CheckLock("", "log", "log:1") + require.False(t, locked, "a transaction whose prepared rows were lost kept its row lock") + _, closer, err := recovered.db.Get(pebbleCDCManifestKey(txnID)) + if err == nil { + closer.Close() + } + require.ErrorIs(t, err, pebble.ErrNotFound, "the lost transaction's manifest was kept") +} diff --git a/db/replication_engine.go b/db/replication_engine.go index 7e0ff789..5f76af45 100644 --- a/db/replication_engine.go +++ b/db/replication_engine.go @@ -38,6 +38,19 @@ type PrepareResult struct { // SQLite cannot apply. Infrastructure failures (timeouts, storage errors) leave // it false so the coordinator keeps treating them as a missing ACK. Rejected bool + // AutoIDStoredBase is this participant's own committed base for the table a + // rejected AUTO_INCREMENT range claim named, so the claimant can retry above + // it rather than spin. Zero on every response that is not such a rejection. + AutoIDStoredBase uint64 + // ErrorCode is the MySQL server error code this rejection must reach the + // client with, when the underlying error named one (a + // transform.CodedError, for example the width-ceiling refusal's + // ER_WARN_DATA_OUT_OF_RANGE). Zero means "not supplied": the coordinator + // then falls back to classifying the message, as it did before. It exists + // because Error is a string - the typed error cannot survive the hop to + // the coordinator, local or remote, and without the code every + // deterministic refusal reaches the client as 1105 HY000. + ErrorCode uint16 } // CommitRequest contains parameters for the commit phase @@ -109,6 +122,19 @@ func (re *ReplicationEngine) isDatabaseOperation(statements []protocol.Statement return stmt.Type == protocol.StatementCreateDatabase || stmt.Type == protocol.StatementDropDatabase } +// statementsCarryAutoIDClaim reports whether a transaction carries an +// AUTO_INCREMENT range claim. It reads the flag only, never the payload: the +// COMMIT message is decision metadata, and the claim's values are read from the +// intent this node wrote at PREPARE. +func statementsCarryAutoIDClaim(statements []protocol.Statement) bool { + for _, stmt := range statements { + if stmt.AutoIDClaim { + return true + } + } + return false +} + // prepareDatabaseOperation handles CREATE/DROP DATABASE operations using system database func (re *ReplicationEngine) prepareDatabaseOperation(req *PrepareRequest) *PrepareResult { stmt := req.Statements[0] @@ -198,7 +224,8 @@ func (re *ReplicationEngine) prepareRegularTransaction(ctx context.Context, req Uint64("node_id", re.nodeID). Bool("rejected", rejected). Msg("DDL validation failed during PREPARE") - return &PrepareResult{Success: false, Error: err.Error(), Rejected: rejected} + return &PrepareResult{Success: false, Error: err.Error(), Rejected: rejected, + ErrorCode: mysqlCodeForError(err)} } } @@ -212,6 +239,19 @@ func (re *ReplicationEngine) prepareRegularTransaction(ctx context.Context, req for _, stmt := range req.Statements { stmtSeq++ + // An AUTO_INCREMENT range claim is evaluated here, before + // processStatement, because it needs replicatedDB - this node's own + // SQLite - and because it must not reach either the empty-IntentKey + // gate or createDMLIntent's row-image check, both of which are written + // for ordinary DML. + if stmt.AutoIDClaim { + if result := re.prepareAutoIncClaim(replicatedDB, txn, txnMgr, metaStore, stmt, req); result != nil { + _ = txnMgr.AbortTransaction(txn) + return result + } + continue + } + if result := re.processStatement(txn, txnMgr, metaStore, stmt, req, stmtSeq, adoptCapturedRows); result != nil { return result } @@ -225,6 +265,167 @@ func (re *ReplicationEngine) prepareRegularTransaction(ctx context.Context, req return &PrepareResult{Success: true} } +// autoIncClaimStore resolves the AUTO_INCREMENT claim store, which is always +// backed by the system database rather than req.Database (db/autoinc_claim.go): +// a claim applied inside a pinned session on the user database would try to +// take that database's single SQLite writer a second time and block on its +// own BEGIN. Callers must reject the PREPARE / refuse the COMMIT on error +// rather than silently skip the claim. +func (re *ReplicationEngine) autoIncClaimStore() (*AutoIncClaimStore, error) { + systemDB, err := re.dbMgr.GetDatabase(SystemDatabaseName) + if err != nil { + return nil, fmt.Errorf("system database unavailable: %w", err) + } + return NewAutoIncClaimStore(systemDB), nil +} + +// prepareAutoIncClaim evaluates an AUTO_INCREMENT range claim on this +// participant and, on acceptance, writes the claim intent whose DataSnapshot +// the COMMIT handler will apply. It returns nil to accept and a failed +// PrepareResult to reject. +// +// The condition, evaluated against this node's OWN committed state: +// +// storedBase <= prevBase && newBase == prevBase && size >= 1 && newBase+size <= widthMax +// +// The second term is not "newBase > prevBase": a claim proposes its own view +// of the base as newBase, so that form could never hold and no claim would +// ever be accepted. The quantity that increases monotonically is the +// COMMITTED base, which COMMIT sets to newBase + size, and size >= 1. +// +// The term is "==" rather than ">=" on purpose: allocation is lowest-free, so a +// claimant never proposes above its own view of the base. A claimant that wants +// a higher base gets one only by retrying with the base a participant returned, +// which is what keeps a single-node cluster minting 1, 2, 3 with no jump at any +// range boundary. +// +// and an ABSENT stored base REJECTS. That last clause is the protocol's whole +// safety argument and it is the one an implementer will get wrong: the natural +// implementation is "absent means 0 means yes", and a cache that defaults to 0 +// and accepts is precisely the thing that cannot cast the rejection that would +// repair it. +// +// widthMax is derived from THIS node's own sqlite_master, never taken from the +// claimant: the width marker rides in the replicated DDL text so every node +// derives it identically, and trusting the claimant would let a node with a +// stale schema be rubber-stamped. +// +// Soundness: any committed claim reached a majority, this claim needs a +// majority, and majorities intersect, so at least one participant holds a +// storedBase above prevBase and rejects. A rejection returns this node's own +// base so the claimant can retry above it rather than spin. +// +// That argument only holds if a participant's read of its own base and its +// vote on that read cannot straddle another claimant's commit. The intent +// this function writes is what makes them atomic: MetaStore.WriteIntent takes +// the per-(table, intentKey) row lock and holds it until the transaction +// commits or aborts (db/meta_store_pebble.go WriteIntent -> rowLocks. +// AcquireLock; released by DeleteIntentsByTxn / ReleaseByTxn on abort and by +// CommitTransaction on commit; a pending holder is never overwritten by +// another writer, however old its heartbeat - see resolveIntentConflictPebble), +// and every claim for one table shares one intent key +// (protocol.AutoIncClaimKey). The COMMIT handler writes the new base BEFORE +// it marks the transaction committed (Commit's ApplyClaims call runs ahead of +// txnMgr.CommitTransaction), so the base is durable while the lock is still +// held. ApplyClaims also refuses a COMMIT whose claim intent is gone or whose +// prevBase the stored base has passed, so an ACK never depends on the lock +// alone (see ApplyClaims for the per-node disjointness argument). +// +// The intent is therefore written FIRST and the base read SECOND. Read first +// and the read is unlocked: two claimants can both read the pre-commit base, +// both find themselves in range, and both be granted the same ids - a +// check-then-act gap that -race cannot see, because each half is individually +// synchronised and only the pair is not. A claimant that loses the race for +// the lock gets a write-write conflict rather than a rejection, which the +// coordinator surfaces as the retryable 1205; a claimant that takes the lock +// after the winner committed reads the advanced base and rejects with it. +// +// Every rejection below returns a non-nil PrepareResult, and Prepare aborts +// the transaction on one (db/replication_engine.go, the AutoIDClaim branch of +// the statement loop), which releases the intent and its lock. No rejection +// path may return without going through that abort. +func (re *ReplicationEngine) prepareAutoIncClaim( + replicatedDB *ReplicatedDatabase, + txn *Transaction, + txnMgr *TransactionManager, + metaStore MetaStore, + stmt protocol.Statement, + req *PrepareRequest, +) *PrepareResult { + claim, err := protocol.DecodeAutoIncClaim(stmt.AutoIDClaimPayload) + if err != nil { + return &PrepareResult{Success: false, Rejected: true, Error: err.Error()} + } + if len(stmt.IntentKey) == 0 { + return &PrepareResult{Success: false, Rejected: true, + Error: "auto-increment claim carries no intent key"} + } + + // Take the claim's row lock before reading anything this node will vote + // on - see the "Soundness" paragraph above. AddStatement precedes it for + // the same reason processStatement orders the two that way: the statement + // must be on the transaction before an intent refers to it. + if err := txnMgr.AddStatement(txn, stmt); err != nil { + return &PrepareResult{Success: false, Error: err.Error()} + } + if err := metaStore.WriteIntent(req.TxnID, IntentTypeAutoIDClaim, AutoIncClaimTable, + string(stmt.IntentKey), OpTypeInsert, "", stmt.AutoIDClaimPayload, req.StartTS, req.NodeID); err != nil { + // A lost race for the lock is not a verdict on the claim, so it is a + // missing ACK rather than a rejection: the coordinator turns it into + // the retryable 1205 and the claimant tries again. + return &PrepareResult{Success: false, Error: err.Error()} + } + + claimStore, err := re.autoIncClaimStore() + if err != nil { + // The claim store is backed by the system database (db/autoinc_claim.go); + // if it cannot be resolved this node must refuse the claim rather than + // silently skip the check. + return &PrepareResult{Success: false, Rejected: true, + Error: fmt.Sprintf("auto-increment claim for %s refused: %v", claim.Table, err)} + } + + storedBase, err := claimStore.ReadBase(req.Database, claim.Table) + if err != nil { + // Absent, unreadable - either way this node cannot vote yes. + return &PrepareResult{Success: false, Rejected: true, + Error: fmt.Sprintf("auto-increment claim for %s refused: %v", claim.Table, err)} + } + + widthMax, err := replicatedDB.AutoIncWidthMax(claim.Table) + if err != nil { + return &PrepareResult{Success: false, Rejected: true, + Error: fmt.Sprintf("auto-increment claim for %s refused: %v", claim.Table, err), + AutoIDStoredBase: storedBase} + } + + switch { + case storedBase > claim.PrevBase: + return &PrepareResult{Success: false, Rejected: true, + Error: fmt.Sprintf("auto-increment claim for %s is stale: this node holds base %d", claim.Table, storedBase), + AutoIDStoredBase: storedBase} + case claim.NewBase != claim.PrevBase: + return &PrepareResult{Success: false, Rejected: true, + Error: fmt.Sprintf("auto-increment claim for %s must propose its own view of the base, not %d over %d", claim.Table, claim.NewBase, claim.PrevBase), + AutoIDStoredBase: storedBase} + case claim.Size == 0: + // A zero-size claim would commit base = newBase + 0, leaving the base + // where it was, so the same range could be handed out again. + return &PrepareResult{Success: false, Rejected: true, + Error: fmt.Sprintf("auto-increment claim for %s does not advance the base", claim.Table), + AutoIDStoredBase: storedBase} + case claim.NewBase > widthMax || claim.Size > widthMax-claim.NewBase: + return &PrepareResult{Success: false, Rejected: true, + // Written as two subtractions rather than newBase+size > widthMax: + // the sum is uint64 and a large size wraps, so the additive form + // lets a claim past the ceiling read as if it were inside it. + Error: fmt.Sprintf("auto-increment claim for %s exhausts the column: %d+%d > %d", claim.Table, claim.NewBase, claim.Size, widthMax), + AutoIDStoredBase: storedBase} + } + + return nil +} + // processStatement processes a single statement within a transaction func (re *ReplicationEngine) processStatement(txn *Transaction, txnMgr *TransactionManager, metaStore MetaStore, stmt protocol.Statement, req *PrepareRequest, stmtSeq uint64, adoptCapturedRows bool) *PrepareResult { if err := txnMgr.AddStatement(txn, stmt); err != nil { @@ -498,6 +699,47 @@ func (re *ReplicationEngine) Commit(ctx context.Context, req *CommitRequest) *Co } txn.Statements = req.Statements + // Apply any AUTO_INCREMENT range claim BEFORE committing, and refuse the + // commit if it cannot be applied (the claim's first invariant: a + // participant must never ACK COMMIT unless the claim row is durably in its + // own SQLite). The order follows the CREATE DATABASE precedent above, which + // executes its operation first and only then marks the transaction + // committed; applying afterwards would leave the transaction committed + // locally while this node correctly reports failure. + // + // Unlike that precedent this does NOT abort the transaction. A database + // operation is alone in its transaction, so aborting discards only itself, + // whereas a user-database transaction may carry client DML beside the + // claim. Leaving it prepared is what 2PC expects of a participant that + // cannot complete: it withholds the ACK and lets recovery resolve the + // transaction, instead of unilaterally discarding writes the rest of the + // cluster may be committing. + // + // The gate is the statement flag, not a store lookup: reading intents on + // every commit would put a Pebble scan on the write path. The flag decides + // only whether to look; every value written comes from the intent. + if statementsCarryAutoIDClaim(req.Statements) { + claimStore, csErr := re.autoIncClaimStore() + if csErr != nil { + log.Error(). + Err(csErr). + Uint64("txn_id", req.TxnID). + Uint64("node_id", re.nodeID). + Str("database", req.Database). + Msg("COMMIT REFUSED: system database unavailable for auto-increment claim apply") + return &CommitResult{Success: false, Error: fmt.Sprintf("auto-increment claim apply failed: %v", csErr)} + } + if err := claimStore.ApplyClaims(req.Database, req.TxnID, replicatedDB.GetMetaStore()); err != nil { + log.Error(). + Err(err). + Uint64("txn_id", req.TxnID). + Uint64("node_id", re.nodeID). + Str("database", req.Database). + Msg("COMMIT REFUSED: auto-increment claim could not be applied") + return &CommitResult{Success: false, Error: fmt.Sprintf("auto-increment claim apply failed: %v", err)} + } + } + if err := txnMgr.CommitTransaction(txn); err != nil { return &CommitResult{Success: false, Error: err.Error()} } @@ -566,6 +808,8 @@ func (pr *PrepareResult) ToCoordinatorResponse() *coordinator.ReplicationRespons ConflictDetected: pr.ConflictDetected, ConflictDetails: pr.ConflictDetails, Rejected: pr.Rejected, + ErrorCode: pr.ErrorCode, + AutoIDStoredBase: pr.AutoIDStoredBase, } } diff --git a/db/row_lock_store.go b/db/row_lock_store.go index 3d3f1568..281c1b1b 100644 --- a/db/row_lock_store.go +++ b/db/row_lock_store.go @@ -7,14 +7,17 @@ import ( ) // RowLockStore implements in-memory row locking using lock-free concurrent maps. -// Supports fine-grained row locks, GC markers, and efficient bulk cleanup operations. type RowLockStore struct { - // tables: "db:table" → ("rowkey" → txnID) + // tables: "db:table" → ("rowkey" → txnID). + // + // A table's row map is created on first use and never removed, even once + // it is empty. AcquireLock loads the row map and inserts into it as two + // separate steps; a release that deleted an empty map in between would + // detach the map holding the new lock, and the next acquirer would take + // the same row in a fresh map. The maps are bounded by the number of + // distinct tables this meta store has locked rows in. tables *xsync.MapOf[string, *xsync.MapOf[string, uint64]] - // gc: "db:table" → ("rowkey" → struct{}) - gc *xsync.MapOf[string, *xsync.MapOf[string, struct{}]] - // byTxn: reverse index txnID → set of "db:table:rowkey" byTxn *xsync.MapOf[uint64, *xsync.MapOf[string, struct{}]] } @@ -23,7 +26,6 @@ type RowLockStore struct { func NewRowLockStore() *RowLockStore { return &RowLockStore{ tables: xsync.NewMapOf[string, *xsync.MapOf[string, uint64]](), - gc: xsync.NewMapOf[string, *xsync.MapOf[string, struct{}]](), byTxn: xsync.NewMapOf[uint64, *xsync.MapOf[string, struct{}]](), } } @@ -64,13 +66,49 @@ func (s *RowLockStore) ReleaseLock(db, table, rowKey string) { if txnMap, ok := s.byTxn.Load(txnID); ok { txnMap.Delete(fullKey) } - - // Clean up empty maps - s.cleanupEmptyRowMap(tableKey, rowMap) } } } +// ReleaseLockIfHeldBy releases the row lock only while txnID still holds it, +// and reports whether it did. Unconditional release is a bug wherever the +// caller has a specific transaction in mind: between the decision to release +// and the release itself another transaction can take the lock, and dropping +// its lock hands the row to a third writer that never resolved the conflict. +func (s *RowLockStore) ReleaseLockIfHeldBy(db, table, rowKey string, txnID uint64) bool { + if !s.releaseRowIfHeldBy(db, table, rowKey, txnID) { + return false + } + + fullKey := makeFullKey(db, table, rowKey) + if txnMap, ok := s.byTxn.Load(txnID); ok { + txnMap.Delete(fullKey) + s.cleanupEmptyTxnMap(txnID, txnMap) + } + + return true +} + +// releaseRowIfHeldBy atomically deletes a row's lock entry if txnID holds it. +// It does not touch the reverse index. +func (s *RowLockStore) releaseRowIfHeldBy(db, table, rowKey string, txnID uint64) bool { + rowMap, ok := s.tables.Load(makeTableKey(db, table)) + if !ok { + return false + } + released := false + rowMap.Compute(rowKey, func(holder uint64, loaded bool) (uint64, bool) { + if loaded && holder == txnID { + released = true + return 0, true + } + // Leave a lock held by anyone else exactly as it is; an absent row + // (loaded false) is deleted, which is a no-op. + return holder, !loaded + }) + return released +} + // CheckLock checks if a lock exists on the specified row. // Returns (txnID, true) if locked, (0, false) if not locked. func (s *RowLockStore) CheckLock(db, table, rowKey string) (txnID uint64, exists bool) { @@ -91,156 +129,36 @@ func (s *RowLockStore) GetLockHolder(db, table, rowKey string) uint64 { return txnID } -// SetGCMarker marks a row for garbage collection. -func (s *RowLockStore) SetGCMarker(db, table, rowKey string) { - tableKey := makeTableKey(db, table) - - gcMap, _ := s.gc.LoadOrStore(tableKey, xsync.NewMapOf[string, struct{}]()) - gcMap.Store(rowKey, struct{}{}) -} - -// CheckGCMarker checks if a row is marked for garbage collection. -func (s *RowLockStore) CheckGCMarker(db, table, rowKey string) bool { - tableKey := makeTableKey(db, table) - - if gcMap, ok := s.gc.Load(tableKey); ok { - _, exists := gcMap.Load(rowKey) - return exists - } - - return false -} - -// DeleteGCMarker removes the GC marker for the specified row. -func (s *RowLockStore) DeleteGCMarker(db, table, rowKey string) { - tableKey := makeTableKey(db, table) - - if gcMap, ok := s.gc.Load(tableKey); ok { - gcMap.Delete(rowKey) - s.cleanupEmptyGCMap(tableKey, gcMap) - } -} - -// MarkGCAndRelease marks GC for all locks, releases them, and returns the keys. -// Single iteration over byTxn instead of three separate passes. -// Uses LoadAndDelete to atomically remove from byTxn before iteration, -// preventing race with concurrent AcquireLock. -func (s *RowLockStore) MarkGCAndRelease(txnID uint64) []string { - // Atomically load and delete to prevent concurrent AcquireLock from adding - // to this map while we iterate. New locks will create a fresh map. +// ReleaseByTxn releases every row lock txnID still holds and returns the +// "db:table:rowkey" keys its reverse index named. +// +// The reverse index is removed with LoadAndDelete before it is walked, so a +// concurrent AcquireLock by the same transaction lands in a fresh map rather +// than in one being iterated. Each release is conditional on txnID still +// holding the row: the reverse index can name a row whose lock was already +// released and retaken by another transaction, and releasing that row is not +// this transaction's business. All keys are returned regardless, because the +// caller deletes this transaction's own intent records by them. +func (s *RowLockStore) ReleaseByTxn(txnID uint64) []string { txnMap, ok := s.byTxn.LoadAndDelete(txnID) if !ok { return nil } - // Collect keys and process in single pass var keys []string txnMap.Range(func(fullKey string, _ struct{}) bool { keys = append(keys, fullKey) return true }) - // Mark GC and release locks for _, fullKey := range keys { db, table, rowKey := parseFullKey(fullKey) - tableKey := makeTableKey(db, table) - - // Set GC marker - s.SetGCMarker(db, table, rowKey) - - // Release lock - if rowMap, ok := s.tables.Load(tableKey); ok { - rowMap.Delete(rowKey) - s.cleanupEmptyRowMap(tableKey, rowMap) - } + s.releaseRowIfHeldBy(db, table, rowKey, txnID) } return keys } -// MarkGCByTxn sets GC markers for all locks without releasing them. -func (s *RowLockStore) MarkGCByTxn(txnID uint64) { - if txnMap, ok := s.byTxn.Load(txnID); ok { - txnMap.Range(func(fullKey string, _ struct{}) bool { - db, table, rowKey := parseFullKey(fullKey) - s.SetGCMarker(db, table, rowKey) - return true - }) - } -} - -// ReleaseByTxn releases all locks held by the specified transaction. -func (s *RowLockStore) ReleaseByTxn(txnID uint64) { - s.MarkGCAndRelease(txnID) -} - -// ReleaseByTable releases all locks for the specified table. -func (s *RowLockStore) ReleaseByTable(db, table string) { - tableKey := makeTableKey(db, table) - - if rowMap, ok := s.tables.Load(tableKey); ok { - // Collect all row keys and their transaction IDs - type lockInfo struct { - rowKey string - txnID uint64 - } - var locks []lockInfo - - rowMap.Range(func(rowKey string, txnID uint64) bool { - locks = append(locks, lockInfo{rowKey: rowKey, txnID: txnID}) - return true - }) - - // Release all locks and update reverse index - fullKeyPrefix := tableKey + ":" - for _, lock := range locks { - rowMap.Delete(lock.rowKey) - - fullKey := fullKeyPrefix + lock.rowKey - if txnMap, ok := s.byTxn.Load(lock.txnID); ok { - txnMap.Delete(fullKey) - s.cleanupEmptyTxnMap(lock.txnID, txnMap) - } - } - - // Remove the table map - s.tables.Delete(tableKey) - } - - // Clean up GC markers for this table - s.gc.Delete(tableKey) -} - -// ReleaseByDatabase releases all locks for the specified database. -func (s *RowLockStore) ReleaseByDatabase(db string) { - prefix := db + ":" - - // Collect all table keys for this database - var tableKeys []string - s.tables.Range(func(tableKey string, _ *xsync.MapOf[string, uint64]) bool { - if strings.HasPrefix(tableKey, prefix) { - tableKeys = append(tableKeys, tableKey) - return true - } - return true - }) - - // Release locks for each table - for _, tableKey := range tableKeys { - parts := strings.SplitN(tableKey, ":", 2) - if len(parts) == 2 { - s.ReleaseByTable(parts[0], parts[1]) - } - } -} - -// Clear removes all locks from the store. -func (s *RowLockStore) Clear() { - s.tables = xsync.NewMapOf[string, *xsync.MapOf[string, uint64]]() - s.gc = xsync.NewMapOf[string, *xsync.MapOf[string, struct{}]]() - s.byTxn = xsync.NewMapOf[uint64, *xsync.MapOf[string, struct{}]]() -} - // GetLocksByTxn returns all "db:table:rowkey" strings held by the specified transaction. func (s *RowLockStore) GetLocksByTxn(txnID uint64) []string { if txnMap, ok := s.byTxn.Load(txnID); ok { @@ -290,32 +208,6 @@ func parseFullKey(fullKey string) (db, table, rowKey string) { return "", "", "" } -// cleanupEmptyRowMap removes the row map if it's empty to prevent memory leaks. -func (s *RowLockStore) cleanupEmptyRowMap(tableKey string, rowMap *xsync.MapOf[string, uint64]) { - isEmpty := true - rowMap.Range(func(_ string, _ uint64) bool { - isEmpty = false - return false // Stop after first element - }) - - if isEmpty { - s.tables.Delete(tableKey) - } -} - -// cleanupEmptyGCMap removes the GC map if it's empty to prevent memory leaks. -func (s *RowLockStore) cleanupEmptyGCMap(tableKey string, gcMap *xsync.MapOf[string, struct{}]) { - isEmpty := true - gcMap.Range(func(_ string, _ struct{}) bool { - isEmpty = false - return false // Stop after first element - }) - - if isEmpty { - s.gc.Delete(tableKey) - } -} - // cleanupEmptyTxnMap removes the transaction map if it's empty to prevent memory leaks. func (s *RowLockStore) cleanupEmptyTxnMap(txnID uint64, txnMap *xsync.MapOf[string, struct{}]) { isEmpty := true @@ -330,8 +222,8 @@ func (s *RowLockStore) cleanupEmptyTxnMap(txnID uint64, txnMap *xsync.MapOf[stri } // Stats returns current statistics about the lock store. -// Returns: (activeLocks, activeTransactions, gcMarkers, tablesWithLocks) -func (s *RowLockStore) Stats() (activeLocks int, activeTransactions int, gcMarkers int, tablesWithLocks int) { +// Returns: (activeLocks, activeTransactions, tablesWithLocks) +func (s *RowLockStore) Stats() (activeLocks int, activeTransactions int, tablesWithLocks int) { // Count active transactions (transactions holding locks) s.byTxn.Range(func(_ uint64, txnMap *xsync.MapOf[string, struct{}]) bool { hasLocks := false @@ -359,14 +251,5 @@ func (s *RowLockStore) Stats() (activeLocks int, activeTransactions int, gcMarke return true }) - // Count GC markers - s.gc.Range(func(_ string, gcMap *xsync.MapOf[string, struct{}]) bool { - gcMap.Range(func(_ string, _ struct{}) bool { - gcMarkers++ - return true - }) - return true - }) - - return activeLocks, activeTransactions, gcMarkers, tablesWithLocks + return activeLocks, activeTransactions, tablesWithLocks } diff --git a/db/row_lock_store_test.go b/db/row_lock_store_test.go index 731deb6a..8ecda3d6 100644 --- a/db/row_lock_store_test.go +++ b/db/row_lock_store_test.go @@ -2,7 +2,9 @@ package db import ( "fmt" + "runtime" "sync" + "sync/atomic" "testing" "github.com/stretchr/testify/require" @@ -14,7 +16,6 @@ func TestNewRowLockStore(t *testing.T) { store := NewRowLockStore() require.NotNil(t, store) require.NotNil(t, store.tables) - require.NotNil(t, store.gc) require.NotNil(t, store.byTxn) } @@ -127,50 +128,6 @@ func TestGetLockHolder(t *testing.T) { require.Equal(t, uint64(100), holder) } -func TestSetGCMarker(t *testing.T) { - t.Parallel() - - store := NewRowLockStore() - store.SetGCMarker("db1", "table1", "row1") - - hasMarker := store.CheckGCMarker("db1", "table1", "row1") - require.True(t, hasMarker) -} - -func TestCheckGCMarker_Exists(t *testing.T) { - t.Parallel() - - store := NewRowLockStore() - store.SetGCMarker("db1", "table1", "row1") - - hasMarker := store.CheckGCMarker("db1", "table1", "row1") - require.True(t, hasMarker) -} - -func TestCheckGCMarker_NotExists(t *testing.T) { - t.Parallel() - - store := NewRowLockStore() - hasMarker := store.CheckGCMarker("db1", "table1", "row1") - require.False(t, hasMarker) -} - -func TestDeleteGCMarker(t *testing.T) { - t.Parallel() - - store := NewRowLockStore() - store.SetGCMarker("db1", "table1", "row1") - require.True(t, store.CheckGCMarker("db1", "table1", "row1")) - - store.DeleteGCMarker("db1", "table1", "row1") - require.False(t, store.CheckGCMarker("db1", "table1", "row1")) - - // Deleting non-existent marker should not panic - require.NotPanics(t, func() { - store.DeleteGCMarker("db1", "table1", "row2") - }) -} - func TestReleaseByTxn_SingleLock(t *testing.T) { t.Parallel() @@ -227,123 +184,6 @@ func TestReleaseByTxn_NoLocks(t *testing.T) { }) } -func TestReleaseByTable_SingleTable(t *testing.T) { - t.Parallel() - - store := NewRowLockStore() - _, acquired := store.AcquireLock("db1", "table1", "row1", 100) - require.True(t, acquired) - - store.ReleaseByTable("db1", "table1") - - holder := store.GetLockHolder("db1", "table1", "row1") - require.Equal(t, uint64(0), holder) -} - -func TestReleaseByTable_MultipleRows(t *testing.T) { - t.Parallel() - - store := NewRowLockStore() - _, acquired1 := store.AcquireLock("db1", "table1", "row1", 100) - require.True(t, acquired1) - - _, acquired2 := store.AcquireLock("db1", "table1", "row2", 200) - require.True(t, acquired2) - - _, acquired3 := store.AcquireLock("db1", "table1", "row3", 300) - require.True(t, acquired3) - - store.ReleaseByTable("db1", "table1") - - require.Equal(t, uint64(0), store.GetLockHolder("db1", "table1", "row1")) - require.Equal(t, uint64(0), store.GetLockHolder("db1", "table1", "row2")) - require.Equal(t, uint64(0), store.GetLockHolder("db1", "table1", "row3")) -} - -func TestReleaseByTable_OtherTablesUnaffected(t *testing.T) { - t.Parallel() - - store := NewRowLockStore() - _, acquired1 := store.AcquireLock("db1", "table1", "row1", 100) - require.True(t, acquired1) - - _, acquired2 := store.AcquireLock("db1", "table2", "row2", 200) - require.True(t, acquired2) - - _, acquired3 := store.AcquireLock("db2", "table1", "row3", 300) - require.True(t, acquired3) - - store.ReleaseByTable("db1", "table1") - - // table1 in db1 should be released - require.Equal(t, uint64(0), store.GetLockHolder("db1", "table1", "row1")) - - // Other tables should be unaffected - require.Equal(t, uint64(200), store.GetLockHolder("db1", "table2", "row2")) - require.Equal(t, uint64(300), store.GetLockHolder("db2", "table1", "row3")) -} - -func TestReleaseByDatabase_SingleDB(t *testing.T) { - t.Parallel() - - store := NewRowLockStore() - _, acquired1 := store.AcquireLock("db1", "table1", "row1", 100) - require.True(t, acquired1) - - _, acquired2 := store.AcquireLock("db1", "table2", "row2", 200) - require.True(t, acquired2) - - store.ReleaseByDatabase("db1") - - require.Equal(t, uint64(0), store.GetLockHolder("db1", "table1", "row1")) - require.Equal(t, uint64(0), store.GetLockHolder("db1", "table2", "row2")) -} - -func TestReleaseByDatabase_OtherDBsUnaffected(t *testing.T) { - t.Parallel() - - store := NewRowLockStore() - _, acquired1 := store.AcquireLock("db1", "table1", "row1", 100) - require.True(t, acquired1) - - _, acquired2 := store.AcquireLock("db2", "table1", "row2", 200) - require.True(t, acquired2) - - store.ReleaseByDatabase("db1") - - // db1 should be released - require.Equal(t, uint64(0), store.GetLockHolder("db1", "table1", "row1")) - - // db2 should be unaffected - require.Equal(t, uint64(200), store.GetLockHolder("db2", "table1", "row2")) -} - -func TestClear(t *testing.T) { - t.Parallel() - - store := NewRowLockStore() - _, acquired1 := store.AcquireLock("db1", "table1", "row1", 100) - require.True(t, acquired1) - - _, acquired2 := store.AcquireLock("db2", "table2", "row2", 200) - require.True(t, acquired2) - - store.SetGCMarker("db1", "table1", "row1") - - store.Clear() - - // All locks should be cleared - require.Equal(t, uint64(0), store.GetLockHolder("db1", "table1", "row1")) - require.Equal(t, uint64(0), store.GetLockHolder("db2", "table2", "row2")) - - // All GC markers should be cleared - require.False(t, store.CheckGCMarker("db1", "table1", "row1")) - - // Should be empty - locks := store.GetLocksByTxn(100) - require.Empty(t, locks) -} - func TestGetLocksByTxn(t *testing.T) { t.Parallel() @@ -604,52 +444,6 @@ func TestLargeNumberOfLocks(t *testing.T) { } } -func TestGCMarkerWithLocks(t *testing.T) { - t.Parallel() - - store := NewRowLockStore() - - // Set GC marker and lock on same row - store.SetGCMarker("db1", "table1", "row1") - _, acquired := store.AcquireLock("db1", "table1", "row1", 100) - require.True(t, acquired) - - // Both should exist independently - require.True(t, store.CheckGCMarker("db1", "table1", "row1")) - require.Equal(t, uint64(100), store.GetLockHolder("db1", "table1", "row1")) - - // Release lock shouldn't affect GC marker - store.ReleaseLock("db1", "table1", "row1") - require.True(t, store.CheckGCMarker("db1", "table1", "row1")) - require.Equal(t, uint64(0), store.GetLockHolder("db1", "table1", "row1")) - - // Delete GC marker shouldn't affect lock (if re-acquired) - _, acquired = store.AcquireLock("db1", "table1", "row1", 200) - require.True(t, acquired) - - store.DeleteGCMarker("db1", "table1", "row1") - require.False(t, store.CheckGCMarker("db1", "table1", "row1")) - require.Equal(t, uint64(200), store.GetLockHolder("db1", "table1", "row1")) -} - -func TestReleaseByTable_WithGCMarkers(t *testing.T) { - t.Parallel() - - store := NewRowLockStore() - - // Add locks and GC markers - _, acquired1 := store.AcquireLock("db1", "table1", "row1", 100) - require.True(t, acquired1) - - store.SetGCMarker("db1", "table1", "row2") - - // ReleaseByTable releases locks and also cleans up GC markers for that table - store.ReleaseByTable("db1", "table1") - - require.Equal(t, uint64(0), store.GetLockHolder("db1", "table1", "row1")) - require.False(t, store.CheckGCMarker("db1", "table1", "row2")) -} - func TestMultipleTxnsSameRow_Sequential(t *testing.T) { t.Parallel() @@ -674,29 +468,119 @@ func TestMultipleTxnsSameRow_Sequential(t *testing.T) { require.Equal(t, uint64(200), store.GetLockHolder("db1", "table1", "row1")) } -func TestReleaseByDatabase_WithGCMarkers(t *testing.T) { +// TestReleaseKeepsTheTableMapAnAcquirerAlreadyLoaded replays, step by step, +// the interleaving that let two transactions hold one row. AcquireLock loads a +// table's row map and inserts into it as two separate steps. A release that +// empties the map used to delete it from the store in between, so the insert +// landed in a detached map and the next acquirer took the same row in a fresh +// one. Every release path is driven. +// +// Mutation: in releaseRowIfHeldBy (or ReleaseLock), delete the table's row map +// from s.tables once it is empty. "a third acquirer took a row B holds" fires. +func TestReleaseKeepsTheTableMapAnAcquirerAlreadyLoaded(t *testing.T) { t.Parallel() - store := NewRowLockStore() - - // Add locks and GC markers - _, acquired1 := store.AcquireLock("db1", "table1", "row1", 100) - require.True(t, acquired1) + const table, row = "t", "autoinc:testdb:users" + releases := map[string]func(s *RowLockStore){ + "ReleaseLockIfHeldBy": func(s *RowLockStore) { s.ReleaseLockIfHeldBy("", table, row, 1) }, + "ReleaseByTxn": func(s *RowLockStore) { s.ReleaseByTxn(1) }, + "ReleaseLock": func(s *RowLockStore) { s.ReleaseLock("", table, row) }, + } + for name, release := range releases { + t.Run(name, func(t *testing.T) { + s := NewRowLockStore() + _, ok := s.AcquireLock("", table, row, 1) + require.True(t, ok) + + // B has done AcquireLock's first step: it holds the table's row map. + rowMap, ok := s.tables.Load(makeTableKey("", table)) + require.True(t, ok) + + // A releases the only lock in the table. + release(s) + + // B's second step inserts into the map it loaded. + _, loaded := rowMap.LoadOrStore(row, 2) + require.False(t, loaded, "A's lock survived its release") + + holder, acquired := s.AcquireLock("", table, row, 3) + require.False(t, acquired, "a third acquirer took a row B holds: B's lock was dropped with a detached map") + require.Equal(t, uint64(2), holder) + }) + } +} - store.SetGCMarker("db1", "table1", "row2") +// TestConcurrentAcquireReleaseKeepsMutualExclusion is a soak for the same +// property under real scheduling; the deterministic proof is +// TestReleaseKeepsTheTableMapAnAcquirerAlreadyLoaded. The mutex stands in for +// PebbleMetaStore.intentLockFor, which serialises acquires on one key while the +// commit path releases without it. +func TestConcurrentAcquireReleaseKeepsMutualExclusion(t *testing.T) { + t.Parallel() + + const ( + goroutines = 4 + iterations = 20000 + row = "autoinc:testdb:users" + ) + store := NewRowLockStore() + var ( + acquireMu sync.Mutex + holders atomic.Int32 + overlaps atomic.Int32 + wg sync.WaitGroup + ) + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := uint64(1); i <= iterations; i++ { + txnID := uint64(g+1)<<40 | i + acquireMu.Lock() + _, acquired := store.AcquireLock("", "t", row, txnID) + acquireMu.Unlock() + if !acquired { + continue + } + if holders.Add(1) > 1 { + overlaps.Add(1) + } + runtime.Gosched() + holders.Add(-1) + store.ReleaseByTxn(txnID) + } + }(g) + } + wg.Wait() + require.Zero(t, overlaps.Load(), "two transactions held the same row at once") +} - _, acquired2 := store.AcquireLock("db2", "table1", "row3", 200) - require.True(t, acquired2) +// TestReleaseByTxnLeavesARowAnotherTransactionHolds pins the holder check in +// ReleaseByTxn. A transaction's reverse index and the row map are updated in +// separate steps, so the index can still name a row whose lock was released +// and retaken: ReleaseLockIfHeldBy deletes the row entry before it removes the +// index entry, and another transaction can acquire the row in between. +// +// Mutation: release each indexed row unconditionally (ReleaseLock) instead of +// releaseRowIfHeldBy. "ReleaseByTxn released a row another transaction holds" +// fires. +func TestReleaseByTxnLeavesARowAnotherTransactionHolds(t *testing.T) { + t.Parallel() - store.SetGCMarker("db2", "table1", "row4") + store := NewRowLockStore() + _, ok := store.AcquireLock("", "t", "r", 1) + require.True(t, ok) - // ReleaseByDatabase releases locks and GC markers for all tables in that DB - store.ReleaseByDatabase("db1") + // The row passes to txn 2 while txn 1's index still names it. + rowMap, ok := store.tables.Load(makeTableKey("", "t")) + require.True(t, ok) + rowMap.Store("r", 2) - require.Equal(t, uint64(0), store.GetLockHolder("db1", "table1", "row1")) - require.Equal(t, uint64(200), store.GetLockHolder("db2", "table1", "row3")) + keys := store.ReleaseByTxn(1) + require.Equal(t, []string{makeFullKey("", "t", "r")}, keys, + "every key the index named must be returned for intent-record cleanup") - // GC markers for db1 should be cleaned up, but db2 should remain - require.False(t, store.CheckGCMarker("db1", "table1", "row2")) - require.True(t, store.CheckGCMarker("db2", "table1", "row4")) + holder, held := store.CheckLock("", "t", "r") + require.True(t, held, "ReleaseByTxn released a row another transaction holds") + require.Equal(t, uint64(2), holder) } diff --git a/db/snapshot/restorer.go b/db/snapshot/restorer.go index 0dc7d6d8..b5a71c72 100644 --- a/db/snapshot/restorer.go +++ b/db/snapshot/restorer.go @@ -27,6 +27,10 @@ type Restorer struct { connMgr ConnectionManager systemDB string // "__marmot_system" by default + // systemDBMerge, when set, rewrites a downloaded system database before + // it replaces the local one (see SetSystemDBMerge). + systemDBMerge func(incomingPath, localPath string) error + mu sync.RWMutex progress Progress } @@ -41,6 +45,14 @@ func NewRestorer(dataDir string, connMgr ConnectionManager) *Restorer { } } +// SetSystemDBMerge installs fn, which runs on a downloaded system database +// before it replaces the local one: fn(incomingPath, localPath) may rewrite +// incomingPath, and an error aborts the restore before any file is swapped. +// It carries state the local system database must not lose to a peer's copy. +func (r *Restorer) SetSystemDBMerge(fn func(incomingPath, localPath string) error) { + r.systemDBMerge = fn +} + // GetProgress returns current restore progress (thread-safe) func (r *Restorer) GetProgress() Progress { r.mu.RLock() @@ -254,6 +266,10 @@ func (r *Restorer) updateFilesComplete(filesComplete int) { // atomicApply swaps snapshot files into place atomically func (r *Restorer) atomicApply(tempDir string, files []DatabaseFileInfo) error { + if err := r.mergeSystemDB(tempDir, files); err != nil { + return err + } + // STEP 5: Close all SQLite connections for _, dbInfo := range files { if dbInfo.Name == r.systemDB { @@ -320,6 +336,28 @@ func (r *Restorer) atomicApply(tempDir string, files []DatabaseFileInfo) error { return nil } +// mergeSystemDB runs systemDBMerge on the downloaded system database, if the +// snapshot carries one, before anything is swapped. +func (r *Restorer) mergeSystemDB(tempDir string, files []DatabaseFileInfo) error { + if r.systemDBMerge == nil { + return nil + } + for _, dbInfo := range files { + if dbInfo.Name != r.systemDB { + continue + } + relPath := r.systemDB + ".db" + srcPath := filepath.Join(tempDir, relPath) + if _, err := os.Stat(srcPath); os.IsNotExist(err) { + return nil + } + if err := r.systemDBMerge(srcPath, filepath.Join(r.dataDir, relPath)); err != nil { + return fmt.Errorf("merge local state into snapshot system database: %w", err) + } + } + return nil +} + // reopenConnections attempts to reopen connections after a failed apply func (r *Restorer) reopenConnections(files []DatabaseFileInfo) { if r.connMgr == nil { diff --git a/db/snapshot/restorer_test.go b/db/snapshot/restorer_test.go index 324ecf60..09eff25a 100644 --- a/db/snapshot/restorer_test.go +++ b/db/snapshot/restorer_test.go @@ -512,3 +512,63 @@ func TestRestorer_RemovesOldWALAndSHM(t *testing.T) { t.Error("expected new content in database file") } } + +// TestRestorer_SystemDBMergeRunsBeforeTheSwap: the merge sees the downloaded +// system database and the local one it is about to replace, and a failed +// merge leaves every local file untouched. +// +// Mutation: skip mergeSystemDB in atomicApply. "the system database was +// installed without the merge" fires. +func TestRestorer_SystemDBMergeRunsBeforeTheSwap(t *testing.T) { + srcDir := t.TempDir() + dstDir := t.TempDir() + incoming := []byte("peer system db") + local := []byte("local system db") + userDB := []byte("peer user db") + if err := os.MkdirAll(filepath.Join(srcDir, "databases"), 0755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(dstDir, "databases"), 0755); err != nil { + t.Fatal(err) + } + srcSys := filepath.Join(srcDir, "__marmot_system.db") + srcUser := filepath.Join(srcDir, "databases", "app.db") + dstSys := filepath.Join(dstDir, "__marmot_system.db") + dstUser := filepath.Join(dstDir, "databases", "app.db") + for path, content := range map[string][]byte{srcSys: incoming, srcUser: userDB, dstSys: local, dstUser: []byte("local user db")} { + if err := os.WriteFile(path, content, 0644); err != nil { + t.Fatal(err) + } + } + files := []DatabaseFileInfo{ + {Name: "__marmot_system", Filename: "__marmot_system.db", SizeBytes: int64(len(incoming)), SHA256Checksum: calculateSHA256(incoming)}, + {Name: "app", Filename: "databases/app.db", SizeBytes: int64(len(userDB)), SHA256Checksum: calculateSHA256(userDB)}, + } + + var gotIncoming, gotLocal string + failing := NewRestorer(dstDir, nil) + failing.SetSystemDBMerge(func(in, loc string) error { + gotIncoming, gotLocal = in, loc + return os.ErrPermission + }) + if err := failing.RestoreFiles(srcDir, files); err == nil { + t.Fatal("a failed merge must abort the restore") + } + if gotIncoming != srcSys || gotLocal != dstSys { + t.Fatalf("merge saw (%s, %s), want (%s, %s)", gotIncoming, gotLocal, srcSys, dstSys) + } + for path, want := range map[string]string{dstSys: "local system db", dstUser: "local user db"} { + if got, _ := os.ReadFile(path); string(got) != want { + t.Fatalf("%s was swapped although the merge failed: %q", path, got) + } + } + + r := NewRestorer(dstDir, nil) + r.SetSystemDBMerge(func(in, _ string) error { return os.WriteFile(in, []byte("merged"), 0644) }) + if err := r.RestoreFiles(srcDir, files); err != nil { + t.Fatalf("restore: %v", err) + } + if got, _ := os.ReadFile(dstSys); string(got) != "merged" { + t.Fatalf("the system database was installed without the merge: %q", got) + } +} diff --git a/db/sqlite_batch_committer.go b/db/sqlite_batch_committer.go index e09b6212..2193b222 100644 --- a/db/sqlite_batch_committer.go +++ b/db/sqlite_batch_committer.go @@ -3,6 +3,7 @@ package db import ( "context" "database/sql" + "errors" "fmt" "os" "sort" @@ -107,6 +108,7 @@ type sealCapturedRowsFunc func(txnID uint64) error type SQLiteBatchCommitter struct { dbPath string db *sql.DB + gate *writeGate // the owning ReplicatedDatabase's gate; nil when standalone mu sync.Mutex pending map[uint64]*pendingCommit @@ -202,7 +204,13 @@ func (bc *SQLiteBatchCommitter) openOptimizedConnection() (*sql.DB, error) { } } - db, err := sql.Open(SQLiteDriverName, dsn) + var db *sql.DB + var err error + if bc.gate != nil { + db, err = bc.gate.openDB(SQLiteDriverName, dsn) + } else { + db, err = sql.Open(SQLiteDriverName, dsn) + } if err != nil { return nil, err } @@ -238,6 +246,10 @@ func (bc *SQLiteBatchCommitter) openOptimizedConnection() (*sql.DB, error) { return db, nil } +// ErrBatchCommitterStopped refuses a commit queued after the committer +// stopped - for example on a database detached for a snapshot restore. +var ErrBatchCommitterStopped = errors.New("batch committer stopped") + func (bc *SQLiteBatchCommitter) Stop() { if !bc.stopped.CompareAndSwap(false, true) { return @@ -255,6 +267,14 @@ func (bc *SQLiteBatchCommitter) Enqueue(txnID uint64, commitTS hlc.Timestamp, cd p := future.NewPromise[error]() bc.mu.Lock() + // A stopped committer never flushes again, so a commit queued now would + // wait forever. Stop sets stopped before its final flush takes bc.mu, so + // every commit queued before this check is in that flush. + if bc.stopped.Load() { + bc.mu.Unlock() + p.Set(nil, ErrBatchCommitterStopped) + return p.Future() + } bc.pending[txnID] = &pendingCommit{ cdcEntries: cdcEntries, stmts: stmts, diff --git a/db/sqlite_batch_committer_test.go b/db/sqlite_batch_committer_test.go index 37f3319d..75781b58 100644 --- a/db/sqlite_batch_committer_test.go +++ b/db/sqlite_batch_committer_test.go @@ -5,6 +5,7 @@ package db import ( "database/sql" + "errors" "fmt" "os" "path/filepath" @@ -1011,3 +1012,31 @@ func TestBatchCommitter_CheckpointMetrics(t *testing.T) { t.Error("checkpoint efficiency metric should be initialized") } } + +// TestEnqueueAfterStopIsRefused: a stopped committer never flushes again, so +// a commit queued after Stop - a COMMIT still holding a database that was +// detached for a snapshot restore - must be refused, not left waiting forever +// on its future. +// +// Mutation: drop the stopped check in Enqueue. "a commit queued after Stop +// was never answered" fires. +func TestEnqueueAfterStopIsRefused(t *testing.T) { + bc, _, cleanup := setupTestBatchCommitter(t, 10, 10*time.Millisecond) + defer cleanup() + bc.Stop() + + fut := bc.Enqueue(1, hlc.Timestamp{WallTime: 1}, nil, nil) + answered := make(chan error, 1) + go func() { + _, err := fut.Get() + answered <- err + }() + select { + case err := <-answered: + if !errors.Is(err, ErrBatchCommitterStopped) { + t.Fatalf("a commit queued after Stop got %v, want ErrBatchCommitterStopped", err) + } + case <-time.After(2 * time.Second): + t.Fatal("a commit queued after Stop was never answered") + } +} diff --git a/db/sqlite_driver.go b/db/sqlite_driver.go index d2369fa9..334ceb92 100644 --- a/db/sqlite_driver.go +++ b/db/sqlite_driver.go @@ -3,6 +3,7 @@ package db import ( "database/sql" "regexp" + "runtime" "github.com/mattn/go-sqlite3" ) @@ -10,35 +11,65 @@ import ( // SQLiteDriverName is the custom driver name with REGEXP support const SQLiteDriverName = "sqlite3_marmot" -func init() { - // Register custom SQLite driver with MySQL compatibility functions - sql.Register(SQLiteDriverName, &sqlite3.SQLiteDriver{ +// SQLiteDurableDriverName is SQLiteDriverName for a database whose commits +// must survive a power loss (WithDurableCommits). On darwin its connections +// also set fullfsync: without it, synchronous=FULL syncs with fsync(), which +// there does not flush the drive's write cache; with it, commits and +// checkpoints sync with F_FULLFSYNC. +const SQLiteDurableDriverName = "sqlite3_marmot_durable" + +// sqliteDrivers holds the registered drivers by name, so a database can open +// its pools through a connector of its own (see writeGate) with the same +// per-connection setup as sql.Open. +var sqliteDrivers = map[string]*sqlite3.SQLiteDriver{ + // Custom SQLite driver with MySQL compatibility functions + SQLiteDriverName: {ConnectHook: registerConnFunctions}, + SQLiteDurableDriverName: { ConnectHook: func(conn *sqlite3.SQLiteConn) error { - // Register REGEXP function for MySQL compatibility - if err := conn.RegisterFunc("regexp", regexpMatch, true); err != nil { + if err := registerConnFunctions(conn); err != nil { return err } - - // Register all MySQL-compatible functions for WordPress support - if err := RegisterAllMySQLCompatFuncs(conn); err != nil { - return err + if runtime.GOOS != "darwin" { + return nil } + _, err := conn.Exec("PRAGMA fullfsync=ON", nil) + return err + }, + }, +} - // Register v5.4 vector-search UDFs (vec_distance_*, __marmot_vec_*). - if err := RegisterVectorUDFs(conn); err != nil { - return err - } +func init() { + for name, drv := range sqliteDrivers { + sql.Register(name, drv) + } +} - // Load all registered extensions (always_loaded and dynamically loaded) - if extMgr := GetExtensionManager(); extMgr != nil { - if err := extMgr.LoadAllExtensions(conn); err != nil { - return err - } - } +// registerConnFunctions registers Marmot's SQL functions and extensions on a +// new connection. +func registerConnFunctions(conn *sqlite3.SQLiteConn) error { + // Register REGEXP function for MySQL compatibility + if err := conn.RegisterFunc("regexp", regexpMatch, true); err != nil { + return err + } - return nil - }, - }) + // Register all MySQL-compatible functions for WordPress support + if err := RegisterAllMySQLCompatFuncs(conn); err != nil { + return err + } + + // Register v5.4 vector-search UDFs (vec_distance_*, __marmot_vec_*). + if err := RegisterVectorUDFs(conn); err != nil { + return err + } + + // Load all registered extensions (always_loaded and dynamically loaded) + if extMgr := GetExtensionManager(); extMgr != nil { + if err := extMgr.LoadAllExtensions(conn); err != nil { + return err + } + } + + return nil } // regexpMatch implements MySQL-compatible REGEXP behavior diff --git a/db/transaction.go b/db/transaction.go index 203d4034..71d06f19 100644 --- a/db/transaction.go +++ b/db/transaction.go @@ -3,6 +3,7 @@ package db import ( "context" "database/sql" + "errors" "fmt" "sort" "strings" @@ -71,6 +72,7 @@ type TransactionManager struct { batchCommitter *SQLiteBatchCommitter // SQLite write batcher (nil if disabled) notifier CDCNotifier // Injected, can be nil vectorCDCNotifier VectorCDCNotifier // Injected, can be nil + autoIncClaimStore *AutoIncClaimStore // AUTO_INCREMENT claim store; always backed by the system database } // NewTransactionManager creates a new transaction manager @@ -169,6 +171,26 @@ func (tm *TransactionManager) SetVectorCDCNotifier(notifier VectorCDCNotifier) { tm.vectorCDCNotifier = notifier } +// SetAutoIncClaimStore injects the AUTO_INCREMENT claim store used by +// seedAutoIncBasesForDDL (db/autoinc_seed.go). The store always backs onto +// the system database, never onto tm's own db; DatabaseManager.wireGCCoordination +// wires the same store instance into every TransactionManager, user and +// system alike (db/database_manager.go). +func (tm *TransactionManager) SetAutoIncClaimStore(s *AutoIncClaimStore) { + tm.mu.Lock() + defer tm.mu.Unlock() + tm.autoIncClaimStore = s +} + +// autoIncClaimStoreAndDatabaseName returns the injected claim store and this +// manager's database name under a single read lock, for callers (currently +// only seedAutoIncBasesForDDL) that need both together. +func (tm *TransactionManager) autoIncClaimStoreAndDatabaseName() (*AutoIncClaimStore, string) { + tm.mu.RLock() + defer tm.mu.RUnlock() + return tm.autoIncClaimStore, tm.databaseName +} + // BeginTransaction starts a new distributed transaction with auto-generated ID func (tm *TransactionManager) BeginTransaction(nodeID uint64) (*Transaction, error) { startTS := tm.clock.Now() @@ -292,6 +314,10 @@ func (tm *TransactionManager) CommitTransaction(txn *Transaction) error { txn.Statements = tm.rebuildStatementsFromCDC(cdcEntries, nil) } } else { + if statementsNamePreparedRows(txn.Statements) { + return fmt.Errorf("transaction %d: %w", txn.ID, ErrPreparedRowsMissing) + } + // Statement/DDL path: flush pending DML first to ensure isolation if tm.batchCommitEnabled() { tm.batchCommitter.Flush() @@ -335,6 +361,27 @@ func (tm *TransactionManager) CommitTransaction(txn *Transaction) error { return nil } +// ErrPreparedRowsMissing refuses the COMMIT of a transaction that names DML +// rows whose captured images are gone. Every such row was captured at PREPARE +// (createDMLIntent refuses a DML row without its image), so finding none means +// something discarded them - the stale-transaction GC aborting the transaction +// while this COMMIT was under way, or a recovery that dropped them - and +// committing would ACK a write this node never applied. +var ErrPreparedRowsMissing = errors.New("prepared DML rows are gone; refusing to commit nothing") + +// statementsNamePreparedRows reports whether statements carry a DML row that +// PREPARE captured: a row change with an intent key. An INSERT without one +// (an auto-increment key the CDC hook assigns) and an AUTO_INCREMENT claim +// capture nothing at PREPARE. +func statementsNamePreparedRows(statements []protocol.Statement) bool { + for _, stmt := range statements { + if protocol.IsDML(stmt) && len(stmt.IntentKey) > 0 && !stmt.AutoIDClaim { + return true + } + } + return false +} + func (tm *TransactionManager) notifyVectorCDC(txn *Transaction, entries []*IntentEntry) { tm.mu.RLock() notifier := tm.vectorCDCNotifier @@ -483,6 +530,7 @@ func (tm *TransactionManager) applyNonDMLIntents(txnID uint64, intents []*WriteI }) hasDDL := false + var ddlTables []ddlTableOwner for _, intent := range nonDMLIntents { var vectorChange common.VectorIndexChange if err := DeserializeData(intent.DataSnapshot, &vectorChange); err == nil && vectorChange.Action != 0 { @@ -509,6 +557,7 @@ func (tm *TransactionManager) applyNonDMLIntents(txnID uint64, intents []*WriteI return fmt.Errorf("failed to execute DDL statement: %w", err) } hasDDL = true + ddlTables = append(ddlTables, ddlTableOwner{table: intent.TableName, owner: intent.NodeID}) log.Debug().Uint64("txn_id", txnID).Str("sql", intent.SQLStatement).Msg("DDL statement executed") } @@ -523,6 +572,22 @@ func (tm *TransactionManager) applyNonDMLIntents(txnID uint64, intents []*WriteI } } + // Seed __marmot__autoinc for every table this DDL tagged AUTO_INCREMENT, + // after the DDL has executed and the schema cache reflects it. The claim table now lives in the system database - a + // separate SQLite file from tm.db (db/autoinc_claim.go) - so the seed can + // no longer share the DDL's own transaction the way it once did: it is + // written in its own system-database transaction, still before this + // COMMIT is ACKed. A seeding failure therefore still fails the DDL apply + // and the COMMIT with it, the same durability boundary + // AutoIncClaimStore.ApplyClaims relies on for the claim itself: a table + // left untagged here would let a subsequent range claim be evaluated + // against a missing base row. + if len(ddlTables) > 0 { + if err := tm.seedAutoIncBasesForDDL(ddlTables); err != nil { + return fmt.Errorf("failed to seed auto-increment base after DDL (txn %d): %w", txnID, err) + } + } + return nil } diff --git a/db/write_gate.go b/db/write_gate.go new file mode 100644 index 00000000..841d8db2 --- /dev/null +++ b/db/write_gate.go @@ -0,0 +1,78 @@ +package db + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "sync/atomic" + + "github.com/mattn/go-sqlite3" +) + +// writeGate refuses every commit on one ReplicatedDatabase's SQLite file once +// the database is taken out of service. Each connection the database opens +// (its three pools and its batch committer's) carries a SQLite commit hook +// that consults the gate, so the refusal holds whatever path the write takes: +// an explicit transaction, an autocommit statement, a prepared statement, or +// a caller that still holds a *sql.Tx begun before the detach. SQLite turns a +// refused commit into a rollback and the caller's COMMIT fails with +// SQLITE_CONSTRAINT_COMMITHOOK, so nothing it wrote is ACKed. A read-only +// transaction commits without calling the hook, so reads pay nothing; a write +// commit pays one atomic load. +type writeGate struct { + closed atomic.Bool +} + +// close refuses every commit from now on. A commit already past its hook +// completes; draining it is the caller's job (ReplicatedDatabase.closeSQLite). +func (g *writeGate) close() { + g.closed.Store(true) +} + +// commitHook is the SQLite commit hook: non-zero turns the commit into a +// rollback. +func (g *writeGate) commitHook() int { + if g.closed.Load() { + return 1 + } + return 0 +} + +// openDB is sql.Open for one of the registered SQLite drivers, with every +// connection it opens gated by g. +func (g *writeGate) openDB(driverName, dsn string) (*sql.DB, error) { + drv, ok := sqliteDrivers[driverName] + if !ok { + return nil, fmt.Errorf("unknown SQLite driver %q", driverName) + } + return sql.OpenDB(&gatedConnector{driver: drv, dsn: dsn, gate: g}), nil +} + +// gatedConnector opens connections through a registered SQLite driver and +// installs the gate's commit hook on each. It hands database/sql the driver's +// own *sqlite3.SQLiteConn, so code that reaches the raw connection through +// sql.Conn.Raw keeps working. +type gatedConnector struct { + driver *sqlite3.SQLiteDriver + dsn string + gate *writeGate +} + +func (c *gatedConnector) Connect(context.Context) (driver.Conn, error) { + conn, err := c.driver.Open(c.dsn) + if err != nil { + return nil, err + } + sqliteConn, ok := conn.(*sqlite3.SQLiteConn) + if !ok { + _ = conn.Close() + return nil, fmt.Errorf("unexpected SQLite driver connection type: %T", conn) + } + sqliteConn.RegisterCommitHook(c.gate.commitHook) + return sqliteConn, nil +} + +func (c *gatedConnector) Driver() driver.Driver { + return c.driver +} diff --git a/db/write_gate_bench_test.go b/db/write_gate_bench_test.go new file mode 100644 index 00000000..a05a576b --- /dev/null +++ b/db/write_gate_bench_test.go @@ -0,0 +1,36 @@ +package db + +import ( + "testing" + + "github.com/maxpert/marmot/hlc" +) + +// BenchmarkUserDatabaseWriteCommit measures one autocommit write on a user +// database's write pool: the path every SQLite commit takes, and so every +// call of the write gate's commit hook. +func BenchmarkUserDatabaseWriteCommit(b *testing.B) { + dm, err := NewDatabaseManager(b.TempDir(), 1, hlc.NewClock(1)) + if err != nil { + b.Fatal(err) + } + defer dm.Close() + if err := dm.CreateDatabase("app"); err != nil { + b.Fatal(err) + } + app, err := dm.GetDatabase("app") + if err != nil { + b.Fatal(err) + } + writeDB := app.GetWriteDB() + if _, err := writeDB.Exec("CREATE TABLE t (id INTEGER PRIMARY KEY, v INTEGER)"); err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := writeDB.Exec("INSERT OR REPLACE INTO t (id, v) VALUES (?, ?)", i%1024, i); err != nil { + b.Fatal(err) + } + } +} diff --git a/grpc/anti_entropy.go b/grpc/anti_entropy.go index 2f803ecb..ed9a17fa 100644 --- a/grpc/anti_entropy.go +++ b/grpc/anti_entropy.go @@ -286,9 +286,36 @@ func (ae *AntiEntropyService) performAntiEntropy() { } } + ae.restoreAwaitingDatabases(ctx, aliveNodes) + log.Debug().Msg("Anti-entropy round completed") } +// restoreAwaitingDatabases retries the snapshot restore of every database +// whose reattach after an earlier restore failed (DatabasesAwaitingRestore). +// Such a database is out of service and absent from ListDatabases, so the +// loop above never reaches it; each alive peer is tried in turn until one +// restore succeeds. +func (ae *AntiEntropyService) restoreAwaitingDatabases(ctx context.Context, aliveNodes []*NodeState) { + if ae.snapshotFunc == nil { + return + } + for _, dbName := range ae.dbManager.DatabasesAwaitingRestore() { + for _, peer := range aliveNodes { + err := ae.snapshotFunc(ctx, peer.NodeId, peer.Address, dbName) + if err == nil { + telemetry.AntiEntropySyncsTotal.With("snapshot", "success").Inc() + log.Info().Uint64("peer_node", peer.NodeId).Str("database", dbName). + Msg("Restored a database whose earlier reattach failed") + break + } + telemetry.AntiEntropySyncsTotal.With("snapshot", "failed").Inc() + log.Warn().Err(err).Uint64("peer_node", peer.NodeId).Str("database", dbName). + Msg("Restore of a database awaiting one failed") + } + } +} + // findBestPeerForDatabase finds the peer with highest max_txn_id or most transactions // Uses two criteria: // 1. Higher max_txn_id indicates more recent transactions diff --git a/grpc/autoinc_claim_handler_test.go b/grpc/autoinc_claim_handler_test.go new file mode 100644 index 00000000..5c1a39b7 --- /dev/null +++ b/grpc/autoinc_claim_handler_test.go @@ -0,0 +1,150 @@ +package grpc + +import ( + "context" + "testing" + + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/maxpert/marmot/protocol/mysqlcode" + "github.com/stretchr/testify/require" +) + +// newClaimTestHandler builds a real ReplicationHandler over a DatabaseManager +// holding database "testdb" with one table, created from ddl. +func newClaimTestHandler(t *testing.T, ddl string) (*ReplicationHandler, *db.DatabaseManager) { + t.Helper() + clock := hlc.NewClock(1) + dm, err := db.NewDatabaseManager(t.TempDir(), 1, clock) + require.NoError(t, err) + t.Cleanup(func() { dm.Close() }) + require.NoError(t, dm.CreateDatabase("testdb")) + mdb, err := dm.GetDatabase("testdb") + require.NoError(t, err) + _, err = mdb.GetWriteDB().Exec(ddl) + require.NoError(t, err) + require.NoError(t, mdb.ReloadSchema()) + return NewReplicationHandler(1, dm, clock, db.NewSchemaVersionManager(dm.GetSystemDatabase().GetMetaStore())), dm +} + +// prepareOverWire sends statements through the wire conversion and the +// handler's PREPARE entry point, as a remote coordinator's PREPARE arrives. +func prepareOverWire(t *testing.T, rh *ReplicationHandler, txnID uint64, stmts []protocol.Statement, onWire func(*Statement)) *TransactionResponse { + t.Helper() + protoStmts, err := convertStatementsToProto(stmts, "testdb", txnID) + require.NoError(t, err) + for _, s := range protoStmts { + onWire(s) + } + resp, err := rh.HandleReplicateTransaction(context.Background(), &TransactionRequest{ + TxnId: txnID, + SourceNodeId: 2, + Database: "testdb", + Phase: TransactionPhase_PREPARE, + Statements: protoStmts, + Timestamp: &HLC{WallTime: int64(txnID), NodeId: 2}, + }) + require.NoError(t, err) + return resp +} + +func claimStatementForWire(t *testing.T, prevBase, size uint64) protocol.Statement { + t.Helper() + payload, err := protocol.EncodeAutoIncClaim(protocol.AutoIncClaim{ + Table: "users", PrevBase: prevBase, NewBase: prevBase, Size: size, + }) + require.NoError(t, err) + return protocol.Statement{ + Type: protocol.StatementInsert, + Database: "testdb", + TableName: "users", + IntentKey: []byte(protocol.AutoIncClaimKey("testdb", "users")), + AutoIDClaim: true, + AutoIDClaimPayload: payload, + } +} + +func unchanged(*Statement) {} + +// TestHandlePrepareReturnsTheStoredBaseOfARejectedClaim: a participant that +// rejects a stale claim returns its own stored base on the wire, so the +// claimant can retry above it instead of spinning to 1205. +// +// Mutation: drop AutoIdStoredBase from the TransactionResponse literal in +// ReplicationHandler.handlePrepare. "the rejecting participant's base did not +// reach the wire" fires. +func TestHandlePrepareReturnsTheStoredBaseOfARejectedClaim(t *testing.T) { + rh, dm := newClaimTestHandler(t, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + require.NoError(t, db.NewAutoIncClaimStore(dm.GetSystemDatabase()).Seed("testdb", "users", 500, 1)) + + resp := prepareOverWire(t, rh, 100, []protocol.Statement{claimStatementForWire(t, 100, 10)}, unchanged) + require.False(t, resp.Success) + require.True(t, resp.Rejected) + require.Equal(t, uint64(500), resp.AutoIdStoredBase, "the rejecting participant's base did not reach the wire") +} + +// TestHandlePrepareReturnsTheParticipantsErrorCode: a DDL this participant +// refuses with a MySQL-coded error (here the width ceiling, 1264) returns that +// code on the wire, so a client whose coordinator is another node sees 1264 +// rather than 1105. +// +// Mutation: drop ErrorCode from the TransactionResponse literal in +// ReplicationHandler.handlePrepare. "the participant's MySQL code did not +// reach the wire" fires. +func TestHandlePrepareReturnsTheParticipantsErrorCode(t *testing.T) { + rh, dm := newClaimTestHandler(t, "CREATE TABLE t (id INTEGER /*M:8a*/ PRIMARY KEY, v TEXT)") + mdb, err := dm.GetDatabase("testdb") + require.NoError(t, err) + _, err = mdb.GetWriteDB().Exec("INSERT INTO t (id, v) VALUES (200, 'x')") + require.NoError(t, err) + + resp := prepareOverWire(t, rh, 101, []protocol.Statement{{ + Type: protocol.StatementDDL, + Database: "testdb", + TableName: "t", + SQL: "ALTER TABLE t ADD COLUMN extra INTEGER", + }}, unchanged) + require.False(t, resp.Success) + require.True(t, resp.Rejected) + require.Equal(t, uint32(mysqlcode.ErrCodeDataOutOfRange), resp.GetErrorCode(), + "the participant's MySQL code did not reach the wire") +} + +// TestTransactionResponseConversionKeepsTheErrorCode is the coordinator-side +// half: the code read off the wire must reach coordinator.ReplicationResponse. +// +// Mutation: drop ErrorCode from convertTransactionResponse. "the error code +// was lost converting the wire response" fires. +func TestTransactionResponseConversionKeepsTheErrorCode(t *testing.T) { + resp := convertTransactionResponse(&TransactionResponse{ + Rejected: true, + ErrorCode: uint32(mysqlcode.ErrCodeDataOutOfRange), + }) + require.Equal(t, mysqlcode.ErrCodeDataOutOfRange, resp.ErrorCode, "the error code was lost converting the wire response") +} + +// TestClaimIsRejectedByABinaryThatIgnoresTheFlag is the rolling-upgrade proof. +// Two binaries cannot run in the harness, so the old binary is simulated at +// the point it differs: it does not know wire fields 9 and 10, so it drops the +// claim flag and payload, and its PREPARE handler sees an ordinary INSERT. That +// statement must be refused at the engine's "DML prepare missing encoded CDC +// row" gate, never applied as a write. +// +// Mutation: make the converter copy the claim payload into the row image +// (convertStatementsToProto) - the old binary then finds a row image, passes +// the gate, and "the old binary did not refuse the claim at the CDC-row gate" +// fires. The fixture carries a real payload for exactly this reason. +func TestClaimIsRejectedByABinaryThatIgnoresTheFlag(t *testing.T) { + rh, dm := newClaimTestHandler(t, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + require.NoError(t, db.NewAutoIncClaimStore(dm.GetSystemDatabase()).Seed("testdb", "users", 0, 1)) + + oldBinary := func(s *Statement) { + s.AutoIdClaim = false + s.AutoIdClaimPayload = nil + } + resp := prepareOverWire(t, rh, 102, []protocol.Statement{claimStatementForWire(t, 0, 10)}, oldBinary) + require.False(t, resp.Success, "the old binary accepted a claim as a write") + require.Contains(t, resp.ErrorMessage, "DML prepare missing encoded CDC row", + "the old binary did not refuse the claim at the CDC-row gate") +} diff --git a/grpc/autoinc_claim_wire_test.go b/grpc/autoinc_claim_wire_test.go new file mode 100644 index 00000000..5e6414da --- /dev/null +++ b/grpc/autoinc_claim_wire_test.go @@ -0,0 +1,97 @@ +package grpc + +import ( + "testing" + + pb "github.com/maxpert/marmot/grpc/common" + "github.com/maxpert/marmot/protocol" +) + +// TestClaimFlagRoundTripsOnAnExistingStatementType pins the shape the design +// requires of a range claim on the wire: an EXISTING StatementType carrying a +// flag, a non-empty intent key, and deliberately NO row image. +// +// A new StatementType was rejected by the design because an older binary would +// reach a conversion that does not know it. A flag is ignored by an older +// binary, which then sees a DML statement with no row image and rejects it at +// the "DML prepare missing encoded CDC row" gate - a clean failed transaction +// (TestClaimIsRejectedByABinaryThatIgnoresTheFlag). +// +// Mutation: drop the AutoIdClaim assignment in convertStatementsToProto, or the +// GetAutoIdClaim read in protocolStatementFromProto. The round trip loses the +// flag and the claim is applied as ordinary DML. +func TestClaimFlagRoundTripsOnAnExistingStatementType(t *testing.T) { + claim := protocol.Statement{ + Type: protocol.StatementInsert, + TableName: "users", + Database: "testdb", + IntentKey: []byte("autoinc:testdb:users"), + AutoIDClaim: true, + } + + protoStmts, err := convertStatementsToProto([]protocol.Statement{claim}, "testdb", 1) + if err != nil { + t.Fatalf("convertStatementsToProto: %v", err) + } + if len(protoStmts) != 1 { + t.Fatalf("converted %d statements, want 1", len(protoStmts)) + } + wire := protoStmts[0] + + // An EXISTING type, not a new one. + // Mutation: introduce a dedicated StatementType for claims. + if wire.Type != pb.StatementType_INSERT { + t.Errorf("wire type = %v, want the existing INSERT type", wire.Type) + } + if !wire.GetAutoIdClaim() { + t.Error("the claim flag did not reach the wire") + } + // No row image: this is what makes an older binary reject rather than + // mishandle. Mutation: attach a RowChange payload to the claim. + if wire.GetRowChange() != nil { + t.Error("a claim statement carries a row image; an older binary would not reject it at the CDC-row gate") + } + + back, err := protocolStatementFromProto(wire) + if err != nil { + t.Fatalf("protocolStatementFromProto: %v", err) + } + if !back.AutoIDClaim { + t.Error("the claim flag did not survive the round trip") + } + if len(back.EncodedRow) != 0 { + t.Error("the claim gained a row image on the way back") + } + if string(back.IntentKey) != "autoinc:testdb:users" { + t.Errorf("intent key = %q, want it preserved", back.IntentKey) + } +} + +// TestAutoIDStoredBaseSurvivesTransactionResponseConversion pins the +// coordinator-facing half of the rejected-claim retry plumbing: a +// db.PrepareResult's AutoIDStoredBase (the rejecting participant's own +// committed base) must survive the conversion to the wire TransactionResponse +// and back to coordinator.ReplicationResponse, or a claimant retrying above a +// participant's stored base loses the value it needs to retry with. +// +// Mutation: drop AutoIdStoredBase from convertTransactionResponse. The +// handlePrepare half is TestHandlePrepareReturnsTheStoredBaseOfARejectedClaim. +func TestAutoIDStoredBaseSurvivesTransactionResponseConversion(t *testing.T) { + wire := &TransactionResponse{ + Success: false, + Rejected: true, + ErrorMessage: "auto-increment claim for orders is stale: this node holds base 77", + // This is what ReplicationHandler.handlePrepare copies from + // db.PrepareResult.AutoIDStoredBase. + AutoIdStoredBase: 77, + } + + resp := convertTransactionResponse(wire) + + if !resp.Rejected { + t.Error("Rejected did not survive the conversion") + } + if resp.AutoIDStoredBase != 77 { + t.Errorf("AutoIDStoredBase = %d, want 77", resp.AutoIDStoredBase) + } +} diff --git a/grpc/catch_up.go b/grpc/catch_up.go index a44d84f8..64d63426 100644 --- a/grpc/catch_up.go +++ b/grpc/catch_up.go @@ -16,6 +16,8 @@ import ( "github.com/maxpert/marmot/db/snapshot" "github.com/rs/zerolog/log" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // CatchUpStrategy determines how a node should catch up with the cluster @@ -78,30 +80,6 @@ func (c *CatchUpClient) persistSchemaVersions(versions map[string]uint64) error return persistSnapshotSchemaVersions(c.dataDir, versions) } -// SchemaVersionRestoreError indicates that a snapshot's database files were -// already swapped onto disk before restoring its schema versions failed. A -// caller that reopens database connections only on success must still reopen -// them when it sees this error: the files changed underneath the old -// connection regardless of whether the schema version could be recorded. -type SchemaVersionRestoreError struct { - err error -} - -func (e *SchemaVersionRestoreError) Error() string { return e.err.Error() } -func (e *SchemaVersionRestoreError) Unwrap() error { return e.err } - -// FilesSwappedDespiteError reports whether err indicates that a snapshot's -// database files were already swapped onto disk, even though the overall -// catch-up operation failed. Callers must still reopen database connections in -// that case, or they keep serving stale connections against fresh files. -func FilesSwappedDespiteError(err error) bool { - if err == nil { - return true - } - var schemaErr *SchemaVersionRestoreError - return errors.As(err, &schemaErr) -} - // CatchUpFromPeer downloads a snapshot of a specific database from a peer // Used by anti-entropy to trigger snapshots for lagging databases func (c *CatchUpClient) CatchUpFromPeer(ctx context.Context, peerNodeID uint64, peerAddr string, database string) error { @@ -120,12 +98,52 @@ func (c *CatchUpClient) CatchUpFromPeer(ctx context.Context, peerNodeID uint64, client := NewMarmotServiceClient(conn) - // Get snapshot info (will include all databases, we filter later) + // Take the database out of service before the peer describes and takes + // its snapshot and keep it out until its file is replaced, so no write is + // ACKed into a file this restore then discards; the reattach opens + // whatever file is in place, the peer's or, if the restore failed first, + // the old one. A detach whose drain did not finish leaves the file in + // place: a write that passed the gate first may still commit into it. + dbMgr := c.dbManager.Load() + if dbMgr == nil { + return fmt.Errorf("catch-up of %s needs the running node's DatabaseManager", database) + } + var snapshotInfo *SnapshotInfoResponse + var applyErr error + switch detachErr := dbMgr.DetachDatabase(ctx, database); { + case errors.Is(detachErr, db.ErrDrainIncomplete): + applyErr = fmt.Errorf("restore of %s aborted before replacing its file: %w", database, detachErr) + case detachErr != nil: + return fmt.Errorf("failed to detach %s for restore: %w", database, detachErr) + default: + snapshotInfo, applyErr = c.downloadDatabaseSnapshot(ctx, client, database) + } + if err := dbMgr.AttachDatabase(database); err != nil { + return errors.Join(applyErr, fmt.Errorf("failed to reattach %s after restore: %w", database, err)) + } + if applyErr != nil { + return fmt.Errorf("failed to apply snapshot: %w", applyErr) + } + + log.Info(). + Str("database", database). + Uint64("snapshot_txn_id", snapshotInfo.SnapshotTxnId). + Msg("Snapshot download completed for database") + + return nil +} + +// downloadDatabaseSnapshot gets the peer's snapshot info for database alone +// and streams and installs that database's file. Both requests name the +// database, so the peer refuses them only while that database is out of +// service there, not while another of its databases awaits a restore. +func (c *CatchUpClient) downloadDatabaseSnapshot(ctx context.Context, client MarmotServiceClient, database string) (*SnapshotInfoResponse, error) { snapshotInfo, err := client.GetSnapshotInfo(ctx, &SnapshotInfoRequest{ RequestingNodeId: c.nodeID, + Database: database, }) if err != nil { - return fmt.Errorf("failed to get snapshot info: %w", err) + return nil, fmt.Errorf("failed to get snapshot info: %w", err) } log.Info(). @@ -134,17 +152,10 @@ func (c *CatchUpClient) CatchUpFromPeer(ctx context.Context, peerNodeID uint64, Int64("size_bytes", snapshotInfo.SnapshotSizeBytes). Msg("Received snapshot info for database") - // Apply snapshot for this database - if err := c.applySnapshot(ctx, client, snapshotInfo); err != nil { - return fmt.Errorf("failed to apply snapshot: %w", err) + if err := c.applySnapshot(ctx, client, snapshotInfo, database); err != nil { + return nil, err } - - log.Info(). - Str("database", database). - Uint64("snapshot_txn_id", snapshotInfo.SnapshotTxnId). - Msg("Snapshot download completed for database") - - return nil + return snapshotInfo, nil } // CatchUp performs the full catch-up process @@ -174,12 +185,48 @@ func (c *CatchUpClient) CatchUp(ctx context.Context) (uint64, error) { client := NewMarmotServiceClient(conn) - // Step 1: Get snapshot info + // Steps 1 and 2, retried while the seed's snapshot is unavailable (a + // database of its own is out of service for a restore) until ctx ends. + var snapshotInfo *SnapshotInfoResponse + for { + snapshotInfo, err = c.downloadFullSnapshot(ctx, client) + if status.Code(err) != codes.Unavailable { + break + } + log.Warn().Err(err).Str("seed", seedAddr).Msg("Seed snapshot unavailable, retrying") + select { + case <-ctx.Done(): + return 0, errors.Join(err, ctx.Err()) + case <-time.After(snapshotRetryInterval): + } + } + if err != nil { + return 0, err + } + + // Step 3: Apply delta changes (transactions after snapshot) + // Note: For full snapshot-based sync, this may not be needed immediately + // The snapshot itself should be consistent + + log.Info(). + Uint64("snapshot_txn_id", snapshotInfo.SnapshotTxnId). + Msg("Catch-up completed successfully - node stays JOINING until fully initialized") + + return snapshotInfo.SnapshotTxnId, nil +} + +// snapshotRetryInterval is how long CatchUp waits before asking a seed again +// whose snapshot is unavailable. +const snapshotRetryInterval = time.Second + +// downloadFullSnapshot gets the seed's snapshot info (step 1) and streams and +// applies every database in it (step 2). +func (c *CatchUpClient) downloadFullSnapshot(ctx context.Context, client MarmotServiceClient) (*SnapshotInfoResponse, error) { snapshotInfo, err := client.GetSnapshotInfo(ctx, &SnapshotInfoRequest{ RequestingNodeId: c.nodeID, }) if err != nil { - return 0, fmt.Errorf("failed to get snapshot info: %w", err) + return nil, fmt.Errorf("failed to get snapshot info: %w", err) } log.Info(). @@ -189,20 +236,10 @@ func (c *CatchUpClient) CatchUp(ctx context.Context) (uint64, error) { Int("databases", len(snapshotInfo.Databases)). Msg("Received snapshot info") - // Step 2: Stream and apply snapshot - if err := c.applySnapshot(ctx, client, snapshotInfo); err != nil { - return 0, fmt.Errorf("failed to apply snapshot: %w", err) + if err := c.applySnapshot(ctx, client, snapshotInfo, ""); err != nil { + return nil, fmt.Errorf("failed to apply snapshot: %w", err) } - - // Step 3: Apply delta changes (transactions after snapshot) - // Note: For full snapshot-based sync, this may not be needed immediately - // The snapshot itself should be consistent - - log.Info(). - Uint64("snapshot_txn_id", snapshotInfo.SnapshotTxnId). - Msg("Catch-up completed successfully - node stays JOINING until fully initialized") - - return snapshotInfo.SnapshotTxnId, nil + return snapshotInfo, nil } // findAvailableSeed finds an available seed node to catch up from @@ -254,7 +291,15 @@ func (c *CatchUpClient) checkNodeAvailable(ctx context.Context, addr string) boo // applySnapshot downloads and applies a snapshot from the seed node. // Uses the unified snapshot.Restorer for atomic download and apply. -func (c *CatchUpClient) applySnapshot(ctx context.Context, client MarmotServiceClient, info *SnapshotInfoResponse) error { +// +// database names the one database to install, or is empty to install every +// file the snapshot carries (startup catch-up, before any database is open). +// A running node must name its database: every other file, the system +// database included, is open and in use, and swapping it would leave its +// connections writing to an unlinked file while the next restart loads the +// peer's copy. A system database that is installed has this node's +// AUTO_INCREMENT claim bases merged into it first, so none is ever lowered. +func (c *CatchUpClient) applySnapshot(ctx context.Context, client MarmotServiceClient, info *SnapshotInfoResponse, database string) error { log.Info().Msg("Downloading snapshot using unified restorer") // Create data directory structure @@ -265,32 +310,30 @@ func (c *CatchUpClient) applySnapshot(ctx context.Context, client MarmotServiceC return fmt.Errorf("failed to create databases directory: %w", err) } - // Stream snapshot + // Stream the snapshot: every database, or only database when named. stream, err := client.StreamSnapshot(ctx, &SnapshotRequest{ RequestingNodeId: c.nodeID, + Database: database, }) if err != nil { return fmt.Errorf("failed to start snapshot stream: %w", err) } - // Convert gRPC DatabaseFileInfo to snapshot.DatabaseFileInfo - files := make([]snapshot.DatabaseFileInfo, 0, len(info.Databases)) - for _, dbInfo := range info.Databases { - files = append(files, snapshot.DatabaseFileInfo{ - Name: dbInfo.Name, - Filename: dbInfo.Filename, - SizeBytes: dbInfo.SizeBytes, - SHA256Checksum: dbInfo.Sha256Checksum, - }) + files, err := snapshotFilesToRestore(info.Databases, database) + if err != nil { + return err } // Create adapter for gRPC stream adapter := &grpcSnapshotStreamAdapter{stream: stream} - // Use snapshot.Restorer for atomic download and apply - // Note: For cluster catch-up, we don't have a connection manager since - // the DatabaseManager hasn't been initialized yet + // Use snapshot.Restorer for atomic download and apply. It is given no + // ConnectionManager: during startup catch-up nothing is open yet, and on a + // running node CatchUpFromPeer has detached the database for the whole + // restore. Closing the pools here instead would nil them under callers + // that already hold the database. restorer := snapshot.NewRestorer(c.dataDir, nil) + restorer.SetSystemDBMerge(db.RaiseAutoIncBasesFrom) if err := restorer.RestoreFromStream(adapter, files); err != nil { return fmt.Errorf("snapshot restore failed: %w", err) @@ -304,10 +347,11 @@ func (c *CatchUpClient) applySnapshot(ctx context.Context, client MarmotServiceC // files (the stream trailer) over the earlier, potentially stale read from // GetSnapshotInfo. versions := SnapshotVersionsForRestore(info.DatabaseMetadata, stream) + if database != "" { + versions = versionsFor(versions, database) + } if err := c.persistSchemaVersions(versions); err != nil { - return &SchemaVersionRestoreError{ - err: fmt.Errorf("failed to restore schema versions from snapshot: %w", err), - } + return fmt.Errorf("failed to restore schema versions from snapshot: %w", err) } log.Info(). @@ -317,6 +361,36 @@ func (c *CatchUpClient) applySnapshot(ctx context.Context, client MarmotServiceC return nil } +// snapshotFilesToRestore converts a snapshot's file list into the restorer's +// form, keeping only database when it is non-empty. +func snapshotFilesToRestore(databases []*DatabaseFileInfo, database string) ([]snapshot.DatabaseFileInfo, error) { + files := make([]snapshot.DatabaseFileInfo, 0, len(databases)) + for _, dbInfo := range databases { + if database != "" && dbInfo.Name != database { + continue + } + files = append(files, snapshot.DatabaseFileInfo{ + Name: dbInfo.Name, + Filename: dbInfo.Filename, + SizeBytes: dbInfo.SizeBytes, + SHA256Checksum: dbInfo.Sha256Checksum, + }) + } + if database != "" && len(files) == 0 { + return nil, fmt.Errorf("peer snapshot carries no database %q", database) + } + return files, nil +} + +// versionsFor keeps only database's entry of a snapshot's schema versions. +func versionsFor(versions map[string]uint64, database string) map[string]uint64 { + v, ok := versions[database] + if !ok { + return nil + } + return map[string]uint64{database: v} +} + // grpcSnapshotStreamAdapter adapts MarmotService_StreamSnapshotClient to snapshot.ChunkReceiver type grpcSnapshotStreamAdapter struct { stream MarmotService_StreamSnapshotClient diff --git a/grpc/catch_up_detach_test.go b/grpc/catch_up_detach_test.go new file mode 100644 index 00000000..022a74d7 --- /dev/null +++ b/grpc/catch_up_detach_test.go @@ -0,0 +1,400 @@ +package grpc + +import ( + "context" + "errors" + "net" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/maxpert/marmot/db" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// servePeer serves dm over real gRPC as node nodeID, through wrap when it is +// not nil, and returns the address. +func servePeer(t *testing.T, dm *db.DatabaseManager, nodeID uint64, wrap func(*Server) MarmotServiceServer) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + server := &Server{ + nodeID: nodeID, + registry: NewNodeRegistry(nodeID, listener.Addr().String()), + } + server.SetDatabaseManager(dm) + var service MarmotServiceServer = server + if wrap != nil { + service = wrap(server) + } + grpcServer := grpc.NewServer() + RegisterMarmotServiceServer(grpcServer, service) + go func() { _ = grpcServer.Serve(listener) }() + t.Cleanup(grpcServer.Stop) + return listener.Addr().String() +} + +// appRows reads table t of dm's database "app". +func appRows(t *testing.T, dm *db.DatabaseManager) map[int64]string { + t.Helper() + app, err := dm.GetDatabase("app") + require.NoError(t, err) + rows, err := app.GetReadDB().Query("SELECT id, v FROM t") + require.NoError(t, err) + defer rows.Close() + got := map[int64]string{} + for rows.Next() { + var id int64 + var v string + require.NoError(t, rows.Scan(&id, &v)) + got[id] = v + } + require.NoError(t, rows.Err()) + return got +} + +// TestCatchUpFromPeer_WriteInFlightAcrossTheDetachIsRefusedOrRestored: a +// running-node restore under concurrent load - writers and readers that look +// the database up per call, a writer and a reader that hold it throughout - +// with one write transaction begun before the catch-up and committed after +// its detach has begun. That transaction's commit is refused or its row +// survives the restore; no write that began after the detach was observed is +// ACKed and then lost; no caller sees a nil pool, a panic, or rows from both +// files. +// +// Mutation: drop the commit hook's refusal (writeGate.commitHook returns 0). +// The detach waits for the transaction, which then commits into the file the +// restore replaces, and "an in-flight transaction was ACKed and lost to the +// restore" fires. +func TestCatchUpFromPeer_WriteInFlightAcrossTheDetachIsRefusedOrRestored(t *testing.T) { + peerRows := map[int]string{} + for i := 1; i <= 20; i++ { + peerRows[i] = "peer" + } + peer := newCatchUpTestDB(t, t.TempDir(), 2, peerRows) + var gated *gatedSnapshotServer + addr := servePeer(t, peer, 2, func(s *Server) MarmotServiceServer { + gated = &gatedSnapshotServer{Server: s, requested: make(chan struct{}), release: make(chan struct{})} + return gated + }) + + localDir := t.TempDir() + local := newCatchUpTestDB(t, localDir, 1, map[int]string{1: "local", 2: "local-only"}) + held, err := local.GetDatabase("app") + require.NoError(t, err) + client := NewCatchUpClient(1, localDir, NewNodeRegistry(1, "localhost:5001"), nil) + client.SetDatabaseManager(local) + + inflight, err := held.GetWriteDB().Begin() + require.NoError(t, err) + _, err = inflight.Exec("INSERT INTO t (id, v) VALUES (500, 'in flight')") + require.NoError(t, err) + + type ack struct { + id int64 + beganAfterDetach bool + } + var ( + nextID atomic.Int64 + detached atomic.Bool + panics, nilPools, mixed atomic.Int64 + acksMu sync.Mutex + acks []ack + stop = make(chan struct{}) + wg sync.WaitGroup + ) + nextID.Store(1000) + run := func(step func()) { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + func() { + defer func() { + if recover() != nil { + panics.Add(1) + } + }() + step() + }() + } + }() + } + write := func(mdb *db.ReplicatedDatabase) { + beganAfterDetach := detached.Load() + w := mdb.GetWriteDB() + if w == nil { + nilPools.Add(1) + return + } + id := nextID.Add(1) + if _, err := w.Exec("INSERT INTO t (id, v) VALUES (?, 'w')", id); err != nil { + return + } + acksMu.Lock() + acks = append(acks, ack{id: id, beganAfterDetach: beganAfterDetach}) + acksMu.Unlock() + } + read := func(mdb *db.ReplicatedDatabase) { + r := mdb.GetReadDB() + if r == nil { + nilPools.Add(1) + return + } + rows, err := r.Query("SELECT v FROM t") + if err != nil { + return + } + defer rows.Close() + seen := map[string]bool{} + for rows.Next() { + var v string + if rows.Scan(&v) == nil { + seen[v] = true + } + } + if seen["peer"] && (seen["local"] || seen["local-only"]) { + mixed.Add(1) + } + } + lookup := func() *db.ReplicatedDatabase { + mdb, err := local.GetDatabase("app") + if err != nil { + detached.Store(true) + return nil + } + return mdb + } + for i := 0; i < 3; i++ { + run(func() { + if mdb := lookup(); mdb != nil { + write(mdb) + } + }) + run(func() { + if mdb := lookup(); mdb != nil { + read(mdb) + } + }) + } + run(func() { write(held) }) + run(func() { read(held) }) + + done := make(chan error, 1) + go func() { done <- client.CatchUpFromPeer(context.Background(), 2, addr, "app") }() + require.Eventually(t, func() bool { + _, err := local.GetDatabase("app") + return errors.Is(err, db.ErrDatabaseDetached) + }, 10*time.Second, time.Millisecond, "the catch-up never detached the database") + detached.Store(true) + inflightErr := inflight.Commit() + <-gated.requested + close(gated.release) + require.NoError(t, <-done) + time.Sleep(100 * time.Millisecond) // writes after the reattach + close(stop) + wg.Wait() + + present := appRows(t, local) + require.Equal(t, "peer", present[1], "the peer snapshot was not installed") + require.Equal(t, "peer", present[20], "the peer snapshot was not installed") + require.Equal(t, "peer", present[2], "the local file survived the restore") + require.Zero(t, panics.Load(), "a caller panicked") + require.Zero(t, nilPools.Load(), "a caller observed a nil pool") + require.Zero(t, mixed.Load(), "a read saw rows from both files") + for _, a := range acks { + if a.beganAfterDetach { + require.Contains(t, present, a.id, "a write begun after the detach was ACKed and lost") + } + } + _, inflightPresent := present[500] + t.Logf("in-flight commit: err=%v, row present after restore=%v; %d writes ACKed", inflightErr, inflightPresent, len(acks)) + require.False(t, inflightErr == nil && !inflightPresent, "an in-flight transaction was ACKed and lost to the restore") +} + +// unavailableOnceServer records the first GetSnapshotInfo it answers with +// codes.Unavailable. +type unavailableOnceServer struct { + *Server + once sync.Once + unavailable chan struct{} +} + +func (u *unavailableOnceServer) GetSnapshotInfo(ctx context.Context, req *SnapshotInfoRequest) (*SnapshotInfoResponse, error) { + resp, err := u.Server.GetSnapshotInfo(ctx, req) + if status.Code(err) == codes.Unavailable { + u.once.Do(func() { close(u.unavailable) }) + } + return resp, err +} + +// TestCatchUpRetriesASeedWhoseSnapshotIsUnavailable: a seed with one of its +// databases detached for its own restore cannot ship a complete snapshot, so +// it answers codes.Unavailable instead of silently leaving that database's +// file out. A joining node's startup catch-up retries, and completes with +// every database once the seed has reattached it. +// +// Mutation: return the producer's error as is from GetSnapshotInfo (no +// snapshotError). "a startup catch-up gave up on a seed whose snapshot was +// only temporarily unavailable" fires. Mutation: drop CatchUp's retry loop. +// The same assertion fires. +func TestCatchUpRetriesASeedWhoseSnapshotIsUnavailable(t *testing.T) { + seed := newCatchUpTestDB(t, t.TempDir(), 2, map[int]string{1: "seed"}) + require.NoError(t, seed.DetachDatabase(context.Background(), "app")) + var wrapped *unavailableOnceServer + addr := servePeer(t, seed, 2, func(s *Server) MarmotServiceServer { + wrapped = &unavailableOnceServer{Server: s, unavailable: make(chan struct{})} + return wrapped + }) + + _, err := wrapped.Server.GetSnapshotInfo(context.Background(), &SnapshotInfoRequest{RequestingNodeId: 1}) + require.Equal(t, codes.Unavailable, status.Code(err), "a seed with a detached database did not report its snapshot unavailable: %v", err) + + joinDir := t.TempDir() + client := NewCatchUpClient(1, joinDir, NewNodeRegistry(1, "localhost:5001"), []string{addr}) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := client.CatchUp(ctx) + done <- err + }() + select { + case <-wrapped.unavailable: + case err := <-done: + t.Fatalf("a startup catch-up gave up on a seed whose snapshot was only temporarily unavailable: %v", err) + } + require.NoError(t, seed.AttachDatabase("app")) + require.NoError(t, <-done, "a startup catch-up gave up on a seed whose snapshot was only temporarily unavailable") + _, err = os.Stat(filepath.Join(joinDir, "databases", "app.db")) + require.NoError(t, err, "the startup catch-up left out the database that was detached on the seed") +} + +// TestAntiEntropyRestoresADatabaseWhoseReattachFailed: a restore whose +// reattach cannot open the installed file leaves the database out of service +// and out of ListDatabases. Anti-entropy's next round restores it from the +// first alive peer that can serve it, and it is back in service with the +// peer's rows. +// +// Mutation: make DetachDatabase refuse a database whose reattach failed. The +// restore fails and "a database whose reattach failed was never restored" +// fires. Mutation: stop restoreAwaitingDatabases after the first peer. The +// same assertion fires. +func TestAntiEntropyRestoresADatabaseWhoseReattachFailed(t *testing.T) { + peer := newCatchUpTestDB(t, t.TempDir(), 3, map[int]string{1: "peer"}) + addr := servePeer(t, peer, 3, nil) + + localDir := t.TempDir() + local := newCatchUpTestDB(t, localDir, 1, map[int]string{1: "local"}) + path, err := local.GetDatabasePath("app") + require.NoError(t, err) + require.NoError(t, local.DetachDatabase(context.Background(), "app")) + for _, suffix := range []string{"-wal", "-shm"} { + require.NoError(t, os.RemoveAll(path+suffix)) + } + require.NoError(t, os.WriteFile(path, []byte("not a SQLite database, not at all"), 0o644)) + require.Error(t, local.AttachDatabase("app")) + require.Equal(t, []string{"app"}, local.DatabasesAwaitingRestore()) + require.NotContains(t, local.ListDatabases(), "app") + + client := NewCatchUpClient(1, localDir, NewNodeRegistry(1, "localhost:5001"), nil) + client.SetDatabaseManager(local) + var tried []uint64 + ae := &AntiEntropyService{ + dbManager: local, + snapshotFunc: func(ctx context.Context, peerNodeID uint64, peerAddr string, database string) error { + tried = append(tried, peerNodeID) + return client.CatchUpFromPeer(ctx, peerNodeID, peerAddr, database) + }, + } + // Peer 2 is alive in the registry but its address refuses connections. + closed, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + unreachable := closed.Addr().String() + require.NoError(t, closed.Close()) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + ae.restoreAwaitingDatabases(ctx, []*NodeState{{NodeId: 2, Address: unreachable}, {NodeId: 3, Address: addr}}) + + require.Empty(t, local.DatabasesAwaitingRestore(), "a database whose reattach failed was never restored") + require.Equal(t, []uint64{2, 3}, tried) + require.Equal(t, map[int64]string{1: "peer"}, appRows(t, local)) +} + +// strandAwaitingRestore leaves dm's database name out of service awaiting a +// restore: its file is replaced by garbage during a detach, so the reattach +// fails. +func strandAwaitingRestore(t *testing.T, dm *db.DatabaseManager, name string) { + t.Helper() + path, err := dm.GetDatabasePath(name) + require.NoError(t, err) + require.NoError(t, dm.DetachDatabase(context.Background(), name)) + for _, suffix := range []string{"-wal", "-shm"} { + require.NoError(t, os.RemoveAll(path+suffix)) + } + require.NoError(t, os.WriteFile(path, []byte("not a SQLite database, not at all"), 0o644)) + require.Error(t, dm.AttachDatabase(name)) + require.Equal(t, []string{name}, dm.DatabasesAwaitingRestore()) +} + +// TestTwoNodesAwaitingRestoresRecoverFromEachOther: nodes x and y are each +// other's only peer, and each has a different database awaiting a restore. +// A per-database snapshot is refused only while that database is out of +// service on the peer, so one anti-entropy round on each node restores both. +// A whole-node snapshot is still refused while any database is detached. +// +// Mutation: CatchUpFromPeer asks for every database's snapshot info (no +// SnapshotInfoRequest.Database). Each peer refuses because of its own +// awaiting database, and "a database awaiting a restore was never restored +// by a peer awaiting another" fires. Mutation: TakeSnapshotForDatabase refuses +// while any database is detached (errIfDetachedLocked). The same assertion +// fires. Mutation: return TakeSnapshotForDatabase's error as is from +// GetSnapshotInfo (no snapshotError). "a per-database snapshot of a detached +// database was not reported unavailable" fires. +func TestTwoNodesAwaitingRestoresRecoverFromEachOther(t *testing.T) { + xDir, yDir := t.TempDir(), t.TempDir() + x := newCatchUpTestDB(t, xDir, 1, map[int]string{1: "x"}) + y := newCatchUpTestDB(t, yDir, 2, map[int]string{1: "y"}) + require.NoError(t, x.CreateDatabase("b")) + require.NoError(t, y.CreateDatabase("b")) + var yServer *Server + xAddr := servePeer(t, x, 1, nil) + yAddr := servePeer(t, y, 2, func(s *Server) MarmotServiceServer { + yServer = s + return s + }) + strandAwaitingRestore(t, x, "app") + strandAwaitingRestore(t, y, "b") + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, err := yServer.GetSnapshotInfo(ctx, &SnapshotInfoRequest{RequestingNodeId: 1}) + require.Equal(t, codes.Unavailable, status.Code(err), "a whole-node snapshot was served while a database was detached: %v", err) + _, err = yServer.GetSnapshotInfo(ctx, &SnapshotInfoRequest{RequestingNodeId: 1, Database: "b"}) + require.Equal(t, codes.Unavailable, status.Code(err), "a per-database snapshot of a detached database was not reported unavailable: %v", err) + + antiEntropy := func(dm *db.DatabaseManager, nodeID uint64, dir string) *AntiEntropyService { + client := NewCatchUpClient(nodeID, dir, NewNodeRegistry(nodeID, "localhost:5001"), nil) + client.SetDatabaseManager(dm) + return &AntiEntropyService{dbManager: dm, snapshotFunc: client.CatchUpFromPeer} + } + antiEntropy(x, 1, xDir).restoreAwaitingDatabases(ctx, []*NodeState{{NodeId: 2, Address: yAddr}}) + antiEntropy(y, 2, yDir).restoreAwaitingDatabases(ctx, []*NodeState{{NodeId: 1, Address: xAddr}}) + + require.Empty(t, x.DatabasesAwaitingRestore(), "a database awaiting a restore was never restored by a peer awaiting another") + require.Empty(t, y.DatabasesAwaitingRestore(), "a database awaiting a restore was never restored by a peer awaiting another") + require.Equal(t, map[int64]string{1: "y"}, appRows(t, x)) + _, err = y.GetDatabase("b") + require.NoError(t, err) +} diff --git a/grpc/catch_up_test.go b/grpc/catch_up_test.go index 37f8891f..1765bc12 100644 --- a/grpc/catch_up_test.go +++ b/grpc/catch_up_test.go @@ -2,10 +2,10 @@ package grpc import ( "context" - "errors" - "fmt" "io" "net" + "sync" + "sync/atomic" "testing" "github.com/maxpert/marmot/cfg" @@ -329,22 +329,6 @@ func TestCatchUpClient_PersistSchemaVersions_PathOpenFailsAgainstLiveManager(t * require.Error(t, err) } -// SchemaVersionRestoreError must be detectable through fmt.Errorf's %w -// wrapping, since applySnapshot's own wrapping and CatchUpFromPeer's -// "failed to apply snapshot: %w" wrapping both sit between where the error is -// created and where FilesSwappedDespiteError inspects it. -func TestFilesSwappedDespiteError(t *testing.T) { - require.True(t, FilesSwappedDespiteError(nil), "no error at all means the snapshot succeeded") - - schemaErr := &SchemaVersionRestoreError{err: errors.New("boom")} - wrapped := fmt.Errorf("failed to apply snapshot: %w", schemaErr) - require.True(t, FilesSwappedDespiteError(wrapped), - "a schema-version-restore error means files were already swapped") - - require.False(t, FilesSwappedDespiteError(errors.New("connection refused")), - "an unrelated error must not be treated as files-swapped") -} - // trailerOnlyStreamSnapshotServer implements just enough of MarmotServiceServer // to prove schema versions set via stream.SetTrailer on the server side really // do arrive at stream.Trailer() on the client side over a real gRPC @@ -387,3 +371,176 @@ func TestStreamSnapshotTrailer_PropagatesOverRealGRPCConnection(t *testing.T) { got := SnapshotVersionsForRestore(nil, stream) require.Equal(t, wantVersions, got) } + +// TestSnapshotFilesToRestore_RunningNodeInstallsOnlyItsDatabase: anti-entropy +// on a running node restores one database. Every other file in the peer's +// snapshot, the system database included, is open on this node; swapping it +// would leave its connections writing to an unlinked file and hand the peer's +// AUTO_INCREMENT claim bases to the next restart. +// +// Mutation: ignore the database argument. "a running node's restore would +// install" fires. +func TestSnapshotFilesToRestore_RunningNodeInstallsOnlyItsDatabase(t *testing.T) { + dbs := []*DatabaseFileInfo{ + {Name: db.SystemDatabaseName, Filename: db.SystemDatabaseName + ".db"}, + {Name: "app", Filename: "databases/app.db"}, + {Name: "other", Filename: "databases/other.db"}, + } + + files, err := snapshotFilesToRestore(dbs, "app") + require.NoError(t, err) + names := make([]string, 0, len(files)) + for _, f := range files { + names = append(names, f.Name) + } + require.Equal(t, []string{"app"}, names, "a running node's restore would install %v", names) + + all, err := snapshotFilesToRestore(dbs, "") + require.NoError(t, err) + require.Len(t, all, 3, "startup catch-up installs every file") + + _, err = snapshotFilesToRestore(dbs, "missing") + require.Error(t, err) +} + +// gatedSnapshotServer is a real peer Server whose StreamSnapshot waits for the +// test, so the test can act while the requesting node's catch-up is between +// its detach and the peer taking its snapshot. +type gatedSnapshotServer struct { + *Server + requested chan struct{} + release chan struct{} +} + +func (g *gatedSnapshotServer) StreamSnapshot(req *SnapshotRequest, stream MarmotService_StreamSnapshotServer) error { + close(g.requested) + <-g.release + return g.Server.StreamSnapshot(req, stream) +} + +func newCatchUpTestDB(t *testing.T, dir string, nodeID uint64, rows map[int]string) *db.DatabaseManager { + t.Helper() + dm, err := db.NewDatabaseManager(dir, nodeID, hlc.NewClock(nodeID)) + require.NoError(t, err) + t.Cleanup(func() { _ = dm.Close() }) + require.NoError(t, dm.CreateDatabase("app")) + app, err := dm.GetDatabase("app") + require.NoError(t, err) + _, err = app.GetWriteDB().Exec("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)") + require.NoError(t, err) + for id, v := range rows { + _, err = app.GetWriteDB().Exec("INSERT INTO t (id, v) VALUES (?, ?)", id, v) + require.NoError(t, err) + } + return dm +} + +// TestCatchUpFromPeer_RunningNodeDetachesTheDatabaseForTheRestore: +// anti-entropy replaces one database's file on a running node while clients +// and 2PC keep using it. From before the peer takes its snapshot until the +// file is replaced, the database is detached: a caller that looks it up gets +// ErrDatabaseDetached, a caller still holding it gets "sql: database is +// closed", no pool is ever nil, and so no write is ACKed into the file the +// restore discards. Afterwards the node serves exactly the peer's rows, and +// the system database and other databases were never touched. +// +// Mutation: make DetachDatabase leave the database in service (no map +// removal, no close). "a write was ACKed during the restore window" fires. +// Mutation: close the detached pools the way CloseSQLiteConnections does, +// setting them nil. "a caller holding the database observed a nil pool" fires, +// and under -race so does the race detector. +func TestCatchUpFromPeer_RunningNodeDetachesTheDatabaseForTheRestore(t *testing.T) { + peer := newCatchUpTestDB(t, t.TempDir(), 2, map[int]string{1: "peer"}) + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + peerServer := &Server{} + peerServer.SetDatabaseManager(peer) + gated := &gatedSnapshotServer{Server: peerServer, requested: make(chan struct{}), release: make(chan struct{})} + grpcServer := grpc.NewServer() + RegisterMarmotServiceServer(grpcServer, gated) + go func() { _ = grpcServer.Serve(listener) }() + defer grpcServer.Stop() + + localDir := t.TempDir() + local := newCatchUpTestDB(t, localDir, 1, map[int]string{1: "local", 2: "local-only"}) + require.NoError(t, local.CreateDatabase("other")) + held, err := local.GetDatabase("app") + require.NoError(t, err) + + client := NewCatchUpClient(1, localDir, NewNodeRegistry(1, "localhost:5001"), nil) + client.SetDatabaseManager(local) + + // A caller that holds the database across the whole catch-up, detach and + // reattach included, must only ever see a pool or an error. + var nilPools atomic.Int64 + stop := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + writeDB, readDB := held.GetWriteDB(), held.GetReadDB() + if writeDB == nil || readDB == nil { + nilPools.Add(1) + continue + } + var n int + _ = readDB.QueryRow("SELECT COUNT(*) FROM t").Scan(&n) + } + }() + + done := make(chan error, 1) + go func() { done <- client.CatchUpFromPeer(context.Background(), 2, listener.Addr().String(), "app") }() + <-gated.requested + + // The window: the peer has not taken its snapshot yet. + acked := 0 + if _, err := local.GetDatabase("app"); err == nil { + acked++ + } else { + require.ErrorIs(t, err, db.ErrDatabaseDetached) + } + heldWrite, heldRead := held.GetWriteDB(), held.GetReadDB() + require.NotNil(t, heldWrite, "a caller holding the database observed a nil pool") + require.NotNil(t, heldRead, "a caller holding the database observed a nil pool") + if _, err := heldWrite.Exec("INSERT INTO t (id, v) VALUES (100, 'window')"); err == nil { + acked++ + } + require.Zero(t, acked, "a write was ACKed during the restore window") + var n int + require.Error(t, heldRead.QueryRow("SELECT COUNT(*) FROM t").Scan(&n), "a read was served from the file being replaced") + other, err := local.GetDatabase("other") + require.NoError(t, err, "a database not being restored left service") + _, err = other.GetWriteDB().Exec("CREATE TABLE IF NOT EXISTS o (id INTEGER PRIMARY KEY)") + require.NoError(t, err) + _, err = local.GetSystemDatabase().GetReadDB().Exec("SELECT 1") + require.NoError(t, err, "the system database left service") + + close(gated.release) + require.NoError(t, <-done) + close(stop) + wg.Wait() + require.Zero(t, nilPools.Load(), "a caller holding the database observed a nil pool") + + app, err := local.GetDatabase("app") + require.NoError(t, err) + rows, err := app.GetReadDB().Query("SELECT id, v FROM t ORDER BY id") + require.NoError(t, err) + got := map[int]string{} + for rows.Next() { + var id int + var v string + require.NoError(t, rows.Scan(&id, &v)) + got[id] = v + } + require.NoError(t, rows.Err()) + require.NoError(t, rows.Close()) + require.Equal(t, map[int]string{1: "peer"}, got, "the restored database mixes rows from both files") + _, err = app.GetWriteDB().Exec("INSERT INTO t (id, v) VALUES (3, 'after')") + require.NoError(t, err, "the reattached database does not accept writes") +} diff --git a/grpc/marmot.pb.go b/grpc/marmot.pb.go index b4022074..031cb7ba 100644 --- a/grpc/marmot.pb.go +++ b/grpc/marmot.pb.go @@ -802,9 +802,32 @@ type Statement struct { // *Statement_LoadDataChange // *Statement_VectorIndexChange // *Statement_DmlIntent - Payload isStatement_Payload `protobuf_oneof:"payload"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Payload isStatement_Payload `protobuf_oneof:"payload"` + // auto_id_claim marks this statement as a narrow AUTO_INCREMENT range claim + // rather than ordinary DML. + // + // It is a FLAG on an existing StatementType, never a new StatementType, and + // that is a rolling-upgrade requirement. A node running an older binary + // ignores an unknown field, sees a DML statement carrying no row image, and + // rejects it at the "DML prepare missing encoded CDC row" gate - which is a + // clean failed transaction. A new StatementType would instead reach a + // conversion that does not know it, and before that gate. + // + // The claim's payload does not travel here: it is written into the + // transaction's intent DataSnapshot, which is what the COMMIT handler reads. + AutoIdClaim bool `protobuf:"varint,9,opt,name=auto_id_claim,json=autoIdClaim,proto3" json:"auto_id_claim,omitempty"` + // auto_id_claim_payload carries the claim itself: {table, prevBase, newBase, + // size}, msgpack-encoded. + // + // It needs a field of its own because each PARTICIPANT writes the claim into + // its own transaction intent at PREPARE, and the COMMIT handler reads it back + // from there - so the payload has to reach the participant. It cannot ride in + // the oneof: RowChange would give the statement a row image and defeat the + // rolling-upgrade gate above, DDLChange is the wrong statement type, and + // DMLIntent carries only an intent key. + AutoIdClaimPayload []byte `protobuf:"bytes,10,opt,name=auto_id_claim_payload,json=autoIdClaimPayload,proto3" json:"auto_id_claim_payload,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Statement) Reset() { @@ -910,6 +933,20 @@ func (x *Statement) GetDmlIntent() *DMLIntent { return nil } +func (x *Statement) GetAutoIdClaim() bool { + if x != nil { + return x.AutoIdClaim + } + return false +} + +func (x *Statement) GetAutoIdClaimPayload() []byte { + if x != nil { + return x.AutoIdClaimPayload + } + return nil +} + type isStatement_Payload interface { isStatement_Payload() } @@ -1441,7 +1478,23 @@ type TransactionResponse struct { // Rejected marks a deterministic refusal of the statement itself during // PREPARE (for example DDL SQLite cannot apply), as opposed to a timeout // or storage error. See db.PrepareResult.Rejected. - Rejected bool `protobuf:"varint,6,opt,name=rejected,proto3" json:"rejected,omitempty"` + Rejected bool `protobuf:"varint,6,opt,name=rejected,proto3" json:"rejected,omitempty"` + // auto_id_stored_base is the participant's own committed base for the table a + // rejected AUTO_INCREMENT range claim named. + // + // The claimant retries with the maximum base any participant returned, so a + // rejection has to carry a number. Encoding it into error_message and parsing + // it back out would be exactly the string matching this codebase forbids. + // Zero means "not applicable": every response that is not a claim rejection. + AutoIdStoredBase uint64 `protobuf:"varint,7,opt,name=auto_id_stored_base,json=autoIdStoredBase,proto3" json:"auto_id_stored_base,omitempty"` + // error_code is the MySQL server error code the rejecting participant chose + // for this refusal (for example 1264 ER_WARN_DATA_OUT_OF_RANGE for a + // width-ceiling refusal). error_message carries the text; a typed error + // cannot cross the wire, so without this field every deterministic remote + // refusal reaches the client as 1105 HY000. Zero means "not supplied": an + // older participant, or a failure that named no code, and the coordinator + // then classifies the message as before. + ErrorCode uint32 `protobuf:"varint,8,opt,name=error_code,json=errorCode,proto3" json:"error_code,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1518,6 +1571,20 @@ func (x *TransactionResponse) GetRejected() bool { return false } +func (x *TransactionResponse) GetAutoIdStoredBase() uint64 { + if x != nil { + return x.AutoIdStoredBase + } + return 0 +} + +func (x *TransactionResponse) GetErrorCode() uint32 { + if x != nil { + return x.ErrorCode + } + return 0 +} + type ReadRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` @@ -2053,8 +2120,13 @@ func (x *DatabaseReplicationState) GetMaxSeqNum() uint64 { type SnapshotInfoRequest struct { state protoimpl.MessageState `protogen:"open.v1"` RequestingNodeId uint64 `protobuf:"varint,1,opt,name=requesting_node_id,json=requestingNodeId,proto3" json:"requesting_node_id,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Optional: describe only this database, for a StreamSnapshot naming the + // same database; empty = every database. A per-database request is refused + // only while that database is out of service, a whole-node request while + // any database is. An older server ignores it and describes every database. + Database string `protobuf:"bytes,2,opt,name=database,proto3" json:"database,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SnapshotInfoRequest) Reset() { @@ -2094,6 +2166,13 @@ func (x *SnapshotInfoRequest) GetRequestingNodeId() uint64 { return 0 } +func (x *SnapshotInfoRequest) GetDatabase() string { + if x != nil { + return x.Database + } + return "" +} + type SnapshotInfoResponse struct { state protoimpl.MessageState `protogen:"open.v1"` SnapshotTxnId uint64 `protobuf:"varint,1,opt,name=snapshot_txn_id,json=snapshotTxnId,proto3" json:"snapshot_txn_id,omitempty"` // Keep for backward compatibility @@ -3409,7 +3488,7 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x05phase\x18\x05 \x01(\x0e2\x1b.marmot.v2.TransactionPhaseR\x05phase\x12=\n" + "\vconsistency\x18\x06 \x01(\x0e2\x1b.marmot.v2.ConsistencyLevelR\vconsistency\x12\x1a\n" + "\bdatabase\x18\a \x01(\tR\bdatabase\x126\n" + - "\x17required_schema_version\x18\t \x01(\x04R\x15requiredSchemaVersionJ\x04\b\b\x10\t\"\xbf\x03\n" + + "\x17required_schema_version\x18\t \x01(\x04R\x15requiredSchemaVersionJ\x04\b\b\x10\t\"\x96\x04\n" + "\tStatement\x120\n" + "\x04type\x18\x01 \x01(\x0e2\x1c.marmot.common.StatementTypeR\x04type\x12\x1d\n" + "\n" + @@ -3422,7 +3501,10 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x10load_data_change\x18\x06 \x01(\v2\x19.marmot.v2.LoadDataChangeH\x00R\x0eloadDataChange\x12N\n" + "\x13vector_index_change\x18\a \x01(\v2\x1c.marmot.v2.VectorIndexChangeH\x00R\x11vectorIndexChange\x125\n" + "\n" + - "dml_intent\x18\b \x01(\v2\x14.marmot.v2.DMLIntentH\x00R\tdmlIntentB\t\n" + + "dml_intent\x18\b \x01(\v2\x14.marmot.v2.DMLIntentH\x00R\tdmlIntent\x12\"\n" + + "\rauto_id_claim\x18\t \x01(\bR\vautoIdClaim\x121\n" + + "\x15auto_id_claim_payload\x18\n" + + " \x01(\fR\x12autoIdClaimPayloadB\t\n" + "\apayload\"\xe4\x05\n" + "\x11VectorIndexChange\x124\n" + "\x06action\x18\x01 \x01(\x0e2\x1c.marmot.v2.VectorIndexActionR\x06action\x12\x1a\n" + @@ -3470,7 +3552,7 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x03HLC\x12\x1b\n" + "\twall_time\x18\x01 \x01(\x03R\bwallTime\x12\x18\n" + "\alogical\x18\x02 \x01(\x05R\alogical\x12\x17\n" + - "\anode_id\x18\x03 \x01(\x04R\x06nodeId\"\xf7\x01\n" + + "\anode_id\x18\x03 \x01(\x04R\x06nodeId\"\xc5\x02\n" + "\x13TransactionResponse\x12\x18\n" + "\asuccess\x18\x01 \x01(\bR\asuccess\x12#\n" + "\rerror_message\x18\x02 \x01(\tR\ferrorMessage\x12-\n" + @@ -3478,7 +3560,10 @@ const file_grpc_marmot_proto_rawDesc = "" + "applied_at\x18\x03 \x01(\v2\x0e.marmot.v2.HLCR\tappliedAt\x12+\n" + "\x11conflict_detected\x18\x04 \x01(\bR\x10conflictDetected\x12)\n" + "\x10conflict_details\x18\x05 \x01(\tR\x0fconflictDetails\x12\x1a\n" + - "\brejected\x18\x06 \x01(\bR\brejected\"\xf4\x01\n" + + "\brejected\x18\x06 \x01(\bR\brejected\x12-\n" + + "\x13auto_id_stored_base\x18\a \x01(\x04R\x10autoIdStoredBase\x12\x1d\n" + + "\n" + + "error_code\x18\b \x01(\rR\terrorCode\"\xf4\x01\n" + "\vReadRequest\x12\x14\n" + "\x05query\x18\x01 \x01(\tR\x05query\x12$\n" + "\x0esource_node_id\x18\x02 \x01(\x04R\fsourceNodeId\x12/\n" + @@ -3525,9 +3610,10 @@ const file_grpc_marmot_proto_rawDesc = "" + "syncStatus\x12+\n" + "\x12current_max_txn_id\x18\x06 \x01(\x04R\x0fcurrentMaxTxnId\x12.\n" + "\x13committed_txn_count\x18\a \x01(\x03R\x11committedTxnCount\x12\x1e\n" + - "\vmax_seq_num\x18\b \x01(\x04R\tmaxSeqNum\"C\n" + + "\vmax_seq_num\x18\b \x01(\x04R\tmaxSeqNum\"_\n" + "\x13SnapshotInfoRequest\x12,\n" + - "\x12requesting_node_id\x18\x01 \x01(\x04R\x10requestingNodeId\"\xcc\x02\n" + + "\x12requesting_node_id\x18\x01 \x01(\x04R\x10requestingNodeId\x12\x1a\n" + + "\bdatabase\x18\x02 \x01(\tR\bdatabase\"\xcc\x02\n" + "\x14SnapshotInfoResponse\x12&\n" + "\x0fsnapshot_txn_id\x18\x01 \x01(\x04R\rsnapshotTxnId\x12.\n" + "\x13snapshot_size_bytes\x18\x02 \x01(\x03R\x11snapshotSizeBytes\x12!\n" + diff --git a/grpc/marmot.proto b/grpc/marmot.proto index 1e3fde02..45af0137 100644 --- a/grpc/marmot.proto +++ b/grpc/marmot.proto @@ -162,6 +162,30 @@ message Statement { VectorIndexChange vector_index_change = 7; // For vector index control metadata DMLIntent dml_intent = 8; // DML decision metadata without row payload } + + // auto_id_claim marks this statement as a narrow AUTO_INCREMENT range claim + // rather than ordinary DML. + // + // It is a FLAG on an existing StatementType, never a new StatementType, and + // that is a rolling-upgrade requirement. A node running an older binary + // ignores an unknown field, sees a DML statement carrying no row image, and + // rejects it at the "DML prepare missing encoded CDC row" gate - which is a + // clean failed transaction. A new StatementType would instead reach a + // conversion that does not know it, and before that gate. + // + // The claim's payload does not travel here: it is written into the + // transaction's intent DataSnapshot, which is what the COMMIT handler reads. + bool auto_id_claim = 9; + // auto_id_claim_payload carries the claim itself: {table, prevBase, newBase, + // size}, msgpack-encoded. + // + // It needs a field of its own because each PARTICIPANT writes the claim into + // its own transaction intent at PREPARE, and the COMMIT handler reads it back + // from there - so the payload has to reach the participant. It cannot ride in + // the oneof: RowChange would give the statement a row image and defeat the + // rolling-upgrade gate above, DDLChange is the wrong statement type, and + // DMLIntent carries only an intent key. + bytes auto_id_claim_payload = 10; } enum VectorIndexAction { @@ -241,6 +265,22 @@ message TransactionResponse { // PREPARE (for example DDL SQLite cannot apply), as opposed to a timeout // or storage error. See db.PrepareResult.Rejected. bool rejected = 6; + // auto_id_stored_base is the participant's own committed base for the table a + // rejected AUTO_INCREMENT range claim named. + // + // The claimant retries with the maximum base any participant returned, so a + // rejection has to carry a number. Encoding it into error_message and parsing + // it back out would be exactly the string matching this codebase forbids. + // Zero means "not applicable": every response that is not a claim rejection. + uint64 auto_id_stored_base = 7; + // error_code is the MySQL server error code the rejecting participant chose + // for this refusal (for example 1264 ER_WARN_DATA_OUT_OF_RANGE for a + // width-ceiling refusal). error_message carries the text; a typed error + // cannot cross the wire, so without this field every deterministic remote + // refusal reaches the client as 1105 HY000. Zero means "not supplied": an + // older participant, or a failure that named no code, and the coordinator + // then classifies the message as before. + uint32 error_code = 8; } message ReadRequest { @@ -303,6 +343,11 @@ message DatabaseReplicationState { // ===== SNAPSHOT MESSAGES ===== message SnapshotInfoRequest { uint64 requesting_node_id = 1; + // Optional: describe only this database, for a StreamSnapshot naming the + // same database; empty = every database. A per-database request is refused + // only while that database is out of service, a whole-node request while + // any database is. An older server ignores it and describes every database. + string database = 2; } message SnapshotInfoResponse { diff --git a/grpc/replication_handler.go b/grpc/replication_handler.go index 20eb9267..40b44256 100644 --- a/grpc/replication_handler.go +++ b/grpc/replication_handler.go @@ -165,6 +165,8 @@ func (rh *ReplicationHandler) handlePrepare(ctx context.Context, req *Transactio ConflictDetected: result.ConflictDetected, ConflictDetails: result.ConflictDetails, Rejected: result.Rejected, + AutoIdStoredBase: result.AutoIDStoredBase, + ErrorCode: uint32(result.ErrorCode), } if result.Success { resp.AppliedAt = &HLC{ diff --git a/grpc/replicator_adapter.go b/grpc/replicator_adapter.go index f4106cd2..dd13186b 100644 --- a/grpc/replicator_adapter.go +++ b/grpc/replicator_adapter.go @@ -58,16 +58,24 @@ func (gr *GRPCReplicator) ReplicateTransaction(ctx context.Context, nodeID uint6 return nil, fmt.Errorf("gRPC call failed: %w", err) } - // Convert gRPC TransactionResponse to coordinator.ReplicationResponse - resp := &coordinator.ReplicationResponse{ - Success: grpcResp.Success, - Error: grpcResp.ErrorMessage, - ConflictDetected: grpcResp.ConflictDetected, - ConflictDetails: grpcResp.ConflictDetails, - Rejected: grpcResp.Rejected, - } + return convertTransactionResponse(grpcResp), nil +} - return resp, nil +// convertTransactionResponse converts a gRPC TransactionResponse into +// coordinator.ReplicationResponse. Shared by the regular and streaming RPC +// paths so both carry every field, including AutoIDStoredBase - the +// participant's own committed base on a rejected AUTO_INCREMENT range claim, +// which the claimant needs to retry above rather than spin. +func convertTransactionResponse(resp *TransactionResponse) *coordinator.ReplicationResponse { + return &coordinator.ReplicationResponse{ + Success: resp.Success, + Error: resp.ErrorMessage, + ConflictDetected: resp.ConflictDetected, + ConflictDetails: resp.ConflictDetails, + Rejected: resp.Rejected, + AutoIDStoredBase: resp.AutoIdStoredBase, + ErrorCode: uint16(resp.GetErrorCode()), + } } // convertStatementsToProto converts protocol.Statement to gRPC Statement with CDC @@ -94,8 +102,9 @@ func convertStatementsToProto(stmts []protocol.Statement, database string, txnID Database: stmtDB, } - // For DML operations: send CDC row data - // For DDL operations: send SQL + protoStmt.AutoIdClaim = stmt.AutoIDClaim + protoStmt.AutoIdClaimPayload = stmt.AutoIDClaimPayload + isDML := stmt.Type == protocol.StatementInsert || stmt.Type == protocol.StatementUpdate || stmt.Type == protocol.StatementDelete || @@ -106,6 +115,8 @@ func convertStatementsToProto(stmts []protocol.Statement, database string, txnID stmt.VectorIndexChange = &change } + // For DML operations: send CDC row data + // For DDL operations: send SQL switch { case stmt.VectorIndexChange != nil: protoStmt.Payload = &Statement_VectorIndexChange{ @@ -229,13 +240,5 @@ func (gr *GRPCReplicator) StreamReplicateTransaction(ctx context.Context, nodeID return nil, fmt.Errorf("streaming gRPC call failed: %w", err) } - // Convert gRPC TransactionResponse to coordinator.ReplicationResponse - resp := &coordinator.ReplicationResponse{ - Success: grpcResp.Success, - Error: grpcResp.ErrorMessage, - ConflictDetected: grpcResp.ConflictDetected, - ConflictDetails: grpcResp.ConflictDetails, - } - - return resp, nil + return convertTransactionResponse(grpcResp), nil } diff --git a/grpc/server.go b/grpc/server.go index f66efbc7..0ae6e75d 100644 --- a/grpc/server.go +++ b/grpc/server.go @@ -21,22 +21,15 @@ import ( "github.com/rs/zerolog/log" "github.com/soheilhy/cmux" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/reflection" + "google.golang.org/grpc/status" ) // errStopIteration is a sentinel error used to exit early from transaction iteration. var errStopIteration = errors.New("stop iteration") -// snapshotCacheEntry represents a cached snapshot for a database -type snapshotCacheEntry struct { - snapshotInfo db.SnapshotInfo - maxTxnID uint64 - createdAt time.Time - expiresAt time.Time - tempDir string -} - // Server implements the gRPC server for Marmot type Server struct { nodeID uint64 @@ -62,9 +55,8 @@ type Server struct { // CDC signal-based change streaming cdcSubscriber db.CDCSubscriber - // Snapshot caching - snapshotCache map[string]*snapshotCacheEntry - snapshotCacheMu sync.RWMutex + // Per-database snapshot exports served within snapshot_cache_ttl_seconds + snapshotExports snapshotExportCache mu sync.RWMutex stopCh chan struct{} @@ -88,12 +80,11 @@ type ServerConfig struct { // NewServer creates a new gRPC server func NewServer(config ServerConfig) (*Server, error) { s := &Server{ - nodeID: config.NodeID, - address: config.Address, - port: config.Port, - httpMux: http.NewServeMux(), - snapshotCache: make(map[string]*snapshotCacheEntry), - stopCh: make(chan struct{}), + nodeID: config.NodeID, + address: config.Address, + port: config.Port, + httpMux: http.NewServeMux(), + stopCh: make(chan struct{}), } // Initialize components with advertise address @@ -841,9 +832,20 @@ func (s *Server) GetReplicationState(ctx context.Context, req *ReplicationStateR // SNAPSHOT METHODS // ======================= +// snapshotError reports a snapshot producer's failure. A database detached for +// its own restore makes this node's snapshot temporarily unavailable rather +// than incomplete, so that case is codes.Unavailable, which callers retry. +func snapshotError(msg string, err error) error { + if errors.Is(err, db.ErrDatabaseDetached) { + return status.Errorf(codes.Unavailable, "%s: %v", msg, err) + } + return fmt.Errorf("%s: %w", msg, err) +} + // GetSnapshotInfo returns snapshot metadata for bootstrap. // NOTE: This returns estimated info. The actual snapshot is taken atomically -// during StreamSnapshot to ensure consistency. +// during StreamSnapshot to ensure consistency; a receiver verifies the files +// it streams against this info. func (s *Server) GetSnapshotInfo(ctx context.Context, req *SnapshotInfoRequest) (*SnapshotInfoResponse, error) { s.mu.RLock() dbManager := s.dbManager @@ -855,13 +857,18 @@ func (s *Server) GetSnapshotInfo(ctx context.Context, req *SnapshotInfoRequest) log.Info(). Uint64("requesting_node", req.RequestingNodeId). + Str("database", req.Database). Msg("Snapshot info requested") + if req.Database != "" { + return s.databaseSnapshotInfo(dbManager, req.Database) + } + // Use TakeSnapshot for metadata estimation only // The actual atomic snapshot is taken in StreamSnapshot snapshots, maxTxnID, err := dbManager.TakeSnapshot() if err != nil { - return nil, fmt.Errorf("failed to get snapshot info: %w", err) + return nil, snapshotError("failed to get snapshot info", err) } // Schema versions travel with the snapshot: they live in the MetaStore, which @@ -911,9 +918,36 @@ func (s *Server) GetSnapshotInfo(ctx context.Context, req *SnapshotInfoRequest) }, nil } +// databaseSnapshotInfo is GetSnapshotInfo for database alone. +func (s *Server) databaseSnapshotInfo(dbManager *db.DatabaseManager, database string) (*SnapshotInfoResponse, error) { + snap, txnID, err := dbManager.TakeDatabaseSnapshotInfo(database) + if err != nil { + return nil, snapshotError("failed to get snapshot info", err) + } + chunkSize := int64(coordinator.GetStreamChunkSize()) + return &SnapshotInfoResponse{ + SnapshotTxnId: txnID, + SnapshotSizeBytes: snap.Size, + TotalChunks: int32((snap.Size + chunkSize - 1) / chunkSize), + Databases: []*DatabaseFileInfo{{ + Name: snap.Name, + Filename: snap.Filename, + SizeBytes: snap.Size, + Sha256Checksum: snap.SHA256, + }}, + DatabaseMetadata: []*DatabaseSnapshotMetadata{{ + DatabaseName: snap.Name, + SnapshotTxnId: txnID, + SizeBytes: snap.Size, + Sha256Checksum: snap.SHA256, + SchemaVersion: snapshotSchemaVersions(dbManager)[snap.Name], + }}, + }, nil +} + // StreamSnapshot streams snapshot chunks to requesting node. // Uses atomic snapshot: copies files to temp dir under write lock, then streams from temp. -// Implements caching with TTL to avoid repeated checkpoints when multiple replicas request same snapshot. +// A single-database export is shared for snapshot_cache_ttl_seconds when that is above 0. func (s *Server) StreamSnapshot(req *SnapshotRequest, stream MarmotService_StreamSnapshotServer) error { s.mu.RLock() dbManager := s.dbManager @@ -932,130 +966,31 @@ func (s *Server) StreamSnapshot(req *SnapshotRequest, stream MarmotService_Strea Str("database", req.Database). Msg("Starting snapshot stream") - var snapshots []db.SnapshotInfo - var maxTxnID uint64 - var schemaVersions map[string]uint64 - var tempDir string - var shouldCleanup bool - - if req.Database != "" { - // Single database snapshot - use cache - cacheKey := req.Database - - // Check cache first - s.snapshotCacheMu.RLock() - cached, exists := s.snapshotCache[cacheKey] - s.snapshotCacheMu.RUnlock() - - if exists && time.Now().Before(cached.expiresAt) { - // Cache hit - use cached snapshot - log.Debug(). - Str("database", req.Database). - Uint64("cached_txn_id", cached.maxTxnID). - Time("expires_at", cached.expiresAt). - Msg("Serving cached snapshot") - - snapshots = []db.SnapshotInfo{cached.snapshotInfo} - maxTxnID = cached.maxTxnID - tempDir = cached.tempDir - shouldCleanup = false - - // Single-database snapshots don't capture schema versions in the - // same atomic step as the file copy (unlike TakeSnapshotToDir), so - // read the current value here instead - it is the best available - // estimate for this legacy path. - schemaVersions = snapshotSchemaVersions(dbManager) - } else { - // Cache miss or expired - create new snapshot - if exists { - log.Debug(). - Str("database", req.Database). - Msg("Cache expired, creating new snapshot") - // Clean up old cache entry - s.snapshotCacheMu.Lock() - delete(s.snapshotCache, cacheKey) - s.snapshotCacheMu.Unlock() - os.RemoveAll(cached.tempDir) - } else { - log.Debug(). - Str("database", req.Database). - Msg("Cache miss, creating new snapshot") - } - - // Create temp directory for atomic snapshot - var err error - tempDir, err = os.MkdirTemp(dataDir, "snapshot-export-") - if err != nil { - return fmt.Errorf("failed to create temp directory: %w", err) - } - - snapshot, txnID, err := dbManager.TakeSnapshotForDatabase(tempDir, req.Database) - if err != nil { - os.RemoveAll(tempDir) - return fmt.Errorf("failed to snapshot database %s: %w", req.Database, err) - } - snapshots = []db.SnapshotInfo{snapshot} - maxTxnID = txnID - schemaVersions = snapshotSchemaVersions(dbManager) - - // Cache the snapshot - ttl := time.Duration(cfg.Config.Replica.SnapshotCacheTTLSec) * time.Second - now := time.Now() - s.snapshotCacheMu.Lock() - s.snapshotCache[cacheKey] = &snapshotCacheEntry{ - snapshotInfo: snapshot, - maxTxnID: txnID, - createdAt: now, - expiresAt: now.Add(ttl), - tempDir: tempDir, - } - s.snapshotCacheMu.Unlock() - - log.Info(). - Str("database", req.Database). - Uint64("snapshot_txn_id", txnID). - Dur("ttl", ttl). - Msg("Single database snapshot created and cached") - - shouldCleanup = false - } - - } else { - // All databases snapshot (backward compatible, no caching for full snapshots) - var err error - tempDir, err = os.MkdirTemp(dataDir, "snapshot-export-") - if err != nil { - return fmt.Errorf("failed to create temp directory: %w", err) - } - - snapshots, maxTxnID, schemaVersions, err = dbManager.TakeSnapshotToDir(tempDir) - if err != nil { - os.RemoveAll(tempDir) - return fmt.Errorf("failed to take snapshot: %w", err) - } - - log.Info(). - Int("databases", len(snapshots)). - Uint64("snapshot_txn_id", maxTxnID). - Msg("Full snapshot created (no caching)") - - shouldCleanup = true + export, err := s.exportSnapshot(dbManager, dataDir, req.Database) + if err != nil { + return err } + defer export.release() + snapshots := export.snapshots + maxTxnID := export.maxTxnID // Advertise the schema versions captured with these exact files as stream // trailer metadata. GetSnapshotInfo's earlier estimate can go stale if a // DDL commits between that call and this one; receivers prefer this value // (see grpc.SnapshotVersionsForRestore) so they never restore a version // older than the files they actually received. + schemaVersions := export.schemaVersions + if req.Database != "" { + // Single-database snapshots don't capture schema versions in the + // same atomic step as the file copy (unlike TakeSnapshotToDir), so + // read the current value here instead - it is the best available + // estimate for this legacy path. + schemaVersions = snapshotSchemaVersions(dbManager) + } if trailer := snapshotSchemaVersionsTrailer(schemaVersions); trailer != nil { stream.SetTrailer(trailer) } - // Cleanup temp directory when done (only for non-cached snapshots) - if shouldCleanup { - defer os.RemoveAll(tempDir) - } - log.Info(). Uint64("requesting_node", req.RequestingNodeId). Uint64("snapshot_txn_id", maxTxnID). @@ -1139,6 +1074,49 @@ func (s *Server) StreamSnapshot(req *SnapshotRequest, stream MarmotService_Strea return nil } +// exportSnapshot returns an export of database (every database when empty) +// with one reference held for the caller, who must release it. A +// single-database export is served from, and published to, the cache only +// when snapshot_cache_ttl_seconds is above 0; otherwise the caller's release +// removes it, as it does every whole-node export. +func (s *Server) exportSnapshot(dbManager *db.DatabaseManager, dataDir, database string) (*snapshotExport, error) { + ttl := time.Duration(cfg.Config.Replica.SnapshotCacheTTLSec) * time.Second + cached := database != "" && ttl > 0 + if cached { + if export := s.snapshotExports.acquire(database, time.Now()); export != nil { + log.Debug().Str("database", database).Uint64("cached_txn_id", export.maxTxnID).Msg("Serving cached snapshot") + return export, nil + } + } + + export, err := newSnapshotExport(dataDir) + if err != nil { + return nil, fmt.Errorf("failed to create temp directory: %w", err) + } + + if database == "" { + export.snapshots, export.maxTxnID, export.schemaVersions, err = dbManager.TakeSnapshotToDir(export.dir) + if err != nil { + export.release() + return nil, snapshotError("failed to take snapshot", err) + } + return export, nil + } + + snapshot, txnID, err := dbManager.TakeSnapshotForDatabase(export.dir, database) + if err != nil { + export.release() + return nil, snapshotError(fmt.Sprintf("failed to snapshot database %s", database), err) + } + export.snapshots = []db.SnapshotInfo{snapshot} + export.maxTxnID = txnID + if cached { + export.expiresAt = time.Now().Add(ttl) + s.snapshotExports.publish(database, export) + } + return export, nil +} + // GetNodeRegistry returns the node registry (for testing/debugging) func (s *Server) GetNodeRegistry() *NodeRegistry { return s.registry @@ -1199,10 +1177,15 @@ func (s *Server) SetMetricsHandler(handler http.Handler) { s.metricsHandler = handler } -// SetDatabaseManager sets the database manager for snapshot operations +// SetDatabaseManager sets the database manager for snapshot operations. +// On first wiring it removes the export directories a crash left behind: no +// stream can export before the manager is set. func (s *Server) SetDatabaseManager(manager *db.DatabaseManager) { s.mu.Lock() defer s.mu.Unlock() + if s.dbManager == nil && manager != nil { + removeOrphanedSnapshotExports(manager.GetDataDir()) + } s.dbManager = manager } @@ -1394,48 +1377,12 @@ func (s *Server) runSnapshotCacheCleanup() { } } -// cleanupExpiredSnapshots removes expired cache entries and their temp directories +// cleanupExpiredSnapshots evicts expired cached exports. A stream still +// reading one keeps its directory until the stream finishes. func (s *Server) cleanupExpiredSnapshots() { - now := time.Now() - var toDelete []string - - // Collect expired entries - s.snapshotCacheMu.RLock() - for dbName, entry := range s.snapshotCache { - if now.After(entry.expiresAt) { - toDelete = append(toDelete, dbName) - } - } - s.snapshotCacheMu.RUnlock() - - if len(toDelete) == 0 { - return + if n := s.snapshotExports.evictExpired(time.Now()); n > 0 { + log.Debug().Int("evicted", n).Msg("Snapshot cache cleanup completed") } - - // Delete expired entries - s.snapshotCacheMu.Lock() - for _, dbName := range toDelete { - entry, exists := s.snapshotCache[dbName] - if !exists { - continue - } - - // Remove temp directory - if err := os.RemoveAll(entry.tempDir); err != nil { - log.Warn().Err(err).Str("database", dbName).Str("temp_dir", entry.tempDir).Msg("Failed to cleanup expired snapshot cache") - } - - delete(s.snapshotCache, dbName) - - log.Debug(). - Str("database", dbName). - Time("created_at", entry.createdAt). - Time("expired_at", entry.expiresAt). - Msg("Cleaned up expired snapshot cache entry") - } - s.snapshotCacheMu.Unlock() - - log.Debug().Int("cleaned", len(toDelete)).Msg("Snapshot cache cleanup completed") } // ======================= diff --git a/grpc/snapshot_export.go b/grpc/snapshot_export.go new file mode 100644 index 00000000..f8ba9a27 --- /dev/null +++ b/grpc/snapshot_export.go @@ -0,0 +1,130 @@ +package grpc + +import ( + "os" + "path/filepath" + "sync" + "sync/atomic" + "time" + + "github.com/maxpert/marmot/db" + "github.com/rs/zerolog/log" +) + +// snapshotExportPrefix names the directories StreamSnapshot exports database +// files into, directly under the data directory. Each directory belongs to +// exactly one snapshotExport, so one found before any stream has started was +// left behind by a crash. +const snapshotExportPrefix = "snapshot-export-" + +// snapshotExport is one exported copy of database files in its own directory. +// Every holder owns one reference: each stream reading the files, and the +// cache while the export is mapped. The last release removes the directory, +// so no directory outlives its readers and none is removed while one reads. +type snapshotExport struct { + dir string + snapshots []db.SnapshotInfo + maxTxnID uint64 + schemaVersions map[string]uint64 + expiresAt time.Time + refs atomic.Int32 +} + +// newSnapshotExport creates an empty export directory under dataDir, with one +// reference held for the caller. +func newSnapshotExport(dataDir string) (*snapshotExport, error) { + dir, err := os.MkdirTemp(dataDir, snapshotExportPrefix) + if err != nil { + return nil, err + } + e := &snapshotExport{dir: dir} + e.refs.Store(1) + return e, nil +} + +// retain adds a reference. The caller must already hold one, directly or +// through the cache's lock while the export is mapped. +func (e *snapshotExport) retain() { + e.refs.Add(1) +} + +// release drops a reference and removes the directory with the last one. +func (e *snapshotExport) release() { + if e.refs.Add(-1) != 0 { + return + } + if err := os.RemoveAll(e.dir); err != nil { + log.Warn().Err(err).Str("dir", e.dir).Msg("Failed to remove snapshot export") + } +} + +// snapshotExportCache maps a database name to its most recent export. The +// zero value is ready to use. +type snapshotExportCache struct { + mu sync.Mutex + entries map[string]*snapshotExport +} + +// acquire returns the export cached for database with a reference held for +// the caller, or nil when there is none or it expired before now. +func (c *snapshotExportCache) acquire(database string, now time.Time) *snapshotExport { + c.mu.Lock() + defer c.mu.Unlock() + e := c.entries[database] + if e == nil || !now.Before(e.expiresAt) { + return nil + } + e.retain() + return e +} + +// publish maps e for database with its own reference and releases the +// cache's reference on the export it replaces. The caller keeps its reference. +func (c *snapshotExportCache) publish(database string, e *snapshotExport) { + e.retain() + c.mu.Lock() + if c.entries == nil { + c.entries = make(map[string]*snapshotExport) + } + replaced := c.entries[database] + c.entries[database] = e + c.mu.Unlock() + if replaced != nil { + replaced.release() + } +} + +// evictExpired unmaps every export that expired before now and releases the +// cache's reference on each. It returns how many were evicted. +func (c *snapshotExportCache) evictExpired(now time.Time) int { + var expired []*snapshotExport + c.mu.Lock() + for database, e := range c.entries { + if !now.Before(e.expiresAt) { + expired = append(expired, e) + delete(c.entries, database) + } + } + c.mu.Unlock() + for _, e := range expired { + e.release() + } + return len(expired) +} + +// removeOrphanedSnapshotExports removes every export directory under dataDir. +// Call it only before any stream can export into dataDir. +func removeOrphanedSnapshotExports(dataDir string) { + dirs, err := filepath.Glob(filepath.Join(dataDir, snapshotExportPrefix+"*")) + if err != nil { + log.Warn().Err(err).Str("data_dir", dataDir).Msg("Failed to list orphaned snapshot exports") + return + } + for _, dir := range dirs { + if err := os.RemoveAll(dir); err != nil { + log.Warn().Err(err).Str("dir", dir).Msg("Failed to remove orphaned snapshot export") + continue + } + log.Info().Str("dir", dir).Msg("Removed orphaned snapshot export") + } +} diff --git a/grpc/snapshot_export_test.go b/grpc/snapshot_export_test.go new file mode 100644 index 00000000..7bae14c1 --- /dev/null +++ b/grpc/snapshot_export_test.go @@ -0,0 +1,256 @@ +package grpc + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/maxpert/marmot/cfg" + "github.com/maxpert/marmot/db" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +// collectingSnapshotStream records every chunk StreamSnapshot sends and calls +// onSend, when set, after each one. +type collectingSnapshotStream struct { + grpc.ServerStream + data bytes.Buffer + chunks int + onSend func(chunks int) +} + +func (s *collectingSnapshotStream) Send(c *SnapshotChunk) error { + s.chunks++ + s.data.Write(c.Data) + if s.onSend != nil { + s.onSend(s.chunks) + } + return nil +} + +func (s *collectingSnapshotStream) SetTrailer(metadata.MD) {} + +func (s *collectingSnapshotStream) Context() context.Context { return context.Background() } + +func setSnapshotCacheTTL(t *testing.T, seconds int) { + t.Helper() + prev := cfg.Config.Replica.SnapshotCacheTTLSec + cfg.Config.Replica.SnapshotCacheTTLSec = seconds + t.Cleanup(func() { cfg.Config.Replica.SnapshotCacheTTLSec = prev }) +} + +func fillApp(t *testing.T, dm *db.DatabaseManager, from, n int) { + t.Helper() + app, err := dm.GetDatabase("app") + require.NoError(t, err) + for id := from; id < from+n; id++ { + _, err = app.GetWriteDB().Exec("INSERT INTO t (id, v) VALUES (?, ?)", id, strings.Repeat("p", 200)) + require.NoError(t, err) + } +} + +func snapshotExportDirs(t *testing.T, dm *db.DatabaseManager) []string { + t.Helper() + dirs, err := filepath.Glob(filepath.Join(dm.GetDataDir(), snapshotExportPrefix+"*")) + require.NoError(t, err) + return dirs +} + +// TestOverlappingCatchUpsLeaveNoSnapshotExport: two nodes catch up the same +// database from one peer at once, ten rounds in a row, as two anti-entropy +// loops do. Every catch-up succeeds and installs the peer's rows. With the +// cluster default TTL of 0 no export directory is left on the peer; with a +// TTL above 0 only the cached export is left, and it goes once it expires. +// +// Mutation: cache the export whatever the TTL. The TTL 0 case finds a +// directory left behind. Mutation: drop StreamSnapshot's release of its +// export. The TTL 0 case finds a directory per catch-up. +func TestOverlappingCatchUpsLeaveNoSnapshotExport(t *testing.T) { + for _, tc := range []struct { + name string + ttlSeconds int + cachedDirs int + }{ + {name: "ttl 0", ttlSeconds: 0, cachedDirs: 0}, + {name: "ttl 30", ttlSeconds: 30, cachedDirs: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + setSnapshotCacheTTL(t, tc.ttlSeconds) + peer := newCatchUpTestDB(t, t.TempDir(), 2, map[int]string{1: "peer"}) + fillApp(t, peer, 10, 5000) + var peerServer *Server + addr := servePeer(t, peer, 2, func(s *Server) MarmotServiceServer { + peerServer = s + return s + }) + want := appRows(t, peer) + + nodes := make([]*db.DatabaseManager, 2) + clients := make([]*CatchUpClient, 2) + for i := range nodes { + dir := t.TempDir() + id := uint64(10 + i) + nodes[i] = newCatchUpTestDB(t, dir, id, map[int]string{1: "local"}) + clients[i] = NewCatchUpClient(id, dir, NewNodeRegistry(id, "localhost:5001"), nil) + clients[i].SetDatabaseManager(nodes[i]) + } + + for round := 0; round < 10; round++ { + errs := make([]error, len(clients)) + var wg sync.WaitGroup + for i, client := range clients { + wg.Add(1) + go func() { + defer wg.Done() + errs[i] = client.CatchUpFromPeer(context.Background(), 2, addr, "app") + }() + } + wg.Wait() + for i, err := range errs { + require.NoError(t, err, "round %d node %d", round, i) + } + } + for _, node := range nodes { + require.Equal(t, want, appRows(t, node)) + } + + require.Len(t, snapshotExportDirs(t, peer), tc.cachedDirs, "export directories left on the peer") + peerServer.snapshotExports.evictExpired(time.Now().Add(time.Hour)) + require.Empty(t, snapshotExportDirs(t, peer), "export directories left after the cache expired") + }) + } +} + +// TestSnapshotExportOutlivesEvictionUntilItsReaderFinishes: while a stream +// reads a cached export, the export is evicted and replaced by a newer one. +// Its directory stays until the stream finishes, the stream delivers the +// whole file, and then the directory is removed. Both stream kinds (one +// database and every database) remove their directory when the cache does not +// hold it. +// +// Mutation: make release remove the directory whatever the reference count. +// "the directory was removed while a stream read it" fires. +func TestSnapshotExportOutlivesEvictionUntilItsReaderFinishes(t *testing.T) { + setSnapshotCacheTTL(t, 30) + prevChunk := cfg.Config.Replication.StreamChunkSizeKB + cfg.Config.Replication.StreamChunkSizeKB = 1 + t.Cleanup(func() { cfg.Config.Replication.StreamChunkSizeKB = prevChunk }) + + peer := newCatchUpTestDB(t, t.TempDir(), 2, map[int]string{1: "peer"}) + fillApp(t, peer, 10, 200) + server := &Server{} + server.SetDatabaseManager(peer) + req := &SnapshotRequest{RequestingNodeId: 9, Database: "app"} + + var readDir string + stream := &collectingSnapshotStream{} + stream.onSend = func(chunks int) { + if chunks != 1 { + return + } + dirs := snapshotExportDirs(t, peer) + require.Len(t, dirs, 1) + readDir = dirs[0] + server.snapshotExports.evictExpired(time.Now().Add(time.Hour)) + require.NoError(t, server.StreamSnapshot(req, &collectingSnapshotStream{})) + _, err := os.Stat(readDir) + require.NoError(t, err, "the directory was removed while a stream read it") + } + require.NoError(t, server.StreamSnapshot(req, stream)) + require.Greater(t, stream.chunks, 1) + + info, _, err := peer.TakeDatabaseSnapshotInfo("app") + require.NoError(t, err) + sum := sha256.Sum256(stream.data.Bytes()) + require.Equal(t, info.Size, int64(stream.data.Len()), "the stream was truncated") + require.Equal(t, info.SHA256, hex.EncodeToString(sum[:])) + + _, err = os.Stat(readDir) + require.True(t, os.IsNotExist(err), "the evicted directory outlived its last reader") + require.Len(t, snapshotExportDirs(t, peer), 1, "only the replacing cached export may remain") + + server.snapshotExports.evictExpired(time.Now().Add(time.Hour)) + require.Empty(t, snapshotExportDirs(t, peer)) + require.NoError(t, server.StreamSnapshot(&SnapshotRequest{RequestingNodeId: 9}, &collectingSnapshotStream{})) + require.Empty(t, snapshotExportDirs(t, peer), "a whole-node export outlived its stream") +} + +// TestSnapshotExportCacheServesOneExportWithinTheTTL: within the TTL a second +// stream of the same database reads the cached export instead of exporting +// again; an expired entry is never served. +func TestSnapshotExportCacheServesOneExportWithinTheTTL(t *testing.T) { + setSnapshotCacheTTL(t, 30) + peer := newCatchUpTestDB(t, t.TempDir(), 2, map[int]string{1: "peer"}) + server := &Server{} + server.SetDatabaseManager(peer) + req := &SnapshotRequest{RequestingNodeId: 9, Database: "app"} + + require.NoError(t, server.StreamSnapshot(req, &collectingSnapshotStream{})) + first := snapshotExportDirs(t, peer) + require.Len(t, first, 1) + require.NoError(t, server.StreamSnapshot(req, &collectingSnapshotStream{})) + require.Equal(t, first, snapshotExportDirs(t, peer), "a hit exported again") + + require.Nil(t, server.snapshotExports.acquire("app", time.Now().Add(time.Hour)), "an expired export was served") +} + +// TestSnapshotExportCachePublishReleasesTheReplacedExport: when two misses of +// one database both publish, the export that loses is removed once its own +// stream releases it, and the winner stays until it is evicted. +// +// Mutation: let publish drop the replaced export without releasing it. +// "the replaced export outlived its last reader" fires. +func TestSnapshotExportCachePublishReleasesTheReplacedExport(t *testing.T) { + dataDir := t.TempDir() + var cache snapshotExportCache + expiresAt := time.Now().Add(time.Hour) + exports := make([]*snapshotExport, 2) + for i := range exports { + e, err := newSnapshotExport(dataDir) + require.NoError(t, err) + e.expiresAt = expiresAt + cache.publish("app", e) + exports[i] = e + } + for _, e := range exports { + e.release() + } + + _, err := os.Stat(exports[0].dir) + require.True(t, os.IsNotExist(err), "the replaced export outlived its last reader") + _, err = os.Stat(exports[1].dir) + require.NoError(t, err, "the cached export was removed while mapped") + + require.Equal(t, 1, cache.evictExpired(expiresAt)) + _, err = os.Stat(exports[1].dir) + require.True(t, os.IsNotExist(err), "the evicted export was not removed") +} + +// TestSetDatabaseManagerRemovesOrphanedSnapshotExports: export directories a +// crash left in the data directory are removed when the server is first wired +// to its database manager; nothing else in the data directory is touched. +// +// Mutation: drop the sweep from SetDatabaseManager. "an orphaned export +// survived startup" fires. +func TestSetDatabaseManagerRemovesOrphanedSnapshotExports(t *testing.T) { + dir := t.TempDir() + dm := newCatchUpTestDB(t, dir, 2, map[int]string{1: "peer"}) + orphan := filepath.Join(dir, snapshotExportPrefix+"123", "databases") + require.NoError(t, os.MkdirAll(orphan, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(orphan, "app.db"), []byte("partial copy"), 0o644)) + + server := &Server{} + server.SetDatabaseManager(dm) + + require.Empty(t, snapshotExportDirs(t, dm), "an orphaned export survived startup") + require.Equal(t, map[int64]string{1: "peer"}, appRows(t, dm)) +} diff --git a/grpc/vector_statement.go b/grpc/vector_statement.go index 03269d1c..14e4a8ba 100644 --- a/grpc/vector_statement.go +++ b/grpc/vector_statement.go @@ -113,11 +113,13 @@ func protocolStatementFromProto(stmt *Statement) (protocol.Statement, error) { } internalStmt := protocol.Statement{ - SQL: stmt.GetSQL(), - Type: code, - TableName: stmt.TableName, - Database: stmt.Database, - IntentKey: stmt.GetIntentKey(), + SQL: stmt.GetSQL(), + Type: code, + TableName: stmt.TableName, + Database: stmt.Database, + IntentKey: stmt.GetIntentKey(), + AutoIDClaim: stmt.GetAutoIdClaim(), + AutoIDClaimPayload: stmt.GetAutoIdClaimPayload(), } switch payload := stmt.Payload.(type) { diff --git a/marmot.go b/marmot.go index f9e6a2da..e4b85412 100644 --- a/marmot.go +++ b/marmot.go @@ -372,29 +372,11 @@ func main() { log.Info().Msg("Delta sync completed successfully") } - // Create snapshot function for anti-entropy + // Create snapshot function for anti-entropy. CatchUpFromPeer keeps the + // database detached from before the peer's snapshot until its file is + // replaced, and reattaches it on every path. snapshotFunc := func(ctx context.Context, peerNodeID uint64, peerAddr string, database string) error { - // Download snapshot to disk - catchUpErr := catchUpClient.CatchUpFromPeer(ctx, peerNodeID, peerAddr, database) - - // The download can fail after the snapshot's files were already swapped - // onto disk (e.g. restoring schema versions failed post-swap). Reopen - // unconditionally whenever that happened, or the node keeps serving the - // old connection against freshly-swapped files - even though the - // overall catch-up is reported as failed and gets retried. - if !marmotgrpc.FilesSwappedDespiteError(catchUpErr) { - return catchUpErr - } - - // Reload the database connection to pick up the new snapshot file - // This is critical: the old connection still points to the old data - if err := dbMgr.ReopenDatabase(database); err != nil { - log.Error().Err(err).Str("database", database).Msg("Failed to reload database after snapshot") - return fmt.Errorf("database reload failed after snapshot: %w", err) - } - - log.Info().Str("database", database).Msg("Database reloaded after snapshot download") - return catchUpErr + return catchUpClient.CatchUpFromPeer(ctx, peerNodeID, peerAddr, database) } antiEntropy := marmotgrpc.NewAntiEntropyServiceFromConfig( diff --git a/protocol/autoinc_claim.go b/protocol/autoinc_claim.go new file mode 100644 index 00000000..cb05cc8e --- /dev/null +++ b/protocol/autoinc_claim.go @@ -0,0 +1,54 @@ +package protocol + +import ( + "fmt" + + "github.com/maxpert/marmot/encoding" +) + +// AutoIncClaim is the payload a narrow AUTO_INCREMENT range claim carries in +// its intent's DataSnapshot. msgpack, like every other payload in this +// system. +// +// It lives here, rather than in coordinator or db, because coordinator builds +// claims (coordinator/autoinc_claim.go) and db applies them +// (db/autoinc_claim.go), but db already imports coordinator (for +// coordinator.Replicator), so coordinator importing db back would cycle. +// protocol imports neither db nor coordinator and both already import it, +// making it the neutral home for the type both sides share. +// +// It is what the COMMIT handler reads to write the new base, and what a +// participant evaluates at PREPARE. It carries no width: each participant +// derives widthMax from its OWN sqlite_master, because trusting the claimant +// would let a node with a stale schema be rubber-stamped. +type AutoIncClaim struct { + Table string `msgpack:"table"` + PrevBase uint64 `msgpack:"prev_base"` + NewBase uint64 `msgpack:"new_base"` + Size uint64 `msgpack:"size"` +} + +// EncodeAutoIncClaim renders a claim payload for an intent's DataSnapshot. +func EncodeAutoIncClaim(claim AutoIncClaim) ([]byte, error) { + data, err := encoding.Marshal(claim) + if err != nil { + return nil, fmt.Errorf("encode auto-increment claim: %w", err) + } + return data, nil +} + +// DecodeAutoIncClaim reads a claim payload back. +func DecodeAutoIncClaim(data []byte) (AutoIncClaim, error) { + var claim AutoIncClaim + if err := encoding.Unmarshal(data, &claim); err != nil { + return AutoIncClaim{}, fmt.Errorf("decode auto-increment claim: %w", err) + } + return claim, nil +} + +// AutoIncClaimKey is the intent key a claim locks on: one key per (database, +// table), so two claims for the same table serialise on the existing row lock +// while claims for different tables do not contend. +func AutoIncClaimKey(database, table string) string { + return "autoinc:" + database + ":" + table +} diff --git a/protocol/autoinc_claim_test.go b/protocol/autoinc_claim_test.go new file mode 100644 index 00000000..dc6c72e2 --- /dev/null +++ b/protocol/autoinc_claim_test.go @@ -0,0 +1,35 @@ +package protocol + +import "testing" + +// TestAutoIncClaimWireShapeIsPinned pins the exact msgpack encoding of an +// AutoIncClaim payload: this is the wire contract a claim intent's +// DataSnapshot carries between every participant in the cluster, and it must +// not silently change shape (e.g. a field reorder, rename, or a tag edit) +// without every node in a mixed-version cluster being able to decode it. +// +// The byte length below was captured from this exact field set - table +// "users", prev_base 100, new_base 164, size 64 - encoded through +// encoding.Marshal. A change to this length signals an encoding change that +// needs a deliberate wire-compatibility decision, not a silent drift. +func TestAutoIncClaimWireShapeIsPinned(t *testing.T) { + claim := AutoIncClaim{Table: "users", PrevBase: 100, NewBase: 164, Size: 64} + + data, err := EncodeAutoIncClaim(claim) + if err != nil { + t.Fatalf("EncodeAutoIncClaim: %v", err) + } + + const wantLen = 64 + if len(data) != wantLen { + t.Fatalf("encoded length = %d, want %d (wire shape changed: %x)", len(data), wantLen, data) + } + + got, err := DecodeAutoIncClaim(data) + if err != nil { + t.Fatalf("DecodeAutoIncClaim: %v", err) + } + if got != claim { + t.Fatalf("round trip = %+v, want %+v", got, claim) + } +} diff --git a/protocol/error_mapper.go b/protocol/error_mapper.go index 7885ba20..cbc41586 100644 --- a/protocol/error_mapper.go +++ b/protocol/error_mapper.go @@ -73,12 +73,14 @@ func sqlStateForCode(code uint16) string { return SQLStateNoSuchCol case ErrCodeDupFieldName: return SQLStateDupColumn - case ErrCodeParseError, transform.ErrCodeNotSupportedYet: + case ErrCodeParseError, transform.ErrCodeNotSupportedYet, ErrCodeTableAccessDenied: return SQLStateSyntax case ErrCodeNoDB: return SQLStateNoDB case ErrCodeServerShutdown: return SQLStateConnFailure + case ErrCodeDataOutOfRange: + return SQLStateDataOutOfRange default: return SQLStateGeneral } @@ -97,6 +99,14 @@ func mapSQLiteError(e sqlite3.Error, msg string) *MySQLError { return NewMySQLError(ErrCodeNoReferencedRow, SQLStateIntegrity, msg) case sqlite3.ErrConstraintCheck: return NewMySQLError(ErrCodeCheckConstraint, SQLStateIntegrity, msg) + case sqlite3.ErrConstraintCommitHook: + // A commit hook refused the commit and SQLite rolled the whole + // transaction back. Marmot's only commit hook is a database's write + // gate, which refuses while the database is out of service (restore, + // drop, shutdown): transient, and nothing was written, so the client + // restarts the transaction as after a deadlock (1213, SQLSTATE 40001). + return NewMySQLError(ErrCodeDeadlock, SQLStateDeadlock, + "Transaction rolled back while its database was out of service; try restarting transaction") } // Check primary codes diff --git a/protocol/error_mapper_test.go b/protocol/error_mapper_test.go index 1745748e..e8cb25cb 100644 --- a/protocol/error_mapper_test.go +++ b/protocol/error_mapper_test.go @@ -2,6 +2,7 @@ package protocol import ( "errors" + "fmt" "testing" "github.com/mattn/go-sqlite3" @@ -81,6 +82,31 @@ func TestConvertToMySQLError_SQLiteExtendedCodes(t *testing.T) { } } +// TestConvertToMySQLError_CommitHookRefusalIsRetryable: a commit the write +// gate refused was rolled back whole while its database was out of service, +// so the client must restart the transaction (1213, SQLSTATE 40001), not +// treat it as an integrity violation. It is told apart by its extended code, +// not its text: a plain constraint error with the same message stays an +// integrity error. +// +// Mutation: drop the ErrConstraintCommitHook case. "a refused commit was not +// reported as retryable" fires. +func TestConvertToMySQLError_CommitHookRefusalIsRetryable(t *testing.T) { + refused := sqlite3.Error{Code: sqlite3.ErrConstraint, ExtendedCode: sqlite3.ErrConstraintCommitHook} + for _, err := range []error{refused, fmt.Errorf("commit: %w", refused)} { + result := ConvertToMySQLError(err) + if result.Code != ErrCodeDeadlock || result.SQLState != SQLStateDeadlock { + t.Errorf("a refused commit was not reported as retryable: %v gave %d/%s", err, result.Code, result.SQLState) + } + } + + plain := sqlite3.Error{Code: sqlite3.ErrConstraint, ExtendedCode: sqlite3.ErrNoExtended(sqlite3.ErrConstraint)} + result := ConvertToMySQLError(fmt.Errorf("%w: constraint failed", plain)) + if result.Code == ErrCodeDeadlock || result.SQLState != SQLStateIntegrity { + t.Errorf("a plain constraint error was reported as %d/%s", result.Code, result.SQLState) + } +} + func TestConvertToMySQLError_SQLitePrimaryCodes(t *testing.T) { tests := []struct { name string diff --git a/protocol/mysql_errors.go b/protocol/mysql_errors.go index 2a55cd8e..342e1e00 100644 --- a/protocol/mysql_errors.go +++ b/protocol/mysql_errors.go @@ -10,36 +10,39 @@ import ( // below protocol - which cannot import protocol without an import cycle - // can reference the same numbers instead of duplicating them. const ( - ErrCodeUnknown = mysqlcode.ErrCodeUnknown - ErrCodeBadNull = mysqlcode.ErrCodeBadNull - ErrCodeTableExists = mysqlcode.ErrCodeTableExists - ErrCodeBadField = mysqlcode.ErrCodeBadField - ErrCodeDupFieldName = mysqlcode.ErrCodeDupFieldName - ErrCodeDupEntry = mysqlcode.ErrCodeDupEntry - ErrCodeParseError = mysqlcode.ErrCodeParseError - ErrCodeTooBigRowsize = mysqlcode.ErrCodeTooBigRowsize - ErrCodeNoSuchTable = mysqlcode.ErrCodeNoSuchTable - ErrCodeNoDB = mysqlcode.ErrCodeNoDB - ErrCodeLockTimeout = mysqlcode.ErrCodeLockTimeout - ErrCodeDeadlock = mysqlcode.ErrCodeDeadlock - ErrCodeReadOnly = mysqlcode.ErrCodeReadOnly - ErrCodeServerShutdown = mysqlcode.ErrCodeServerShutdown - ErrCodeNoReferencedRow = mysqlcode.ErrCodeNoReferencedRow - ErrCodeCheckConstraint = mysqlcode.ErrCodeCheckConstraint + ErrCodeUnknown = mysqlcode.ErrCodeUnknown + ErrCodeBadNull = mysqlcode.ErrCodeBadNull + ErrCodeTableExists = mysqlcode.ErrCodeTableExists + ErrCodeBadField = mysqlcode.ErrCodeBadField + ErrCodeDupFieldName = mysqlcode.ErrCodeDupFieldName + ErrCodeDupEntry = mysqlcode.ErrCodeDupEntry + ErrCodeParseError = mysqlcode.ErrCodeParseError + ErrCodeTooBigRowsize = mysqlcode.ErrCodeTooBigRowsize + ErrCodeNoSuchTable = mysqlcode.ErrCodeNoSuchTable + ErrCodeNoDB = mysqlcode.ErrCodeNoDB + ErrCodeLockTimeout = mysqlcode.ErrCodeLockTimeout + ErrCodeDeadlock = mysqlcode.ErrCodeDeadlock + ErrCodeReadOnly = mysqlcode.ErrCodeReadOnly + ErrCodeServerShutdown = mysqlcode.ErrCodeServerShutdown + ErrCodeNoReferencedRow = mysqlcode.ErrCodeNoReferencedRow + ErrCodeCheckConstraint = mysqlcode.ErrCodeCheckConstraint + ErrCodeTableAccessDenied = mysqlcode.ErrCodeTableAccessDenied + ErrCodeDataOutOfRange = mysqlcode.ErrCodeDataOutOfRange ) // SQLSTATE constants. Values live in protocol/mysqlcode; see above. const ( - SQLStateGeneral = mysqlcode.SQLStateGeneral - SQLStateIntegrity = mysqlcode.SQLStateIntegrity - SQLStateSyntax = mysqlcode.SQLStateSyntax - SQLStateDeadlock = mysqlcode.SQLStateDeadlock - SQLStateTableExists = mysqlcode.SQLStateTableExists - SQLStateNoSuchTable = mysqlcode.SQLStateNoSuchTable - SQLStateNoSuchCol = mysqlcode.SQLStateNoSuchCol - SQLStateDupColumn = mysqlcode.SQLStateDupColumn - SQLStateConnFailure = mysqlcode.SQLStateConnFailure - SQLStateNoDB = mysqlcode.SQLStateNoDB + SQLStateGeneral = mysqlcode.SQLStateGeneral + SQLStateIntegrity = mysqlcode.SQLStateIntegrity + SQLStateSyntax = mysqlcode.SQLStateSyntax + SQLStateDeadlock = mysqlcode.SQLStateDeadlock + SQLStateTableExists = mysqlcode.SQLStateTableExists + SQLStateNoSuchTable = mysqlcode.SQLStateNoSuchTable + SQLStateNoSuchCol = mysqlcode.SQLStateNoSuchCol + SQLStateDupColumn = mysqlcode.SQLStateDupColumn + SQLStateConnFailure = mysqlcode.SQLStateConnFailure + SQLStateNoDB = mysqlcode.SQLStateNoDB + SQLStateDataOutOfRange = mysqlcode.SQLStateDataOutOfRange ) // MySQLError represents a MySQL protocol error with error code and SQLSTATE diff --git a/protocol/mysqlcode/mysqlcode.go b/protocol/mysqlcode/mysqlcode.go index 66a0b8c7..1b21dee8 100644 --- a/protocol/mysqlcode/mysqlcode.go +++ b/protocol/mysqlcode/mysqlcode.go @@ -27,6 +27,14 @@ const ( // ErrCodeNotSupportedYet is MySQL's ER_NOT_SUPPORTED_YET: the server // understands the statement but does not implement this form of it. ErrCodeNotSupportedYet uint16 = 1235 + + // ErrCodeTableAccessDenied is MySQL's ER_TABLEACCESS_DENIED_ERROR: the + // table exists but the client is not permitted to touch it. + ErrCodeTableAccessDenied uint16 = 1142 + + // ErrCodeDataOutOfRange is MySQL's ER_WARN_DATA_OUT_OF_RANGE: a value used + // in an operation is outside the range its target can represent. + ErrCodeDataOutOfRange uint16 = 1264 ) // SQLSTATE constants. @@ -43,4 +51,7 @@ const ( // used for ER_SERVER_SHUTDOWN. SQLStateConnFailure = "08S01" SQLStateNoDB = "3D000" + // SQLStateDataOutOfRange is MySQL's SQLSTATE class for + // ER_WARN_DATA_OUT_OF_RANGE. + SQLStateDataOutOfRange = "22003" ) diff --git a/protocol/query/pipeline.go b/protocol/query/pipeline.go index 0aac5abb..c0987ca2 100644 --- a/protocol/query/pipeline.go +++ b/protocol/query/pipeline.go @@ -1,9 +1,11 @@ package query import ( + "sort" "strings" "github.com/maxpert/marmot/id" + "github.com/maxpert/marmot/protocol/query/rules" "github.com/rs/zerolog/log" ) @@ -25,6 +27,7 @@ func NewPipeline(cacheSize int, idGen id.Generator) (*Pipeline, error) { if err != nil { return nil, err } + registerClaimTableGuard(t) return &Pipeline{ parser: p, @@ -32,6 +35,20 @@ func NewPipeline(cacheSize int, idGen id.Generator) (*Pipeline, error) { }, nil } +// registerClaimTableGuard adds rules.ClaimTableGuardRule to the transpiler's +// rule set. It reaches into Transpiler.transformRules directly, rather than +// editing the literal in transpiler.go, because pipeline.go and +// transpiler.go share this package - this is the pipeline construction step +// registering one more rule, not a new extension mechanism. The set is +// re-sorted by priority exactly as NewTranspiler itself sorts it, since the +// guard (priority 0) must run before every rule already in the set. +func registerClaimTableGuard(t *Transpiler) { + t.transformRules = append(t.transformRules, &rules.ClaimTableGuardRule{}) + sort.Slice(t.transformRules, func(i, j int) bool { + return t.transformRules[i].Priority() < t.transformRules[j].Priority() + }) +} + // Close releases any resources held by the pipeline. Currently a no-op. func (p *Pipeline) Close() { // No resources to clean up diff --git a/protocol/query/rules/claim_table_guard.go b/protocol/query/rules/claim_table_guard.go new file mode 100644 index 00000000..a497171b --- /dev/null +++ b/protocol/query/rules/claim_table_guard.go @@ -0,0 +1,160 @@ +package rules + +import ( + "strings" + + "github.com/maxpert/marmot/common" + "github.com/maxpert/marmot/protocol/mysqlcode" + "github.com/maxpert/marmot/protocol/query/transform" + "vitess.io/vitess/go/vt/sqlparser" +) + +// ClaimTableGuardRule refuses client SQL that could mutate one of Marmot's own +// internal tables (see common.IsInternalTableName) - most importantly +// __marmot__autoinc, the AUTO_INCREMENT claim table Marmot keeps in its system +// database keyed by (database, table). db.AutoIncClaimStore, driven by the +// COMMIT handler and by DDL-time seeding, is the ONLY writer of that table; it never goes through this pipeline (see the doc +// comment below). A client able to INSERT, UPDATE, DELETE or DROP/ALTER/ +// TRUNCATE/RENAME it, or CREATE TABLE over it, could make a live +// AUTO_INCREMENT table re-mint ids it has already handed out - the entire +// safety argument for narrow AUTO_INCREMENT rests on that row being +// authoritative and untouchable by a client. +// +// SELECT is deliberately NOT rejected: the table is already hidden from SHOW +// TABLES and INFORMATION_SCHEMA (see protocol/handlers/metadata.go and +// protocol/handlers/information_schema.go), and a read cannot corrupt the +// claim - only a write to the row or a schema change to the table can. So +// this rule only inspects statement shapes that write data or schema +// (INSERT/REPLACE/UPDATE/DELETE and DDL), never SELECT. +// +// The check is a structural match on the parsed AST's table name(s), never a +// scan of the raw SQL text, and it is a case-insensitive PREFIX match on +// common.InternalTablePrefix rather than an exact-name comparison: the +// "__marmot__" prefix already means "marmot internal" everywhere else in this +// codebase (the CDC skip in db/preupdate_hook.go, the SHOW TABLES / +// INFORMATION_SCHEMA filters), so this protects that whole class of hidden +// tables, not just the one that exists today. It is NOT a copy of the SQLite +// "name NOT LIKE '__marmot__%'" filters used elsewhere: SQLite's LIKE treats +// "_" as a single-character wildcard, so that pattern is broader than it +// looks and would be the wrong thing to port into Go as an exact match. +// +// This rule only ever sees client-facing SQL. Its one execution path is: +// query.Pipeline.Process -> query.Transpiler.Transpile -> this rule, and +// Pipeline.Process is called only from protocol.ParseStatement* and +// MySQLServer.handleStmtPrepare, both client connection handlers. Replicated +// writes (db/replication_engine.go) and the COMMIT handler that actually +// writes the claim row (db/autoinc_claim.go AutoIncClaimStore.ApplyClaims) apply +// directly against SQLite and never call this pipeline, so this rule cannot +// see - and cannot reject - either of them. +type ClaimTableGuardRule struct{} + +func (r *ClaimTableGuardRule) Name() string { return "ClaimTableGuard" } + +// Priority is 0 so this rule runs before every other transform rule: a +// rejection must be decided on the AST exactly as the client sent it, before +// anything else rewrites it. +func (r *ClaimTableGuardRule) Priority() int { return 0 } + +// Transform refuses the statement if it writes to, or changes the schema of, +// one of Marmot's internal tables; otherwise it reports itself not +// applicable so the transpiler moves on to the next rule unchanged. +func (r *ClaimTableGuardRule) Transform( + stmt sqlparser.Statement, + _ []interface{}, + _ transform.SchemaProvider, + _ string, + _ transform.Serializer, +) ([]transform.TranspiledStatement, error) { + table, command := protectedTargetTable(stmt) + if table == "" { + return nil, transform.ErrRuleNotApplicable + } + + return nil, transform.NewCodedError(mysqlcode.ErrCodeTableAccessDenied, + "%s command denied to user for table '%s'", command, table) +} + +// protectedTargetTable returns the internal table name a write or DDL +// statement targets, and the MySQL command name for the error message. It +// returns "" for any statement that does not write to, or change the schema +// of, an internal table - including SELECT, which is never inspected here. +func protectedTargetTable(stmt sqlparser.Statement) (table, command string) { + switch s := stmt.(type) { + case sqlparser.DDLStatement: + // Covers CREATE TABLE, ALTER TABLE, DROP TABLE, TRUNCATE TABLE and + // RENAME TABLE. AffectedTables() includes RENAME TABLE's destination + // name too, so renaming an ordinary table ONTO the claim table's name + // is caught as well as renaming the claim table away. + if name := firstInternalTable(s.AffectedTables()); name != "" { + return name, strings.ToUpper(s.GetAction().ToString()) + } + case *sqlparser.Insert: + if s.Table == nil { + return "", "" + } + if tn, ok := s.Table.Expr.(sqlparser.TableName); ok { + if name := tn.Name.String(); common.IsInternalTableName(name) { + command = "INSERT" + if s.Action == sqlparser.ReplaceAct { + command = "REPLACE" + } + return name, command + } + } + case *sqlparser.Update: + if name := firstInternalTableExpr(s.TableExprs); name != "" { + return name, "UPDATE" + } + case *sqlparser.Delete: + if name := firstInternalTable(s.Targets); name != "" { + return name, "DELETE" + } + if name := firstInternalTableExpr(s.TableExprs); name != "" { + return name, "DELETE" + } + } + return "", "" +} + +// firstInternalTable returns the first internal table name in names, or "". +func firstInternalTable(names sqlparser.TableNames) string { + for _, tn := range names { + if name := tn.Name.String(); common.IsInternalTableName(name) { + return name + } + } + return "" +} + +// firstInternalTableExpr walks a FROM-list looking for a reference to an +// internal table, recursing into JOINs and parenthesized lists. A bare table +// reference resolves via AliasedTableExpr.Expr; a derived table (subquery) +// has no TableName and is skipped, since it cannot itself be the internal +// table. +func firstInternalTableExpr(exprs []sqlparser.TableExpr) string { + for _, expr := range exprs { + if name := internalTableInExpr(expr); name != "" { + return name + } + } + return "" +} + +func internalTableInExpr(expr sqlparser.TableExpr) string { + switch e := expr.(type) { + case *sqlparser.AliasedTableExpr: + if tn, ok := e.Expr.(sqlparser.TableName); ok { + if name := tn.Name.String(); common.IsInternalTableName(name) { + return name + } + } + case *sqlparser.JoinTableExpr: + if name := internalTableInExpr(e.LeftExpr); name != "" { + return name + } + return internalTableInExpr(e.RightExpr) + case *sqlparser.ParenTableExpr: + return firstInternalTableExpr(e.Exprs) + } + return "" +} diff --git a/protocol/query/rules/claim_table_guard_test.go b/protocol/query/rules/claim_table_guard_test.go new file mode 100644 index 00000000..f786109f --- /dev/null +++ b/protocol/query/rules/claim_table_guard_test.go @@ -0,0 +1,98 @@ +package rules + +import ( + "errors" + "testing" + + "github.com/maxpert/marmot/common" + "github.com/maxpert/marmot/protocol/mysqlcode" + "github.com/maxpert/marmot/protocol/query/transform" +) + +func TestClaimTableGuardRule_NameAndPriority(t *testing.T) { + rule := &ClaimTableGuardRule{} + if got := rule.Name(); got != "ClaimTableGuard" { + t.Errorf("Name() = %q, want %q", got, "ClaimTableGuard") + } + // Must run before every other transform rule, so a rejection is decided + // on the AST exactly as the client sent it. + if got := rule.Priority(); got != 0 { + t.Errorf("Priority() = %d, want 0", got) + } +} + +func TestClaimTableGuardRule_Transform(t *testing.T) { + tests := []struct { + name string + sql string + wantBlocked bool + }{ + {name: "INSERT into claim table", sql: "INSERT INTO __marmot__autoinc (tbl, base, owner, granted_at) VALUES ('t', 1, 1, 1)", wantBlocked: true}, + {name: "REPLACE into claim table", sql: "REPLACE INTO __marmot__autoinc (tbl, base, owner, granted_at) VALUES ('t', 1, 1, 1)", wantBlocked: true}, + {name: "UPDATE claim table", sql: "UPDATE __marmot__autoinc SET base = 100 WHERE tbl = 't'", wantBlocked: true}, + {name: "DELETE from claim table", sql: "DELETE FROM __marmot__autoinc WHERE tbl = 't'", wantBlocked: true}, + {name: "DROP TABLE claim table", sql: "DROP TABLE __marmot__autoinc", wantBlocked: true}, + {name: "ALTER TABLE claim table", sql: "ALTER TABLE __marmot__autoinc ADD COLUMN extra INT", wantBlocked: true}, + {name: "TRUNCATE TABLE claim table", sql: "TRUNCATE TABLE __marmot__autoinc", wantBlocked: true}, + {name: "RENAME TABLE away from claim table", sql: "RENAME TABLE __marmot__autoinc TO some_table", wantBlocked: true}, + {name: "RENAME TABLE onto claim table", sql: "RENAME TABLE some_table TO __marmot__autoinc", wantBlocked: true}, + {name: "CREATE TABLE re-creating claim table", sql: "CREATE TABLE __marmot__autoinc (tbl TEXT PRIMARY KEY)", wantBlocked: true}, + {name: "case-varied claim table name", sql: "DELETE FROM __MARMOT__AUTOINC WHERE tbl = 't'", wantBlocked: true}, + {name: "backtick-quoted claim table name", sql: "DELETE FROM `__marmot__autoinc` WHERE tbl = 't'", wantBlocked: true}, + {name: "database-qualified claim table name", sql: "DELETE FROM `mydb`.`__marmot__autoinc` WHERE tbl = 't'", wantBlocked: true}, + + {name: "SELECT from claim table is allowed", sql: "SELECT * FROM __marmot__autoinc", wantBlocked: false}, + {name: "ordinary table with similar prefix", sql: "DELETE FROM marmot_autoinc WHERE tbl = 't'", wantBlocked: false}, + {name: "ordinary table embedding the name", sql: "DELETE FROM x__marmot__autoincy WHERE tbl = 't'", wantBlocked: false}, + {name: "unrelated INSERT", sql: "INSERT INTO users (id, name) VALUES (1, 'a')", wantBlocked: false}, + {name: "unrelated UPDATE", sql: "UPDATE users SET name = 'b' WHERE id = 1", wantBlocked: false}, + {name: "unrelated DDL", sql: "DROP TABLE users", wantBlocked: false}, + } + + rule := &ClaimTableGuardRule{} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stmt := parseOne(t, tt.sql) + + results, err := rule.Transform(stmt, nil, nil, "", nil) + + if !tt.wantBlocked { + if !errors.Is(err, transform.ErrRuleNotApplicable) { + t.Fatalf("Transform() error = %v, want ErrRuleNotApplicable", err) + } + if results != nil { + t.Fatalf("Transform() results = %v, want nil", results) + } + return + } + + var coded *transform.CodedError + if !errors.As(err, &coded) { + t.Fatalf("Transform() error = %v (%T), want *transform.CodedError", err, err) + } + if coded.Code != mysqlcode.ErrCodeTableAccessDenied { + t.Errorf("Code = %d, want %d (ER_TABLEACCESS_DENIED_ERROR)", coded.Code, mysqlcode.ErrCodeTableAccessDenied) + } + if results != nil { + t.Errorf("Transform() results = %v, want nil on rejection", results) + } + }) + } +} + +// TestClaimTableGuardRule_ConstantIsSingleSource pins that the rule guards +// exactly common.AutoIncClaimTableName - the one Go literal for the claim +// table's name, shared with db.AutoIncClaimTable. +func TestClaimTableGuardRule_ConstantIsSingleSource(t *testing.T) { + if common.AutoIncClaimTableName != "__marmot__autoinc" { + t.Fatalf("common.AutoIncClaimTableName = %q, want __marmot__autoinc", common.AutoIncClaimTableName) + } + + rule := &ClaimTableGuardRule{} + sql := "DELETE FROM " + common.AutoIncClaimTableName + " WHERE tbl = 't'" + _, err := rule.Transform(parseOne(t, sql), nil, nil, "", nil) + var coded *transform.CodedError + if !errors.As(err, &coded) { + t.Fatalf("Transform() error = %v, want *transform.CodedError", err) + } +} diff --git a/protocol/query/transform/alter_table_column_type.go b/protocol/query/transform/alter_table_column_type.go index f37dfc7e..6788f453 100644 --- a/protocol/query/transform/alter_table_column_type.go +++ b/protocol/query/transform/alter_table_column_type.go @@ -1,6 +1,9 @@ package transform import ( + "strings" + + "github.com/maxpert/marmot/protocol/query/transform/intmarker" "vitess.io/vitess/go/vt/sqlparser" ) @@ -34,27 +37,96 @@ func (r *AlterTableColumnTypeRule) Transform(stmt sqlparser.Statement, params [] return nil, ErrRuleNotApplicable } + // A table-level "AUTO_INCREMENT=N" rides as its own alter_option (a bare + // sqlparser.TableOptions), separate from whichever option retypes the + // column - e.g. "ALTER TABLE t MODIFY id INT AUTO_INCREMENT, AUTO_INCREMENT=N". + // Order between them in AlterOptions is not guaranteed, so this is a + // dedicated pass rather than a case in the switch below. + floor := uint64(0) + sawTableOptions := false + declaresFloor := false + keptOptions := make([]sqlparser.AlterOption, 0, len(alter.AlterOptions)) for _, opt := range alter.AlterOptions { + if tableOpts, ok := opt.(sqlparser.TableOptions); ok { + floor = autoIncFloorFromOptions(tableOpts) + declaresFloor = declaresFloor || hasAutoIncOption(tableOpts) + sawTableOptions = true + continue + } + keptOptions = append(keptOptions, opt) + } + + // The declared floor has exactly one place to go: the width marker of a + // narrow AUTO_INCREMENT column this same ALTER defines, which is where + // DDL-time seeding reads it from (db/autoinc_seed.go autoIncSeedFloor). + // With no such column the floor would be dropped, and MySQL never ignores + // it: a client reserving the ids below N would later be handed allocator + // ids inside that range. So the statement is refused rather than run + // without it - including the option-only "ALTER TABLE t AUTO_INCREMENT=N". + if declaresFloor && !takesAutoIncFloor(keptOptions) { + return nil, NewCodedError(ErrCodeNotSupportedYet, + "This version of MySQL doesn't yet support 'ALTER TABLE `%s` AUTO_INCREMENT=N without a narrow AUTO_INCREMENT column definition in the same statement'", + alter.Table.Name.String()) + } + + // SQLite's ALTER TABLE grammar has no table options at all, so a + // "AUTO_INCREMENT=N" (or any other table option) riding on the statement + // reaches SQLite as a bare ", AUTO_INCREMENT=800" and is a syntax error. + // They are dropped here for the same reason CREATE TABLE drops + // TableSpec.Options (protocol/query/transform/create_table.go); a declared + // floor is carried into the column's width marker below. + // + // An ALTER whose only options are other table options has nothing left to + // serialize, so it is left untouched rather than rewritten into invalid + // SQL. + if sawTableOptions && len(keptOptions) > 0 { + alter.AlterOptions = keptOptions + } + + for _, colType := range alteredColumnTypes(alter.AlterOptions) { + stripMySQLColumnType(colType) + collapseIntegerTypeWithMarker(colType, floor) + } + + return nil, ErrRuleNotApplicable +} + +// alteredColumnTypes returns the type of every column an ALTER defines: added +// columns, and the new definition of a MODIFY or CHANGE (CHANGE renames as well +// as retypes, so a marker follows the new name). +func alteredColumnTypes(opts []sqlparser.AlterOption) []*sqlparser.ColumnType { + var types []*sqlparser.ColumnType + for _, opt := range opts { switch o := opt.(type) { case *sqlparser.AddColumns: for _, col := range o.Columns { - stripMySQLColumnType(col.Type) - collapseIntegerTypeWithMarker(col.Type) + types = append(types, col.Type) } case *sqlparser.ModifyColumn: if o.NewColDefinition != nil { - stripMySQLColumnType(o.NewColDefinition.Type) - collapseIntegerTypeWithMarker(o.NewColDefinition.Type) + types = append(types, o.NewColDefinition.Type) } case *sqlparser.ChangeColumn: - // CHANGE renames as well as retypes; the marker travels with the - // new column definition, so it follows the new name. if o.NewColDefinition != nil { - stripMySQLColumnType(o.NewColDefinition.Type) - collapseIntegerTypeWithMarker(o.NewColDefinition.Type) + types = append(types, o.NewColDefinition.Type) } } } + return types +} - return nil, ErrRuleNotApplicable +// takesAutoIncFloor reports whether any column the ALTER defines is a narrow +// AUTO_INCREMENT integer, the only column whose marker carries a floor +// (collapseIntegerTypeWithMarker). It must run before that function, which +// clears the AUTO_INCREMENT attribute. +func takesAutoIncFloor(opts []sqlparser.AlterOption) bool { + for _, colType := range alteredColumnTypes(opts) { + if colType == nil || colType.Options == nil || !colType.Options.Autoincrement { + continue + } + if _, narrow := intmarker.BitsForType(strings.ToUpper(colType.Type)); narrow { + return true + } + } + return false } diff --git a/protocol/query/transform/alter_table_column_type_test.go b/protocol/query/transform/alter_table_column_type_test.go index 2d975a10..7fb9fac2 100644 --- a/protocol/query/transform/alter_table_column_type_test.go +++ b/protocol/query/transform/alter_table_column_type_test.go @@ -1,6 +1,7 @@ package transform import ( + "errors" "strings" "testing" @@ -62,6 +63,75 @@ func TestAlterTableEmitsWidthMarker(t *testing.T) { } } +// TestAlterTableAutoIncrementOptionBecomesMarkerFloor pins the ALTER side of +// the AUTO_INCREMENT=N floor: "ALTER TABLE t ADD COLUMN id INT AUTO_INCREMENT, +// AUTO_INCREMENT=5000" must carry N-1 into the new column's marker exactly +// like the CREATE TABLE path does, even though the AUTO_INCREMENT=N option +// and the column definition are two separate, unordered AlterOptions. +// +// Mutation: read alter.AlterOptions for the floor only inside the switch's +// *sqlparser.AddColumns case instead of in the dedicated pre-pass; a floor +// option that is not exactly the first AlterOption is missed. +func TestAlterTableAutoIncrementOptionBecomesMarkerFloor(t *testing.T) { + stmt, err := sqlparser.NewTestParser().Parse( + "ALTER TABLE t ADD COLUMN id INT AUTO_INCREMENT, AUTO_INCREMENT=5000") + if err != nil { + t.Fatalf("parse: %v", err) + } + if _, err := (&AlterTableColumnTypeRule{}).Transform(stmt, nil, nil, "testdb", &SQLiteSerializer{}); err != ErrRuleNotApplicable { + t.Fatalf("Transform returned %v, want ErrRuleNotApplicable", err) + } + got, ok := alteredColumnType(stmt, "id") + if !ok { + t.Fatal("column id not found") + } + want := "INTEGER /*M:32a:4999*/" + if got != want { + t.Errorf("declared type = %q, want %q", got, want) + } +} + +// TestAlterTableDropsTableOptionsSQLiteCannotParse pins the other half of the +// same floor: the AUTO_INCREMENT=N option must not merely be READ, it +// must also be REMOVED from the statement. SQLite's ALTER TABLE grammar takes +// no table options, so one left in place arrives as a bare ", AUTO_INCREMENT=800" +// and the whole DDL fails with "near \",\": syntax error" - which is exactly +// how the cluster test TestAutoIncSeed_AlterAddColumnWithFloorOption failed +// before this. +// +// Mutation: keep every AlterOption (drop the keptOptions rewrite in +// AlterTableColumnTypeRule.Transform). The serialized statement then still +// carries AUTO_INCREMENT=800 and the first assertion fires. +func TestAlterTableDropsTableOptionsSQLiteCannotParse(t *testing.T) { + stmt, err := sqlparser.NewTestParser().Parse( + "ALTER TABLE t ADD COLUMN cnt INT AUTO_INCREMENT, AUTO_INCREMENT=800") + if err != nil { + t.Fatalf("parse: %v", err) + } + if _, err := (&AlterTableColumnTypeRule{}).Transform(stmt, nil, nil, "testdb", &SQLiteSerializer{}); err != ErrRuleNotApplicable { + t.Fatalf("Transform returned %v, want ErrRuleNotApplicable", err) + } + + // Vitess renders the table option as "AUTO_INCREMENT 800" (no equals + // sign), and the rule has already stripped the keyword from the column + // definition, so any occurrence left in the serialized statement is the + // table option SQLite cannot parse. + serialized := (&SQLiteSerializer{}).Serialize(stmt) + if strings.Contains(strings.ToUpper(serialized), "AUTO_INCREMENT") { + t.Errorf("serialized ALTER still carries a table option SQLite cannot parse:\n%s", serialized) + } + + // The floor is not lost with the option: it rides into the marker, which + // is where DDL-time seeding reads it from. + got, ok := alteredColumnType(stmt, "cnt") + if !ok { + t.Fatal("column cnt not found") + } + if want := "INTEGER /*M:32a:799*/"; got != want { + t.Errorf("declared type = %q, want %q", got, want) + } +} + // alteredColumnType returns the declared type of one column in an ALTER. func alteredColumnType(stmt sqlparser.Statement, column string) (string, bool) { alter, ok := stmt.(*sqlparser.AlterTable) @@ -88,3 +158,37 @@ func alteredColumnType(stmt sqlparser.Statement, column string) (string, bool) { } return "", false } + +// TestAlterTableRefusesAFloorNoColumnTakes: MySQL never ignores +// AUTO_INCREMENT=N on an ALTER. The floor can only ride into the marker of a +// narrow AUTO_INCREMENT column the same ALTER defines, so every other shape +// is refused with ER_NOT_SUPPORTED_YET (1235, SQLSTATE 42000: the server +// understands the statement and does not implement this form) rather than run +// with the floor silently dropped. +// +// Mutation: remove the declaresFloor refusal in AlterTableColumnTypeRule.Transform. +// Each case then returns ErrRuleNotApplicable and "the floor was dropped +// instead of refused" fires. +func TestAlterTableRefusesAFloorNoColumnTakes(t *testing.T) { + for _, sql := range []string{ + "ALTER TABLE t AUTO_INCREMENT=800", + "ALTER TABLE t ADD COLUMN note TEXT, AUTO_INCREMENT=5000", + "ALTER TABLE t ADD COLUMN big BIGINT AUTO_INCREMENT, AUTO_INCREMENT=5000", + "ALTER TABLE t ADD COLUMN plain INT, AUTO_INCREMENT=5000", + } { + t.Run(sql, func(t *testing.T) { + stmt, err := sqlparser.NewTestParser().Parse(sql) + if err != nil { + t.Fatalf("parse: %v", err) + } + _, err = (&AlterTableColumnTypeRule{}).Transform(stmt, nil, nil, "testdb", &SQLiteSerializer{}) + var coded *CodedError + if !errors.As(err, &coded) { + t.Fatalf("the floor was dropped instead of refused: Transform returned %v", err) + } + if coded.Code != ErrCodeNotSupportedYet { + t.Errorf("code = %d, want %d (ER_NOT_SUPPORTED_YET)", coded.Code, ErrCodeNotSupportedYet) + } + }) + } +} diff --git a/protocol/query/transform/int_type.go b/protocol/query/transform/int_type.go index fe00850a..e6da1c4f 100644 --- a/protocol/query/transform/int_type.go +++ b/protocol/query/transform/int_type.go @@ -36,6 +36,10 @@ func (r *IntTypeRule) Transform(stmt sqlparser.Statement, params []interface{}, return nil, ErrRuleNotApplicable } + // Read BEFORE CreateTableRule (priority 10, runs after this rule in the + // same transpile pass) clears TableSpec.Options for SQLite serialization. + floor := autoIncFloorFromOptions(create.TableSpec.Options) + modified := false for _, col := range create.TableSpec.Columns { @@ -50,7 +54,7 @@ func (r *IntTypeRule) Transform(stmt sqlparser.Statement, params []interface{}, continue } - if collapseIntegerTypeWithMarker(col.Type) { + if collapseIntegerTypeWithMarker(col.Type, floor) { modified = true } } diff --git a/protocol/query/transform/int_type_test.go b/protocol/query/transform/int_type_test.go index 1d60dbaa..c551cafe 100644 --- a/protocol/query/transform/int_type_test.go +++ b/protocol/query/transform/int_type_test.go @@ -380,8 +380,8 @@ func TestIntTypeRuleMarkerSurvivesSerialization(t *testing.T) { } } -// TestIntegerSpelledOutIsMarked records a deliberate deviation from FINAL.md -// touch point 1, which says Encode returns "" for BIGINT *and* INTEGER. +// TestIntegerSpelledOutIsMarked records a deliberate choice: INTEGER is +// marked, although a looser reading would leave BIGINT *and* INTEGER unmarked. // // Marking INTEGER is correct and the design text is loose: in MySQL, INTEGER is // a synonym for INT and is 32 bits, so a column declared INTEGER AUTO_INCREMENT @@ -416,3 +416,86 @@ func TestIntegerSpelledOutIsMarked(t *testing.T) { }) } } + +// TestCreateTableAutoIncrementOptionBecomesMarkerFloor pins the +// AUTO_INCREMENT=N floor end to end: a client's table-level AUTO_INCREMENT=N option must survive +// the transpiler not as a table option (CreateTableRule discards +// TableSpec.Options entirely - see create_table.go) but as the floor riding in +// the AUTO_INCREMENT column's own marker, because DDL replicates as raw SQL +// text and every node must derive the same floor from that text alone. +// +// Runs both rules in the same priority order the transpiler pipeline uses +// (IntTypeRule, priority 5, mutates the AST; CreateTableRule, priority 10, +// clears TableSpec.Options and serializes), matching how transpiler.go +// actually drives them. +// +// Mutation: read the AUTO_INCREMENT=N option after CreateTableRule has +// already nilled TableSpec.Options; floor is always 0 and this test's core +// assertion (":4999") never appears. +func TestCreateTableAutoIncrementOptionBecomesMarkerFloor(t *testing.T) { + stmt, err := sqlparser.NewTestParser().Parse( + "CREATE TABLE t (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT) AUTO_INCREMENT=5000") + if err != nil { + t.Fatalf("parse: %v", err) + } + + if _, err := (&IntTypeRule{}).Transform(stmt, nil, nil, "testdb", &SQLiteSerializer{}); err != ErrRuleNotApplicable { + t.Fatalf("IntTypeRule.Transform returned %v, want ErrRuleNotApplicable", err) + } + + results, err := (&CreateTableRule{}).Transform(stmt, nil, nil, "testdb", &SQLiteSerializer{}) + if err != nil { + t.Fatalf("CreateTableRule.Transform: %v", err) + } + if len(results) == 0 { + t.Fatal("expected at least one statement") + } + sql := results[0].SQL + + // N=5000 means the next id MySQL issues is 5000, so the stored base floor + // is N-1: the id treated as already issued just before it. + if !strings.Contains(sql, "/*M:32a:4999*/") { + t.Fatalf("emitted SQL missing floor marker /*M:32a:4999*/, got: %q", sql) + } + // The raw table option must not itself leak into the SQLite text - SQLite + // has no AUTO_INCREMENT=N table option syntax. + if strings.Contains(strings.ToUpper(sql), "AUTO_INCREMENT=5000") || + strings.Contains(strings.ToUpper(sql), "AUTO_INCREMENT = 5000") { + t.Fatalf("emitted SQL leaked the raw table option: %q", sql) + } + + decoded := intmarker.Decode(sql) + got, ok := decoded["id"] + if !ok { + t.Fatalf("Decode found no marker for column id in: %q", sql) + } + want := intmarker.Attributes{Bits: 32, ExplicitAutoInc: true, AutoIncFloor: 4999} + if got != want { + t.Errorf("decoded attributes = %+v, want %+v", got, want) + } +} + +// TestCreateTableNoAutoIncrementOptionMeansNoFloor guards the other direction: +// a CREATE TABLE with no AUTO_INCREMENT=N table option must emit a marker +// byte-identical to what the transpiler produced before floors existed. +func TestCreateTableNoAutoIncrementOptionMeansNoFloor(t *testing.T) { + stmt, err := sqlparser.NewTestParser().Parse( + "CREATE TABLE t (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + if err != nil { + t.Fatalf("parse: %v", err) + } + if _, err := (&IntTypeRule{}).Transform(stmt, nil, nil, "testdb", &SQLiteSerializer{}); err != ErrRuleNotApplicable { + t.Fatalf("IntTypeRule.Transform returned %v, want ErrRuleNotApplicable", err) + } + results, err := (&CreateTableRule{}).Transform(stmt, nil, nil, "testdb", &SQLiteSerializer{}) + if err != nil { + t.Fatalf("CreateTableRule.Transform: %v", err) + } + sql := results[0].SQL + if !strings.Contains(sql, "/*M:32a*/") { + t.Fatalf("expected unfloored marker /*M:32a*/, got: %q", sql) + } + if strings.Contains(sql, ":4999") || strings.Contains(sql, "/*M:32a:") { + t.Fatalf("no AUTO_INCREMENT=N was declared, but a floor was emitted anyway: %q", sql) + } +} diff --git a/protocol/query/transform/intmarker/intmarker.go b/protocol/query/transform/intmarker/intmarker.go index 504f0455..ce869581 100644 --- a/protocol/query/transform/intmarker/intmarker.go +++ b/protocol/query/transform/intmarker/intmarker.go @@ -9,8 +9,10 @@ // a comment beside the type, which SQLite preserves verbatim in // sqlite_master.sql and normalises away everywhere else. // -// The grammar is /*M:[u][a]*/ - bits in {8,16,24,32}, u for UNSIGNED, -// a for an explicitly declared AUTO_INCREMENT. It is deliberately NOT MySQL's +// The grammar is /*M:[u][a][:]*/ - bits in {8,16,24,32}, u for +// UNSIGNED, a for an explicitly declared AUTO_INCREMENT, and an optional +// decimal - the base floor a client's AUTO_INCREMENT=N table option +// declared (N-1), present only when non-zero. It is deliberately NOT MySQL's // /*! ... */ executable-comment syntax: that syntax asks another MySQL to // execute the contents, which is the opposite of what this is for. package intmarker @@ -36,6 +38,10 @@ type Attributes struct { // ExplicitAutoInc records that the column was declared AUTO_INCREMENT, // as opposed to merely being a narrow integer. ExplicitAutoInc bool + // AutoIncFloor is the base floor a client's AUTO_INCREMENT=N table option + // declared: N-1. Zero means the client did not declare one. Only ever set + // on the marker of the column ExplicitAutoInc names. + AutoIncFloor uint64 } // Marked reports whether a marker was present. @@ -93,6 +99,10 @@ func Encode(a Attributes) string { if a.ExplicitAutoInc { b.WriteByte('a') } + if a.AutoIncFloor != 0 { + b.WriteByte(':') + b.WriteString(strconv.FormatUint(a.AutoIncFloor, 10)) + } b.WriteString(markerClose) return b.String() } @@ -117,14 +127,24 @@ func parseMarkerBody(body string) (Attributes, error) { default: return a, fmt.Errorf("marker width %d is not one of 8, 16, 24, 32", bits) } - for ; i < len(body); i++ { + for i < len(body) { switch body[i] { case 'u': a.Unsigned = true + i++ case 'a': a.ExplicitAutoInc = true + i++ + case ':': + floorStr := body[i+1:] + floor, err := strconv.ParseUint(floorStr, 10, 64) + if err != nil { + return Attributes{}, fmt.Errorf("marker floor %q: %w", floorStr, err) + } + a.AutoIncFloor = floor + i = len(body) default: - return a, fmt.Errorf("marker flag %q is not u or a", body[i]) + return a, fmt.Errorf("marker flag %q is not u, a or :", body[i]) } } return a, nil diff --git a/protocol/query/transform/intmarker/intmarker_test.go b/protocol/query/transform/intmarker/intmarker_test.go index 6bd219f2..7e239da6 100644 --- a/protocol/query/transform/intmarker/intmarker_test.go +++ b/protocol/query/transform/intmarker/intmarker_test.go @@ -45,7 +45,7 @@ func TestEncodeRoundTrip(t *testing.T) { } // TestWidthMax is the single source of the ceiling every other component -// derives its bound from, so the numbers are pinned against FINAL.md's table +// derives its bound from, so the numbers are pinned against MySQL's ranges // rather than recomputed at each use. // // Mutation: use 1<" extension to the marker grammar +// that carries AUTO_INCREMENT=N. A marker with no floor must render byte-identical +// to what Encode produced before AutoIncFloor existed, and a floor must +// round-trip through Decode exactly like every other flag. +// +// Mutation: encode the floor even when it is 0, or drop it from parseMarkerBody; +// the "no floor" and "round trip" cases below both fire. +func TestAutoIncFloorGrammar(t *testing.T) { + cases := []struct { + name string + attrs Attributes + want string + }{ + {"auto-increment, no declared floor, byte-identical to pre-floor grammar", + Attributes{Bits: 32, ExplicitAutoInc: true}, "/*M:32a*/"}, + {"auto-increment, unsigned, no declared floor, byte-identical", + Attributes{Bits: 16, Unsigned: true, ExplicitAutoInc: true}, "/*M:16ua*/"}, + {"auto-increment with declared floor", + Attributes{Bits: 32, ExplicitAutoInc: true, AutoIncFloor: 4999}, "/*M:32a:4999*/"}, + {"unsigned auto-increment with declared floor", + Attributes{Bits: 8, Unsigned: true, ExplicitAutoInc: true, AutoIncFloor: 200}, "/*M:8ua:200*/"}, + {"plain narrow column, no auto-increment, floor is meaningless and stays off", + Attributes{Bits: 16}, "/*M:16*/"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := Encode(tc.attrs) + if got != tc.want { + t.Fatalf("Encode = %q, want %q", got, tc.want) + } + decoded := Decode("CREATE TABLE t (id INTEGER " + got + " PRIMARY KEY)") + attrs, ok := decoded["id"] + if !ok { + t.Fatalf("Decode lost the marker %q entirely", got) + } + if attrs != tc.attrs { + t.Errorf("round trip = %+v, want %+v", attrs, tc.attrs) + } + }) + } +} + +// TestDecodeMalformedFloorIsRejected mirrors the existing "malformed marker +// degrades to no marker" contract for the new ":" suffix: a floor that +// is not a valid decimal number must fail to parse, and Decode's caller-facing +// behaviour is to omit the column entirely rather than propagate a partial +// Attributes. +func TestDecodeMalformedFloorIsRejected(t *testing.T) { + cases := []string{ + "CREATE TABLE t (id INTEGER /*M:32a:*/ PRIMARY KEY)", // empty floor + "CREATE TABLE t (id INTEGER /*M:32a:12x*/ PRIMARY KEY)", // non-numeric floor + "CREATE TABLE t (id INTEGER /*M:32:4999a*/ PRIMARY KEY)", // floor before a flag + } + for _, sql := range cases { + t.Run(sql, func(t *testing.T) { + decoded := Decode(sql) + if _, ok := decoded["id"]; ok { + t.Fatalf("expected malformed marker to yield no entry, got %+v", decoded["id"]) + } + }) + } +} + +// TestStripRemovesFloorMarker pins that Strip (decode.go:229), which matches +// only markerOpen/markerClose and treats everything between them as one +// opaque span, already removes the ":" extension (TestAutoIncFloorGrammar) +// with no code change: a longer body between the same two delimiters is still +// one span to Strip. A multi-column CREATE TABLE carries a floor marker on one +// column and a plain marker on another so both forms, and the spacing repair +// around each, are proven in the same pass. +// +// Mutation: require the close brace immediately after the flags, breaking the +// floor form's longer span; this fires because the floor marker survives. +func TestStripRemovesFloorMarker(t *testing.T) { + in := "CREATE TABLE t (id INTEGER /*M:32a:4999*/ PRIMARY KEY, count INTEGER /*M:16u*/ DEFAULT 0, v TEXT)" + want := "CREATE TABLE t (id INTEGER PRIMARY KEY, count INTEGER DEFAULT 0, v TEXT)" + if got := Strip(in); got != want { + t.Errorf("Strip =\n %q\nwant\n %q", got, want) + } +} diff --git a/protocol/query/transform/table_utils.go b/protocol/query/transform/table_utils.go index b7fb6269..680f95cd 100644 --- a/protocol/query/transform/table_utils.go +++ b/protocol/query/transform/table_utils.go @@ -2,6 +2,7 @@ package transform import ( "fmt" + "strconv" "strings" "github.com/maxpert/marmot/protocol/query/transform/intmarker" @@ -87,6 +88,11 @@ func searchTableExpr(expr sqlparser.TableExpr, targetAlias string) (tableName, a // single type SQLite has, carrying the declared width as a marker comment, and // strips the modifiers SQLite rejects. It reports whether it changed anything. // +// floor is the base floor a table-level AUTO_INCREMENT=N option declared (see +// autoIncFloorFromOptions); it is encoded into the marker only when this +// column is the one being declared AUTO_INCREMENT, so an unrelated narrow +// column never carries another column's floor. +// // Shared by CREATE TABLE (IntTypeRule) and ALTER TABLE ADD/MODIFY/CHANGE COLUMN // (AlterTableColumnTypeRule). Before it was shared, ALTER only ran // stripMySQLColumnType, which does not collapse the type and does not remove @@ -98,7 +104,7 @@ func searchTableExpr(expr sqlparser.TableExpr, targetAlias string) (tableName, a // would make LAST_INSERT_ID() report an unrelated internal rowid. BIGINT and // non-integer types get no marker, so their DDL text is byte-identical to what // it was before markers existed. -func collapseIntegerTypeWithMarker(colType *sqlparser.ColumnType) bool { +func collapseIntegerTypeWithMarker(colType *sqlparser.ColumnType, floor uint64) bool { if colType == nil { return false } @@ -109,13 +115,18 @@ func collapseIntegerTypeWithMarker(colType *sqlparser.ColumnType) bool { } // Capture the declared width BEFORE the strips below erase it. + autoInc := colType.Options != nil && colType.Options.Autoincrement marker := "" if bits, narrow := intmarker.BitsForType(upperType); narrow { - marker = intmarker.Encode(intmarker.Attributes{ + attrs := intmarker.Attributes{ Bits: bits, Unsigned: colType.Unsigned, - ExplicitAutoInc: colType.Options != nil && colType.Options.Autoincrement, - }) + ExplicitAutoInc: autoInc, + } + if autoInc { + attrs.AutoIncFloor = floor + } + marker = intmarker.Encode(attrs) } modified := false @@ -144,3 +155,39 @@ func collapseIntegerTypeWithMarker(colType *sqlparser.ColumnType) bool { } return modified } + +// hasAutoIncOption reports whether a table-option list declares +// AUTO_INCREMENT=N, whatever N is. +func hasAutoIncOption(opts sqlparser.TableOptions) bool { + for _, opt := range opts { + if opt != nil && strings.EqualFold(opt.Name, "auto_increment") { + return true + } + } + return false +} + +// autoIncFloorFromOptions returns the base floor a table's declared +// AUTO_INCREMENT=N option implies: N-1, the id MySQL semantics treat as +// already issued just before the next insert. It returns 0 - "no declared +// floor" - when the option is absent, non-numeric, or N itself is 0: a +// client writing AUTO_INCREMENT=0 is not asking for a floor below the first +// id. +// +// Read from a CREATE TABLE's TableSpec.Options (an sqlparser.TableOptions +// directly) or from one ALTER TABLE alter_option of that same type - MySQL +// accepts "ALTER TABLE t ... , AUTO_INCREMENT=N" as an alter option alongside +// a column change in the same statement. +func autoIncFloorFromOptions(opts sqlparser.TableOptions) uint64 { + for _, opt := range opts { + if opt == nil || opt.Value == nil || !strings.EqualFold(opt.Name, "auto_increment") { + continue + } + n, err := strconv.ParseUint(opt.Value.Val, 10, 64) + if err != nil || n == 0 { + return 0 + } + return n - 1 + } + return 0 +} diff --git a/protocol/query/transpiler_rule_error_test.go b/protocol/query/transpiler_rule_error_test.go index 5efd6daa..d5b9f71f 100644 --- a/protocol/query/transpiler_rule_error_test.go +++ b/protocol/query/transpiler_rule_error_test.go @@ -93,3 +93,28 @@ func TestProcessLeavesTranspileErrNilOnParseFailure(t *testing.T) { t.Errorf("TranspileErr = %v for a parse failure; want nil", ctx.Output.TranspileErr) } } + +// TestTranspileRefusesAnAlterFloorNoColumnTakes pins that the ALTER refusal +// reaches the protocol layer as ER_NOT_SUPPORTED_YET through the real rule set, +// rather than the floor being dropped or the statement reaching SQLite, which +// would report a syntax error (1064). +func TestTranspileRefusesAnAlterFloorNoColumnTakes(t *testing.T) { + pipeline, err := NewPipeline(1000, &mockIDGenerator{}) + if err != nil { + t.Fatalf("NewPipeline: %v", err) + } + for _, sql := range []string{ + "ALTER TABLE orders AUTO_INCREMENT=5000", + "ALTER TABLE orders ADD COLUMN note TEXT, AUTO_INCREMENT=5000", + } { + ctx := NewContext(sql, nil) + ctx.SchemaLookup = mockSchemaLookup(ruleErrorSchemas) + if err := pipeline.parser.Parse(ctx); err != nil { + t.Fatalf("parse %q: %v", sql, err) + } + coded, ok := pipeline.transpiler.Transpile(ctx).(*transform.CodedError) + if !ok || coded.Code != transform.ErrCodeNotSupportedYet { + t.Errorf("%q: Transpile did not refuse with ER_NOT_SUPPORTED_YET", sql) + } + } +} diff --git a/protocol/server.go b/protocol/server.go index 8a79f945..c5d24d1d 100644 --- a/protocol/server.go +++ b/protocol/server.go @@ -1383,7 +1383,7 @@ func (s *MySQLServer) handleStmtExecute(conn net.Conn, session *ConnectionSessio offset, val, err = parseParamValue(payload, offset, paramType, unsigned) if err != nil { if errors.Is(err, errUnsignedBigintOutOfRange) { - _ = s.writeErrorWithState(conn, 1, 1264, "22003", + _ = s.writeErrorWithState(conn, 1, ErrCodeDataOutOfRange, SQLStateDataOutOfRange, fmt.Sprintf("Out of range value for parameter %d", i)) } else { _ = s.writeError(conn, 1, ErrCodeParseError, fmt.Sprintf("Failed to parse parameter %d: %v", i, err)) diff --git a/protocol/transaction.go b/protocol/transaction.go index d53d22b3..9da98591 100644 --- a/protocol/transaction.go +++ b/protocol/transaction.go @@ -128,6 +128,21 @@ type Statement struct { IntentKey []byte `msgpack:"IntentKey"` // Intent key for MVCC conflict detection (binary format) Error string `msgpack:"Error"` // Error message if Type is StatementUnsupported + // AutoIDClaim marks this statement as a narrow AUTO_INCREMENT range claim + // rather than ordinary DML. The claim's payload travels in the + // transaction's intent DataSnapshot, not here. + // + // It is a flag on an existing StatementType, never a new StatementType: an + // older binary ignores the unknown wire field, sees a DML statement with no + // row image, and rejects it at the "DML prepare missing encoded CDC row" + // gate, which is a clean failed transaction rather than a statement it + // mishandles. + AutoIDClaim bool `msgpack:"AutoIDClaim,omitempty"` + // AutoIDClaimPayload is the claim itself, msgpack-encoded. Each participant + // writes it into its own transaction intent at PREPARE; the COMMIT handler + // reads it back from there to write the new base. + AutoIDClaimPayload []byte `msgpack:"AutoIDClaimPayload,omitempty"` + // TranspileErr is the typed error transpilation failed with, set only when // a transformation rule refused the statement (never when parsing failed). // It carries the MySQL error code the client must see; Error holds the same diff --git a/telemetry/collector.go b/telemetry/collector.go index cd92e835..76b87cb6 100644 --- a/telemetry/collector.go +++ b/telemetry/collector.go @@ -7,7 +7,7 @@ import ( // StatsProvider interface for components that provide stats type StatsProvider interface { - GetRowLockStats() (activeLocks, activeTransactions, gcMarkers, tablesWithLocks int) + GetRowLockStats() (activeLocks, activeTransactions, tablesWithLocks int) IntentStats() (pendingIntents int, err error) } @@ -69,7 +69,7 @@ func (mc *MetricsCollector) collect() { return } - var totalLocks, totalTxns, totalGC, totalTables, totalIntents int + var totalLocks, totalTxns, totalTables, totalIntents int for _, dbName := range mc.dbLister.ListDatabases() { provider := mc.dbLister.GetDatabase(dbName) @@ -77,10 +77,9 @@ func (mc *MetricsCollector) collect() { continue } - locks, txns, gc, tables := provider.GetRowLockStats() + locks, txns, tables := provider.GetRowLockStats() totalLocks += locks totalTxns += txns - totalGC += gc totalTables += tables if intents, err := provider.IntentStats(); err == nil { @@ -88,6 +87,6 @@ func (mc *MetricsCollector) collect() { } } - UpdateRowLockStats(totalLocks, totalTxns, totalGC, totalTables) + UpdateRowLockStats(totalLocks, totalTxns, totalTables) PendingIntents.Set(float64(totalIntents)) } diff --git a/telemetry/metrics.go b/telemetry/metrics.go index c0eee743..91950305 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -134,9 +134,6 @@ var ( // RowLockTransactions tracks number of transactions holding row locks RowLockTransactions Gauge = NoopStat{} - // RowLockGCMarkers tracks number of GC markers pending cleanup - RowLockGCMarkers Gauge = NoopStat{} - // RowLockTables tracks number of tables with active locks RowLockTables Gauge = NoopStat{} ) @@ -309,10 +306,6 @@ func InitMetrics() { "row_lock_transactions", "Number of transactions holding row locks", ) - RowLockGCMarkers = NewGauge( - "row_lock_gc_markers", - "Number of GC markers pending cleanup", - ) RowLockTables = NewGauge( "row_lock_tables", "Number of tables with active locks", @@ -320,9 +313,8 @@ func InitMetrics() { } // UpdateRowLockStats updates row lock gauges from stats -func UpdateRowLockStats(activeLocks, activeTransactions, gcMarkers, tablesWithLocks int) { +func UpdateRowLockStats(activeLocks, activeTransactions, tablesWithLocks int) { RowLocksActive.Set(float64(activeLocks)) RowLockTransactions.Set(float64(activeTransactions)) - RowLockGCMarkers.Set(float64(gcMarkers)) RowLockTables.Set(float64(tablesWithLocks)) } diff --git a/test/autoinc_claim_cluster_test.go b/test/autoinc_claim_cluster_test.go new file mode 100644 index 00000000..7ff074d4 --- /dev/null +++ b/test/autoinc_claim_cluster_test.go @@ -0,0 +1,503 @@ +package test + +// Cluster tests for the AUTO_INCREMENT feature's DDL-time seeding +// (coordinator.ClaimRange, db.AutoIncClaimStore, db/autoinc_seed.go) and for +// the hidden claim table's invisibility and write-protection, all driven +// over the real MySQL wire protocol against a 3-node ClusterHarness +// (test/crash_recovery_test.go). +// +// coordinator.WriteCoordinator.ClaimRange itself has no caller reachable +// over the MySQL wire protocol or any admin endpoint yet - the range +// allocator that will wire an INSERT against a narrow AUTO_INCREMENT table +// to ClaimRange has not landed. Its only callers today are in-process Go +// tests (coordinator/autoinc_claim_test.go). This ClusterHarness only talks +// to a node over its MySQL listener (database/sql + the MySQL wire +// protocol) and a small JSON admin API for gossip membership - neither can +// drive a claim - so the claim-protocol scenarios (sequential claim safety, +// concurrent boundary claims, restart and crash recovery, meta-store wipe, +// full-cluster snapshot restore) are covered instead in +// test/autoinc_claim_inprocess_test.go, which drives the real coordinator +// and participant engines directly, in-process, replacing only the gRPC +// transport. Standing up an ad hoc coordinator inside test/ that dials each +// node's gRPC PrepareTransaction/CommitTransaction RPCs directly was +// considered and rejected: WriteCoordinator.localReplicator is written +// assuming in-process co-location with the participant it calls "local" +// (coordinator/write_coordinator.go dispatches with wc.nodeID against it), +// so reproducing it faithfully from outside the process would mean +// re-deriving quorum membership, retry and rejection-base bookkeeping that +// belongs to the coordinator itself, not to a test fixture. + +import ( + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/go-sql-driver/mysql" + _ "github.com/mattn/go-sqlite3" + "github.com/maxpert/marmot/common" + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/protocol/mysqlcode" +) + +// systemDBFile returns the on-disk path of a node's system database, the +// file db.SystemDatabaseName resolves to under its data directory +// (db/database_manager.go: filepath.Join(dataDir, SystemDatabaseName+".db")). +// __marmot__autoinc lives inside this file, never inside a user database +// (db/autoinc_claim.go's AutoIncClaimTable doc). +func systemDBFile(dataDir string) string { + return filepath.Join(dataDir, db.SystemDatabaseName+".db") +} + +// readAutoIncBase opens a node's system database file directly, outside the +// MySQL wire protocol, and reads the committed base for (database, table) +// from __marmot__autoinc. This is the only channel available to a cluster +// test: the table is deliberately hidden from client SQL (see +// common.IsInternalTableName and protocol/query/rules/claim_table_guard.go), +// and nothing on the wire path reveals it. found is false, with no error, +// when the row is absent - not yet seeded, or the table itself does not yet +// exist - matching db.AutoIncClaimStore.ReadBase's own "absent row, absent +// table" equivalence. +func readAutoIncBase(dataDir, database, table string) (base uint64, found bool, err error) { + dsn := fmt.Sprintf("file:%s?mode=ro&_busy_timeout=5000", systemDBFile(dataDir)) + conn, openErr := sql.Open("sqlite3", dsn) + if openErr != nil { + return 0, false, fmt.Errorf("open system db under %s: %w", dataDir, openErr) + } + defer conn.Close() + + var got int64 + scanErr := conn.QueryRow( + "SELECT base FROM "+common.AutoIncClaimTableName+" WHERE db = ? AND tbl = ?", + database, table).Scan(&got) + switch { + case scanErr == sql.ErrNoRows: + return 0, false, nil + case scanErr != nil && strings.Contains(scanErr.Error(), "no such table"): + return 0, false, nil + case scanErr != nil: + return 0, false, fmt.Errorf("read %s under %s: %w", common.AutoIncClaimTableName, dataDir, scanErr) + } + return uint64(got), true, nil +} + +// waitForAutoIncBaseAtLeast polls a node's system database until the +// committed base for (database, table) is >= min, or timeout elapses. DDL +// seeding commits the claim row atomically with the DDL's own COMMIT, before +// that COMMIT is ACKed (db/autoinc_seed.go, db/autoinc_claim.go Seed), so by +// the time WaitForTableExists reports a table visible on a node the seed row +// should already be durable there; this still polls rather than reading +// once, so the assertion is not coupled to that timing guarantee holding +// exactly. +func waitForAutoIncBaseAtLeast(t *testing.T, dataDir, database, table string, min uint64, timeout time.Duration) uint64 { + t.Helper() + deadline := time.Now().Add(timeout) + var base uint64 + var found bool + var lastErr error + for time.Now().Before(deadline) { + base, found, lastErr = readAutoIncBase(dataDir, database, table) + if lastErr == nil && found && base >= min { + return base + } + time.Sleep(200 * time.Millisecond) + } + switch { + case lastErr != nil: + t.Fatalf("reading auto-increment base for %s.%s under %s: %v", database, table, dataDir, lastErr) + case !found: + t.Fatalf("no auto-increment claim row for %s.%s under %s after %v", database, table, dataDir, timeout) + default: + t.Fatalf("auto-increment base for %s.%s under %s = %d after %v, want >= %d", + database, table, dataDir, base, timeout, min) + } + return 0 +} + +// queryStringColumn runs query against db and returns every row's value for +// the named column (matched case-insensitively, since MySQL identifiers are +// case-insensitive), scanning generically so it works regardless of how many +// other columns the result set carries - information_schema.tables returns a +// fixed wide row shape unrelated to the client's SELECT list. +func queryStringColumn(conn *sql.DB, query, colName string, args ...interface{}) ([]string, error) { + rows, err := conn.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + cols, err := rows.Columns() + if err != nil { + return nil, err + } + idx := -1 + for i, c := range cols { + if strings.EqualFold(c, colName) { + idx = i + break + } + } + if idx == -1 { + return nil, fmt.Errorf("column %q not found in result columns %v", colName, cols) + } + + var out []string + for rows.Next() { + dest := make([]interface{}, len(cols)) + raw := make([]sql.NullString, len(cols)) + for i := range dest { + dest[i] = &raw[i] + } + if err := rows.Scan(dest...); err != nil { + return nil, err + } + out = append(out, raw[idx].String) + } + return out, rows.Err() +} + +// asMySQLError extracts *mysql.MySQLError from err via errors.As, failing +// the test if err is not one - a raw driver/transport error is a different +// failure than the coded rejection these tests assert on. +func asMySQLError(t *testing.T, err error, context string) *mysql.MySQLError { + t.Helper() + if err == nil { + t.Fatalf("%s: expected an error, got nil", context) + } + var mysqlErr *mysql.MySQLError + if !errors.As(err, &mysqlErr) { + t.Fatalf("%s: error %v is not a *mysql.MySQLError", context, err) + } + return mysqlErr +} + +// =========================================================================== +// Item 6: DDL-time seeding +// =========================================================================== + +// TestAutoIncSeed_CreateTableWithFloorOption exercises the shape DDL-time +// seeding actually supports: ALTER ... MODIFY/CHANGE to retroactively tag an +// existing BIGINT column AUTO_INCREMENT cannot land, because vitess emits +// MySQL ALTER syntax SQLite cannot apply. CREATE TABLE carrying a +// table-level AUTO_INCREMENT=N option is unaffected by that limitation and +// is the shape production DDL actually uses, so it is what this test +// drives: CREATE TABLE ... AUTO_INCREMENT=5000 must seed base >= 4999 (N-1: +// protocol/query/transform/table_utils.go autoIncFloorFromOptions) on every +// node, since each node seeds from its own DDL apply independently +// (db/autoinc_seed.go). +func TestAutoIncSeed_CreateTableWithFloorOption(t *testing.T) { + harness := NewClusterHarness(t) + defer harness.Cleanup() + + if err := harness.StartCluster(); err != nil { + t.Fatalf("StartCluster: %v", err) + } + + const table = "autoinc_seed_floor" + allNodes := []int{1, 2, 3} + + if _, err := harness.ExecNode(1, fmt.Sprintf( + "CREATE TABLE %s (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT) AUTO_INCREMENT=5000", table)); err != nil { + t.Fatalf("CREATE TABLE ... AUTO_INCREMENT=5000: %v", err) + } + if err := harness.WaitForTableExists(table, allNodes, 15*time.Second); err != nil { + harness.dumpNodeLogs("autoinc_seed_floor_ddl") + t.Fatalf("DDL did not replicate: %v", err) + } + + for _, nodeID := range allNodes { + dataDir := harness.Nodes[nodeID-1].DataDir + base := waitForAutoIncBaseAtLeast(t, dataDir, db.DefaultDatabaseName, table, 4999, 15*time.Second) + t.Logf("node %d: seeded base = %d (want >= 4999)", nodeID, base) + } +} + +// TestAutoIncSeed_AlterSeedsFromExistingMax exercises DDL-time seeding's +// derivation from a table's actual MAX(id), not just a declared floor +// option: rows carrying explicit ids are inserted directly (DML never seeds +// - only a DDL that changes the table's CREATE text does, db/autoinc_seed.go), +// so the claim base must still reflect CREATE TABLE's own empty-table seed +// right up until an unrelated ALTER TABLE ADD COLUMN re-triggers seeding and +// picks up MAX(id) fresh. +func TestAutoIncSeed_AlterSeedsFromExistingMax(t *testing.T) { + harness := NewClusterHarness(t) + defer harness.Cleanup() + + if err := harness.StartCluster(); err != nil { + t.Fatalf("StartCluster: %v", err) + } + + const table = "autoinc_seed_altermax" + const rowCount = 1000 + const batchSize = 100 + allNodes := []int{1, 2, 3} + + if _, err := harness.ExecNode(1, fmt.Sprintf( + "CREATE TABLE %s (id INT AUTO_INCREMENT PRIMARY KEY, v VARCHAR(20))", table)); err != nil { + t.Fatalf("CREATE TABLE: %v", err) + } + if err := harness.WaitForTableExists(table, allNodes, 15*time.Second); err != nil { + harness.dumpNodeLogs("autoinc_seed_altermax_ddl") + t.Fatalf("DDL did not replicate: %v", err) + } + + for start := 1; start <= rowCount; start += batchSize { + var values strings.Builder + for i := start; i < start+batchSize; i++ { + if i > start { + values.WriteString(", ") + } + fmt.Fprintf(&values, "(%d, 'row_%d')", i, i) + } + if _, err := harness.ExecNode(1, fmt.Sprintf( + "INSERT INTO %s (id, v) VALUES %s", table, values.String())); err != nil { + t.Fatalf("explicit-id batch insert starting at %d: %v", start, err) + } + } + if err := harness.WaitForRowCount(table, allNodes, rowCount, 15*time.Second); err != nil { + t.Fatalf("explicit-id rows did not replicate: %v", err) + } + + // Before the ALTER: proves the assertion below is the ALTER's own + // seeding at work, not a base that was already >= rowCount for some + // other reason (e.g. DML itself seeding, which it must not). + preDataDir := harness.Nodes[0].DataDir + preBase, found, err := readAutoIncBase(preDataDir, db.DefaultDatabaseName, table) + if err != nil { + t.Fatalf("read pre-ALTER base: %v", err) + } + if !found { + t.Fatalf("no claim row for %s before ALTER", table) + } + if preBase >= rowCount { + t.Fatalf("pre-ALTER base = %d, want < %d (DML must not have seeded the claim row)", preBase, rowCount) + } + + if _, err := harness.ExecNode(1, fmt.Sprintf("ALTER TABLE %s ADD COLUMN w INT", table)); err != nil { + t.Fatalf("ALTER TABLE ADD COLUMN w: %v", err) + } + + for _, nodeID := range allNodes { + dataDir := harness.Nodes[nodeID-1].DataDir + base := waitForAutoIncBaseAtLeast(t, dataDir, db.DefaultDatabaseName, table, rowCount, 15*time.Second) + t.Logf("node %d: ALTER seeded base = %d (want >= %d)", nodeID, base, rowCount) + } +} + +// TestAutoIncSeed_AlterFloorWithoutAColumnIsRefused asserts the refusal +// AlterTableColumnTypeRule raises (protocol/query/transform/alter_table_column_type.go): +// an ALTER carrying an AUTO_INCREMENT=N option with no narrow AUTO_INCREMENT +// column definition in the same statement has nowhere to store that floor +// (it belongs in the marker of the column it types), so it is refused with +// MySQL error 1235 (ER_NOT_SUPPORTED_YET), SQLSTATE 42000, rather than +// silently dropped. This covers both shapes the rule refuses: the +// option riding alongside an unrelated column add, and the bare +// option-only form. +func TestAutoIncSeed_AlterFloorWithoutAColumnIsRefused(t *testing.T) { + harness := NewClusterHarness(t) + defer harness.Cleanup() + + if err := harness.StartCluster(); err != nil { + t.Fatalf("StartCluster: %v", err) + } + + const table = "autoinc_seed_floorrefused" + allNodes := []int{1, 2, 3} + + if _, err := harness.ExecNode(1, fmt.Sprintf( + "CREATE TABLE %s (id INT PRIMARY KEY, v TEXT)", table)); err != nil { + t.Fatalf("CREATE TABLE: %v", err) + } + if err := harness.WaitForTableExists(table, allNodes, 15*time.Second); err != nil { + harness.dumpNodeLogs("autoinc_seed_floorrefused_ddl") + t.Fatalf("DDL did not replicate: %v", err) + } + + statements := map[string]string{ + "ADD COLUMN alongside floor option": fmt.Sprintf( + "ALTER TABLE %s ADD COLUMN w INT, AUTO_INCREMENT=5000", table), + "floor option alone": fmt.Sprintf( + "ALTER TABLE %s AUTO_INCREMENT=5000", table), + } + + for label, stmt := range statements { + _, err := harness.ExecNode(1, stmt) + mysqlErr := asMySQLError(t, err, label) + if mysqlErr.Number != mysqlcode.ErrCodeNotSupportedYet { + t.Errorf("%s: error code = %d, want %d (ER_NOT_SUPPORTED_YET)", + label, mysqlErr.Number, mysqlcode.ErrCodeNotSupportedYet) + } + if got := string(mysqlErr.SQLState[:]); got != mysqlcode.SQLStateSyntax { + t.Errorf("%s: SQLSTATE = %q, want %q", label, got, mysqlcode.SQLStateSyntax) + } + } + + // Neither refused statement may have left column w behind. + if _, err := harness.ExecNode(1, fmt.Sprintf("SELECT w FROM %s LIMIT 1", table)); err == nil { + t.Errorf("column w exists on %s after both ALTERs were refused", table) + } +} + +// TestAutoIncSeed_RefusesWhenExistingMaxExceedsWidth: a TINYINT +// AUTO_INCREMENT column (widthMax = 127, intmarker.Attributes.WidthMax) whose +// table already holds a row above that ceiling must refuse the next DDL that +// touches that table's schema, with MySQL error 1264 +// (ER_WARN_DATA_OUT_OF_RANGE) / SQLSTATE 22003 +// (db/ddl_prepare_validate.go checkAutoIncWidthCeilings, +// AutoIncWidthExceededError). +// +// SQLite does not enforce MySQL's column width, so INSERT INTO ... (id, ...) +// VALUES (200, ...) against a TINYINT column succeeds; the refusal has to +// come from a later DDL. checkAutoIncWidthCeilings scopes to every table +// whose sqlite_master CREATE TABLE text this DDL statement changed - not +// only DDL that redeclares AUTO_INCREMENT - so an unrelated ADD COLUMN on +// the same table is enough to re-trigger the check and is the shape used +// here. +func TestAutoIncSeed_RefusesWhenExistingMaxExceedsWidth(t *testing.T) { + harness := NewClusterHarness(t) + defer harness.Cleanup() + + if err := harness.StartCluster(); err != nil { + t.Fatalf("StartCluster: %v", err) + } + + const table = "autoinc_width_ceiling" + allNodes := []int{1, 2, 3} + + if _, err := harness.ExecNode(1, fmt.Sprintf( + "CREATE TABLE %s (id TINYINT AUTO_INCREMENT PRIMARY KEY, v TEXT)", table)); err != nil { + t.Fatalf("CREATE TABLE: %v", err) + } + if err := harness.WaitForTableExists(table, allNodes, 15*time.Second); err != nil { + harness.dumpNodeLogs("autoinc_width_ceiling_ddl") + t.Fatalf("DDL did not replicate: %v", err) + } + + if _, err := harness.ExecNode(1, fmt.Sprintf( + "INSERT INTO %s (id, v) VALUES (200, 'beyond_tinyint')", table)); err != nil { + t.Fatalf("explicit-id INSERT beyond widthMax: %v", err) + } + if err := harness.WaitForRowCount(table, allNodes, 1, 15*time.Second); err != nil { + t.Fatalf("row did not replicate: %v", err) + } + + _, err := harness.ExecNode(1, fmt.Sprintf("ALTER TABLE %s ADD COLUMN extra INT", table)) + mysqlErr := asMySQLError(t, err, "ALTER TABLE ADD COLUMN on a table whose MAX(id) exceeds widthMax") + + if mysqlErr.Number != mysqlcode.ErrCodeDataOutOfRange { + t.Errorf("error code = %d, want %d (ER_WARN_DATA_OUT_OF_RANGE)", mysqlErr.Number, mysqlcode.ErrCodeDataOutOfRange) + } + if got := string(mysqlErr.SQLState[:]); got != mysqlcode.SQLStateDataOutOfRange { + t.Errorf("SQLSTATE = %q, want %q", got, mysqlcode.SQLStateDataOutOfRange) + } + t.Logf("refused as required: %v", mysqlErr) +} + +// =========================================================================== +// Item 7: the hidden table is invisible and write-protected +// =========================================================================== + +// TestAutoIncHiddenTable_AbsentFromListings asserts __marmot__autoinc never +// appears in SHOW TABLES or information_schema.tables on any node, for two +// independent reasons that both hold: it lives in the system database, which +// DatabaseManager never lists as a user database at all +// (db/autoinc_claim.go's placement doc), and the listing queries themselves +// filter the "__marmot__" prefix +// (protocol/handlers/metadata.go, protocol/handlers/information_schema.go). +func TestAutoIncHiddenTable_AbsentFromListings(t *testing.T) { + harness := NewClusterHarness(t) + defer harness.Cleanup() + + if err := harness.StartCluster(); err != nil { + t.Fatalf("StartCluster: %v", err) + } + + const table = "autoinc_hidden_listing" + allNodes := []int{1, 2, 3} + + // A table that DOES exist and must be seeded, so the system database + // definitely holds a __marmot__autoinc row by the time listings are + // checked - a table with no row at all would make this assertion trivial. + if _, err := harness.ExecNode(1, fmt.Sprintf( + "CREATE TABLE %s (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)", table)); err != nil { + t.Fatalf("CREATE TABLE: %v", err) + } + if err := harness.WaitForTableExists(table, allNodes, 15*time.Second); err != nil { + t.Fatalf("DDL did not replicate: %v", err) + } + for _, nodeID := range allNodes { + dataDir := harness.Nodes[nodeID-1].DataDir + waitForAutoIncBaseAtLeast(t, dataDir, db.DefaultDatabaseName, table, 0, 15*time.Second) + } + + for _, nodeID := range allNodes { + if harness.tableExistsOnNode(nodeID, common.AutoIncClaimTableName) { + t.Errorf("node %d: SHOW TABLES lists %s", nodeID, common.AutoIncClaimTableName) + } + + conn, err := harness.ConnectToNode(nodeID) + if err != nil { + t.Fatalf("connect to node %d: %v", nodeID, err) + } + // The schema name is formatted into the statement rather than bound as + // a parameter on purpose. A bound parameter makes the client use a + // prepared statement, and the prepared path answers COM_STMT_EXECUTE + // by re-running the TRANSPILED text (protocol/server.go:1411), which + // no longer parses as an INFORMATION_SCHEMA query - so the query lands + // on SQLite as "no such table: tables". That is a pre-existing gap in + // prepared-statement routing for every statement Marmot answers above + // the database, not a property of the claim table, and it is recorded + // as such rather than worked around silently. The value is a constant + // in this test, so formatting it injects nothing. + names, err := queryStringColumn(conn, fmt.Sprintf( + "SELECT table_name FROM information_schema.tables WHERE table_schema = '%s'", + db.DefaultDatabaseName), "table_name") + if err != nil { + t.Fatalf("node %d: information_schema.tables: %v", nodeID, err) + } + for _, name := range names { + if strings.EqualFold(name, common.AutoIncClaimTableName) { + t.Errorf("node %d: information_schema.tables lists %s", nodeID, common.AutoIncClaimTableName) + } + } + } +} + +// TestAutoIncHiddenTable_RejectsClientWrites asserts INSERT, UPDATE, DELETE +// and DROP TABLE against __marmot__autoinc are all refused with MySQL error +// 1142 (ER_TABLEACCESS_DENIED_ERROR), the code +// protocol/query/rules/claim_table_guard.go's ClaimTableGuardRule uses. The +// table need not exist in the user database for the rule to fire - it is a +// structural match on the parsed statement's target table name, evaluated +// before the statement ever reaches SQLite (ClaimTableGuardRule.Priority() +// == 0, runs before every other transform rule). +func TestAutoIncHiddenTable_RejectsClientWrites(t *testing.T) { + harness := NewClusterHarness(t) + defer harness.Cleanup() + + if err := harness.StartCluster(); err != nil { + t.Fatalf("StartCluster: %v", err) + } + + table := common.AutoIncClaimTableName + statements := map[string]string{ + "INSERT": fmt.Sprintf( + "INSERT INTO %s (db, tbl, base, owner, granted_at) VALUES ('marmot', 'x', 1, 1, 1)", table), + "UPDATE": fmt.Sprintf("UPDATE %s SET base = 999 WHERE tbl = 'x'", table), + "DELETE": fmt.Sprintf("DELETE FROM %s WHERE tbl = 'x'", table), + "DROP": fmt.Sprintf("DROP TABLE %s", table), + } + + for label, stmt := range statements { + _, err := harness.ExecNode(1, stmt) + mysqlErr := asMySQLError(t, err, label+" against "+table) + if mysqlErr.Number != mysqlcode.ErrCodeTableAccessDenied { + t.Errorf("%s: error code = %d, want %d (ER_TABLEACCESS_DENIED_ERROR)", + label, mysqlErr.Number, mysqlcode.ErrCodeTableAccessDenied) + } + t.Logf("%s correctly refused: %v", label, mysqlErr) + } +} diff --git a/test/autoinc_claim_inprocess_test.go b/test/autoinc_claim_inprocess_test.go new file mode 100644 index 00000000..ca32dc9d --- /dev/null +++ b/test/autoinc_claim_inprocess_test.go @@ -0,0 +1,1083 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package test + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/maxpert/marmot/coordinator" + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/db/snapshot" + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// This file drives the AUTO_INCREMENT range-claim protocol +// (coordinator.WriteCoordinator.ClaimRange, coordinator/autoinc_claim.go; +// db.ReplicationEngine.prepareAutoIncClaim, db/replication_engine.go:310) +// through the REAL coordinator and REAL participant engines, replacing only +// the gRPC transport. test/autoinc_claim_cluster_test.go correctly found +// that the multi-process harness (test/crash_recovery_test.go) cannot drive +// these five tests: nothing reachable over the MySQL wire calls ClaimRange +// yet (the range allocator is not wired). This fixture is the honest substitute: +// three real db.DatabaseManager + db.LocalReplicator + coordinator.WriteCoordinator +// triples, wired to each other by an in-process fanout instead of gRPC. + +// prepareRecord is one fanout-observed PREPARE round-trip for a node. +type prepareRecord struct { + req *coordinator.ReplicationRequest + resp *coordinator.ReplicationResponse + err error +} + +// fanoutReplicator implements coordinator.Replicator (coordinator/write_coordinator.go:35) +// by routing a ReplicateTransaction call to the target node's own +// *db.LocalReplicator (db/local_replicator.go:20). It is used as every +// WriteCoordinator's remote `replicator`; each coordinator's own +// `localReplicator` is its own node's LocalReplicator directly, matching +// how WriteCoordinator.executePreparePhase dispatches to self +// (coordinator/write_coordinator.go:779-781) versus otherNodes +// (coordinator/write_coordinator.go:774-776) - so a coordinator's own +// PREPARE/COMMIT of itself never passes through this fanout, only the calls +// it makes to its peers do. +// +// It also records every PREPARE round per node, mutex-guarded because +// ClaimRange fans PREPARE out to all nodes concurrently +// (coordinator/write_coordinator.go:774-781), so tests can assert a stale +// proposal was actually REJECTED by a specific participant rather than +// merely superseded. +type fanoutReplicator struct { + mu sync.Mutex + nodes map[uint64]*db.LocalReplicator + prepareCalls map[uint64][]prepareRecord +} + +func newFanoutReplicator() *fanoutReplicator { + return &fanoutReplicator{ + nodes: make(map[uint64]*db.LocalReplicator), + prepareCalls: make(map[uint64][]prepareRecord), + } +} + +func (f *fanoutReplicator) register(nodeID uint64, lr *db.LocalReplicator) { + f.mu.Lock() + defer f.mu.Unlock() + f.nodes[nodeID] = lr +} + +func (f *fanoutReplicator) ReplicateTransaction(ctx context.Context, nodeID uint64, req *coordinator.ReplicationRequest) (*coordinator.ReplicationResponse, error) { + f.mu.Lock() + lr := f.nodes[nodeID] + f.mu.Unlock() + + resp, err := lr.ReplicateTransaction(ctx, nodeID, req) + + if req.Phase == coordinator.PhasePrep { + f.mu.Lock() + f.prepareCalls[nodeID] = append(f.prepareCalls[nodeID], prepareRecord{req: req, resp: resp, err: err}) + f.mu.Unlock() + } + + return resp, err +} + +// prepareCallCount is a checkpoint marker: tests snapshot this before an +// operation and inspect only the records after it via prepareCallsSince. +func (f *fanoutReplicator) prepareCallCount(nodeID uint64) int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.prepareCalls[nodeID]) +} + +func (f *fanoutReplicator) prepareCallsSince(nodeID uint64, since int) []prepareRecord { + f.mu.Lock() + defer f.mu.Unlock() + all := f.prepareCalls[nodeID] + if since >= len(all) { + return nil + } + out := make([]prepareRecord, len(all)-since) + copy(out, all[since:]) + return out +} + +// fixedNodeProvider implements coordinator.NodeProvider (coordinator/node_provider.go:6) +// with a static, never-mutated node set - this fixture never simulates +// membership changes, only claim-protocol behaviour, so the interface's +// four methods are all that is needed. HasSeedNodes is false: this is a +// legitimate fixed-size deployment, not a node still learning its peers +// (coordinator/cluster.go:90's quorum-of-one guard), matching +// coordinator's own mockNodeProvider zero value +// (coordinator/full_replication_test.go:15-18). +type fixedNodeProvider struct { + nodes []uint64 +} + +func (p *fixedNodeProvider) GetAliveNodes() ([]uint64, error) { + out := make([]uint64, len(p.nodes)) + copy(out, p.nodes) + return out, nil +} + +func (p *fixedNodeProvider) GetClusterSize() int { return len(p.nodes) } +func (p *fixedNodeProvider) GetTotalMembershipSize() int { return len(p.nodes) } +func (p *fixedNodeProvider) HasSeedNodes() bool { return false } + +// inprocNode is one participant: its own data directory, clock, DatabaseManager, +// LocalReplicator (db/local_replicator.go:20, which owns the real +// *db.ReplicationEngine) and WriteCoordinator (coordinator/write_coordinator.go:117). +type inprocNode struct { + id uint64 + dataDir string + clock *hlc.Clock + dm *db.DatabaseManager + lr *db.LocalReplicator + wc *coordinator.WriteCoordinator +} + +// claimStore wraps this node's own system database as the AUTO_INCREMENT +// claim store (db/autoinc_claim.go:108, db/database_manager.go:504). +func (n *inprocNode) claimStore() *db.AutoIncClaimStore { + return db.NewAutoIncClaimStore(n.dm.GetSystemDatabase()) +} + +// inprocCluster is three inprocNodes sharing one fanoutReplicator and one +// fixedNodeProvider. +type inprocCluster struct { + t *testing.T + provider *fixedNodeProvider + fanout *fanoutReplicator + nodes map[uint64]*inprocNode +} + +const inprocCoordinatorTimeout = 30 * time.Second + +// newInprocCluster builds nodeIDs participants, each with its own t.TempDir() +// data directory (db/replication_engine_test.go:64-78's own-tempdir +// pattern). It registers a cleanup that closes whatever DatabaseManager is +// current for each node at test end, so a test that reopens a node (restart, +// meta wipe, snapshot restore) is still cleaned up correctly. +func newInprocCluster(t *testing.T, nodeIDs []uint64) *inprocCluster { + t.Helper() + c := &inprocCluster{ + t: t, + provider: &fixedNodeProvider{nodes: append([]uint64(nil), nodeIDs...)}, + fanout: newFanoutReplicator(), + nodes: make(map[uint64]*inprocNode), + } + for _, id := range nodeIDs { + c.nodes[id] = c.buildNode(id, t.TempDir()) + } + t.Cleanup(func() { + for _, n := range c.nodes { + _ = n.dm.Close() + } + }) + return c +} + +func (c *inprocCluster) buildNode(id uint64, dataDir string) *inprocNode { + c.t.Helper() + clock := hlc.NewClock(id) + dm, err := db.NewDatabaseManager(dataDir, id, clock) + require.NoError(c.t, err) + lr := db.NewLocalReplicator(id, dm, clock) + wc := coordinator.NewWriteCoordinator(id, c.provider, c.fanout, lr, inprocCoordinatorTimeout, clock) + c.fanout.register(id, lr) + return &inprocNode{id: id, dataDir: dataDir, clock: clock, dm: dm, lr: lr, wc: wc} +} + +// closeNode stops a node's DatabaseManager without reopening it. +func (c *inprocCluster) closeNode(id uint64) { + c.t.Helper() + require.NoError(c.t, c.nodes[id].dm.Close()) +} + +// reopenNodeFresh rebuilds a node's DatabaseManager, LocalReplicator and +// WriteCoordinator from its (possibly just-mutated) data directory and +// rewires the shared fanout, WITHOUT closing anything first - the caller +// must already have stopped and prepared the data directory. It reuses the +// node's own hlc.Clock: only storage identity is being reopened. +func (c *inprocCluster) reopenNodeFresh(id uint64) { + c.t.Helper() + old := c.nodes[id] + fresh := c.buildNode(id, old.dataDir) + fresh.clock = old.clock + c.nodes[id] = fresh +} + +// reopenNode closes then reopens a node from the same data directory - +// db.NewDatabaseManager(dataDir, ...) (db/database_manager.go:54) against a +// directory a prior DatabaseManager already populated, exactly like a +// process restart's storage-level effect. It does not model kill -9: a +// crash skips Close's graceful shutdown, which this cannot reach from +// inside one process. See TestClaimRange_RestartRecoversCommittedBase's doc +// comment. +func (c *inprocCluster) reopenNode(id uint64) { + c.closeNode(id) + c.reopenNodeFresh(id) +} + +// newInprocClusterFromDirs rebuilds an inprocCluster from data directories a +// prior process already populated - one per node ID - instead of creating +// fresh t.TempDir() storage the way newInprocCluster does. This is what a +// node process reopening its own on-disk state, potentially from a separate +// os.Process that no longer exists, looks like from the test's side; it is +// how TestClaimRange_KillNineRecoversCommittedBase inspects a cluster after +// killing the child process that built it. +func newInprocClusterFromDirs(t *testing.T, dataDirs map[uint64]string) *inprocCluster { + t.Helper() + nodeIDs := make([]uint64, 0, len(dataDirs)) + for id := range dataDirs { + nodeIDs = append(nodeIDs, id) + } + sort.Slice(nodeIDs, func(i, j int) bool { return nodeIDs[i] < nodeIDs[j] }) + + c := &inprocCluster{ + t: t, + provider: &fixedNodeProvider{nodes: nodeIDs}, + fanout: newFanoutReplicator(), + nodes: make(map[uint64]*inprocNode), + } + for _, id := range nodeIDs { + c.nodes[id] = c.buildNode(id, dataDirs[id]) + } + t.Cleanup(func() { + for _, n := range c.nodes { + _ = n.dm.Close() + } + }) + return c +} + +// driveDDL replicates DDL through one coordinator's WriteTransaction +// (coordinator/write_coordinator.go:139), the real 2PC path that also seeds +// __marmot__autoinc via db.TransactionManager.applyNonDMLIntents -> +// seedAutoIncBasesForDDL (db/autoinc_seed.go:97, wired by +// DatabaseManager.wireGCCoordination at db/database_manager.go:199). +func driveDDL(t *testing.T, node *inprocNode, database, table, ddl string) { + t.Helper() + startTS := node.clock.Now() + txn := &coordinator.Transaction{ + ID: startTS.ToTxnID(), + NodeID: node.id, + StartTS: startTS, + Database: database, + WriteConsistency: protocol.ConsistencyQuorum, + Statements: []protocol.Statement{{ + Type: protocol.StatementDDL, + SQL: ddl, + TableName: table, + Database: database, + }}, + } + require.NoError(t, node.wc.WriteTransaction(context.Background(), txn)) +} + +// waitForBase polls a node's own claim store until it reads `want` for +// (database, table) or a 5s deadline passes. Polling replaces a sleep: the +// commit phase only waits for (quorum-1) remote ACKs before returning +// (coordinator/write_coordinator.go:623,636-639,663-668), so a coordinator's +// WriteTransaction / ClaimRange can return success before every node - +// specifically the one node whose ACK it did not wait for - has actually +// applied the commit. +func waitForBase(t *testing.T, node *inprocNode, database, table string, want uint64) { + t.Helper() + store := node.claimStore() + deadline := time.Now().Add(5 * time.Second) + var last uint64 + var lastErr error + for { + last, lastErr = store.ReadBase(database, table) + if lastErr == nil && last == want { + return + } + if time.Now().After(deadline) { + break + } + time.Sleep(10 * time.Millisecond) + } + require.NoError(t, lastErr, "node %d: final ReadBase(%s.%s)", node.id, database, table) + require.Equal(t, want, last, "node %d: base for %s.%s did not converge in time", node.id, database, table) +} + +// waitForQuorumBase polls until at least minNodes of nodes read `want` for +// (database, table), or a 5s deadline passes. +// +// This is intentionally weaker than waitForBase's "every node" check, and +// only TestClaimRange_ConcurrentBoundary uses it. When two claimants +// genuinely race - PREPARE rounds literally overlapping, not one committing +// before the other starts - it is possible, and was observed directly in +// this fixture, for the LOSING side of one round's write-write conflict +// (db/meta_store_pebble.go:1109-1124, the ordinary MVCC lock on the shared +// intent key, protocol/autoinc_claim.go:53's AutoIncClaimKey) to be a +// participant that PREPARED and locked an EARLIER, different attempt but +// was not part of THAT attempt's winning quorum - so it never receives a +// COMMIT (it wasn't in prepResponses for the round it rejected) or an ABORT +// for the round it silently lost (WriteTransaction only calls +// abortTransaction when PREPARE fails outright, coordinator/write_coordinator.go:178-181, +// not when quorum succeeds without unanimity). That participant is now a +// genuine straggler on this key: coordinator/write_coordinator.go's own +// top-of-file design notes name "Straggler catch-up: Dead nodes sync via +// snapshots + delta logs" as a SEPARATE mechanism from the write path +// itself, and this minimal in-process fixture wires no gossip or +// anti-entropy to perform it. QUORUM (majority) is what +// protocol.ConsistencyQuorum actually promises +// (coordinator/quorum.go, coordinator/cluster.go:100); asserting literally +// every node converges after true concurrent contention asserts a stronger +// guarantee than the system gives, so this test checks the quorum the +// coordinator itself required (RequiredQuorum for 3 nodes = 2, coordinator/cluster.go:100). +func waitForQuorumBase(t *testing.T, nodes []*inprocNode, database, table string, want uint64, minNodes int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + var lastCount int + for { + lastCount = 0 + for _, n := range nodes { + if b, err := n.claimStore().ReadBase(database, table); err == nil && b == want { + lastCount++ + } + } + if lastCount >= minNodes { + return + } + if time.Now().After(deadline) { + break + } + time.Sleep(10 * time.Millisecond) + } + require.GreaterOrEqual(t, lastCount, minNodes, + "only %d/%d nodes converged to %d for %s.%s within the deadline", lastCount, len(nodes), want, database, table) +} + +// rangesDisjoint reports whether two claimed ranges (newBase, newBase+size] +// overlap. A claim's owned ids are newBase+1 .. newBase+size +// (db/autoinc_claim.go:229's doc: "The claimant owns newBase+1 .. newBase+size"). +func rangesDisjoint(newBase1, size1, newBase2, size2 uint64) bool { + lo1, hi1 := newBase1+1, newBase1+size1 + lo2, hi2 := newBase2+1, newBase2+size2 + return hi1 < lo2 || hi2 < lo1 +} + +const markedTableDDL = `CREATE TABLE %s (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)` + +// setupClaimCluster builds a 3-node in-process cluster, creates `database` on +// every node directly - CreateDatabase is not itself replicated by DDL, so +// each node's own DatabaseManager.CreateDatabase must be called explicitly - +// then drives the marked-table DDL through node +// 1's coordinator so all three nodes apply it and each seeds its own claim +// row through the real DDL-time seeding path. It verifies the seed landed on +// all three nodes before returning, which is also the fixture's proof that +// driving DDL through WriteTransaction DOES seed correctly - no fallback to +// direct Seed() calls was needed. +func setupClaimCluster(t *testing.T, database, table string) *inprocCluster { + t.Helper() + c := newInprocCluster(t, []uint64{1, 2, 3}) + for _, id := range []uint64{1, 2, 3} { + require.NoError(t, c.nodes[id].dm.CreateDatabase(database)) + } + assertAbsentClaimRowRejected(t, c, database, table+"_absent") + assertWidthCeilingRejected(t, c, database, table+"_narrow") + driveDDL(t, c.nodes[1], database, table, sprintfDDL(table)) + for _, id := range []uint64{1, 2, 3} { + waitForBase(t, c.nodes[id], database, table, 0) + } + return c +} + +// narrowMarkedTableDDL declares an 8-bit signed AUTO_INCREMENT column, whose +// ceiling (intmarker.Attributes.WidthMax, protocol/query/transform/intmarker/intmarker.go:57-65) +// is 2^7-1 = 127. +const narrowMarkedTableDDL = `CREATE TABLE %s (id INTEGER /*M:8a*/ PRIMARY KEY, v TEXT)` + +// assertWidthCeilingRejected creates a table with an 8-bit marked column +// (ceiling 127) through the normal replicated-DDL-plus-seeding path, then +// asserts a claim whose range exceeds that ceiling is rejected. +// +// This is run ahead of every one of the five tests for the same reason +// assertAbsentClaimRowRejected is: every one of prepareAutoIncClaim's +// rejection clauses needs at least one assertion that isolates it, or a +// mutation to that specific clause can slip through five tests whose own +// claim sizes never approach the (very large, 32-bit) ceiling their main +// table uses. +// +// Mutation: a regression that drops prepareAutoIncClaim's +// `newBase+size <= widthMax` clause (db/replication_engine.go:366-372) - +// nothing else in the condition depends on the column's width - would +// wrongly accept a range of 200 against a 127-ceiling column, where this +// assertion expects rejection. +func assertWidthCeilingRejected(t *testing.T, c *inprocCluster, database, table string) { + t.Helper() + driveDDL(t, c.nodes[1], database, table, strings.Replace(narrowMarkedTableDDL, "%s", table, 1)) + for _, id := range []uint64{1, 2, 3} { + waitForBase(t, c.nodes[id], database, table, 0) + } + + mark2 := c.fanout.prepareCallCount(2) + _, _, err := c.nodes[1].wc.ClaimRange(context.Background(), database, table, 0, 200) + require.Error(t, err, "a claim whose range exceeds the column's width ceiling must be rejected") + + round1Node2 := c.fanout.prepareCallsSince(2, mark2) + require.NotEmpty(t, round1Node2) + require.False(t, round1Node2[0].resp.Success) + require.True(t, round1Node2[0].resp.Rejected) + require.Contains(t, round1Node2[0].resp.Error, "exhausts the column", + "rejection must name the width-ceiling cause, not some other failure") +} + +func sprintfDDL(table string) string { + return strings.Replace(markedTableDDL, "%s", table, 1) +} + +// assertAbsentClaimRowRejected creates `table` directly on every node's own +// SQLite connection - bypassing the coordinator's replicated DDL and its +// automatic claim-row seeding (db/autoinc_seed.go's +// seedAutoIncBasesForDDL) - so the table has a valid, width-resolvable +// schema on every node but NO claim row anywhere, then asserts a claim +// against it is rejected. +// +// This isolates the absent-row rejection +// (db/replication_engine.go:337-343, db.ErrAutoIncBaseAbsent's doc at +// db/autoinc_claim.go:90-97) from the width check that runs right after it: +// with a real schema present, AutoIncWidthMax succeeds regardless, so only +// the absent-row check itself stands between this claim and acceptance. It +// is run ahead of every one of the five tests (setupClaimCluster is their +// shared entry point) because "the table has not been created yet" is a +// precondition every one of them briefly passes through. +// +// Mutation: a regression that treats an absent claim row as base 0 instead +// of rejecting would make prepareAutoIncClaim fall through here - with a +// real schema present, nothing else in the condition would stop the claim, +// so it would be wrongly accepted where this assertion expects rejection. +func assertAbsentClaimRowRejected(t *testing.T, c *inprocCluster, database, table string) { + t.Helper() + for _, id := range []uint64{1, 2, 3} { + mdb, err := c.nodes[id].dm.GetDatabase(database) + require.NoError(t, err) + _, err = mdb.GetWriteDB().Exec(sprintfDDL(table)) + require.NoError(t, err) + require.NoError(t, mdb.ReloadSchema()) + } + + mark2 := c.fanout.prepareCallCount(2) + _, _, err := c.nodes[1].wc.ClaimRange(context.Background(), database, table, 0, 1) + require.Error(t, err, "a claim against a table with a real schema but no claim row must be rejected, not silently accepted as base 0") + + round1Node2 := c.fanout.prepareCallsSince(2, mark2) + require.NotEmpty(t, round1Node2) + require.False(t, round1Node2[0].resp.Success) + require.True(t, round1Node2[0].resp.Rejected) + require.Contains(t, round1Node2[0].resp.Error, "no auto-increment claim row", + "rejection must name the absent-row cause, not some other failure") +} + +// --------------------------------------------------------------------- +// Test 1: Sequential claim safety (the primary acceptance test). +// --------------------------------------------------------------------- + +// TestClaimRange_SequentialSafety is THE primary acceptance test. Node 1 +// claims [1,100]. Node 2 then proposes the same starting base from a stale +// view (prevBase=0, which was already true before node 1's claim landed). +// Node 2's coordinator PREPAREs against all three participants, including +// its own node - every participant's prepareAutoIncClaim +// (db/replication_engine.go:310) evaluates storedBase(=100) <= prevBase(=0), +// which is false, so every participant, including node 2 itself, rejects. +// Node 2's own rejection makes this a LocalPrepareError +// (coordinator/errors.go:34), which coordinator/write_coordinator.go:869-875 +// treats as deterministic and returns immediately without needing quorum to +// fail first. ClaimRange (coordinator/autoinc_claim.go:62) then retries +// above the reported base (100) and succeeds, landing [101,150]. +// +// Mutation: a broken storedBase<=prevBase staleness check in +// prepareAutoIncClaim (db/replication_engine.go) would let node 2's stale +// round-1 proposal be accepted instead of rejected. The assertion this test +// pins that such a regression would fail is: node 2's round-1 PREPARE +// recorded on node 1 must be an explicit Rejected=true response carrying +// AutoIDStoredBase=100, and node 1's own range [1,100] must still read back +// as owned after node 2 commits. +func TestClaimRange_SequentialSafety(t *testing.T) { + c := setupClaimCluster(t, "testdb", "seqsafety") + + // Node 1 claims [1,100]. + newBase1, granted1, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "seqsafety", 0, 100) + require.NoError(t, err) + require.Equal(t, uint64(0), newBase1) + require.Equal(t, uint64(100), granted1) + for _, id := range []uint64{1, 2, 3} { + waitForBase(t, c.nodes[id], "testdb", "seqsafety", 100) + } + + // Node 2 proposes the SAME stale prevBase=0, unaware of node 1's claim. + mark1 := c.fanout.prepareCallCount(1) + mark3 := c.fanout.prepareCallCount(3) + newBase2, granted2, err := c.nodes[2].wc.ClaimRange(context.Background(), "testdb", "seqsafety", 0, 50) + require.NoError(t, err, "node 2's retry must still succeed after the stale round is rejected") + require.Equal(t, uint64(100), newBase2, "node 2 must retry starting at node 1's committed base, not spin on 0") + require.Equal(t, uint64(50), granted2) + + // Assert the FIRST round was actually rejected by the other participants, + // each carrying the real stored base (100), not merely superseded by a + // later successful round. + round1Node1 := c.fanout.prepareCallsSince(1, mark1) + require.NotEmpty(t, round1Node1) + require.False(t, round1Node1[0].resp.Success, "node 1 must have rejected node 2's stale round-1 proposal") + require.True(t, round1Node1[0].resp.Rejected) + require.Equal(t, uint64(100), round1Node1[0].resp.AutoIDStoredBase) + + round1Node3 := c.fanout.prepareCallsSince(3, mark3) + require.NotEmpty(t, round1Node3) + require.False(t, round1Node3[0].resp.Success, "node 3 must have rejected node 2's stale round-1 proposal") + require.True(t, round1Node3[0].resp.Rejected) + require.Equal(t, uint64(100), round1Node3[0].resp.AutoIDStoredBase) + + // Node 2's retry must be a SECOND round against these same participants. + require.Len(t, c.fanout.prepareCallsSince(1, mark1), 2, "node 2's claim must retry exactly once against node 1") + require.Len(t, c.fanout.prepareCallsSince(3, mark3), 2, "node 2's claim must retry exactly once against node 3") + + // The two granted ranges are disjoint - node 1's [1,100] was not silently + // overwritten by node 2's claim. + require.True(t, rangesDisjoint(newBase1, granted1, newBase2, granted2), + "node 1's range [%d,%d] and node 2's range [%d,%d] must be disjoint", + newBase1+1, newBase1+granted1, newBase2+1, newBase2+granted2) + + for _, id := range []uint64{1, 2, 3} { + waitForBase(t, c.nodes[id], "testdb", "seqsafety", 150) + } +} + +// --------------------------------------------------------------------- +// Test 2: Concurrent claims at a range boundary. +// --------------------------------------------------------------------- + +type claimOutcome struct { + claimant uint64 + newBase uint64 + granted uint64 + err error +} + +// maxLockWaitClientRetries bounds claimRangeRetryingOnLockWait's own retry +// loop, exactly as maxClaimAttempts bounds ClaimRange's internal one +// (coordinator/autoinc_claim.go:16) - a caller always gets a definitive +// answer instead of spinning against sustained contention. +const maxLockWaitClientRetries = 10 + +// claimRangeRetryingOnLockWait calls ClaimRange and, on the specific +// "lock wait timeout" signal (MySQL 1205, protocol.ErrCodeLockTimeout), +// retries the WHOLE call rather than treating it as final. +// +// This is necessary, and is not a workaround for a bug: when two +// claimants' PREPARE rounds genuinely overlap in time on the same node +// (true concurrency, not one committing before the other starts), the +// SECOND one to arrive does not see prepareAutoIncClaim's own +// storedBase-vs-prevBase staleness check (db/replication_engine.go:352-355) - +// storedBase is still whatever it was before either claim, since neither has +// committed yet. It instead hits the ordinary MVCC write-write-conflict +// check on the shared intent key (db/transaction.go's WriteIntent, the same +// mechanism that serialises two concurrent DML statements on one row), +// which carries no AutoIDStoredBase. ClaimRange's own retry loop +// (coordinator/autoinc_claim.go:102-105) only retries a failure that +// carries a usable base; a bare write-write conflict does not, so +// ClaimRange correctly ends its own attempt loop immediately and returns +// 1205 - "the standard signal this codebase already uses for retry me" +// (coordinator/autoinc_claim.go:53-54). Retrying the call is exactly what a +// real client (or the range allocator that will wrap ClaimRange) does on +// that signal; this helper stands in for that caller. +func claimRangeRetryingOnLockWait(ctx context.Context, node *inprocNode, database, table string, prevBase, size uint64) (newBase, granted uint64, err error) { + for attempt := 0; attempt < maxLockWaitClientRetries; attempt++ { + newBase, granted, err = node.wc.ClaimRange(ctx, database, table, prevBase, size) + if err == nil { + return newBase, granted, nil + } + var mysqlErr *protocol.MySQLError + if !errors.As(err, &mysqlErr) || mysqlErr.Code != protocol.ErrCodeLockTimeout { + return 0, 0, err + } + } + return 0, 0, err +} + +// TestClaimRange_ConcurrentBoundary runs node 1's and node 2's ClaimRange +// concurrently from the same stale prevBase=0 view, in two goroutines. +// Neither goroutine calls t.Fatal/require directly - both send their +// outcome over a channel, and every assertion runs back on the test +// goroutine. +// +// Each goroutine calls ClaimRange through claimRangeRetryingOnLockWait, not +// ClaimRange directly: when both PREPARE rounds genuinely overlap, the loser +// hits the ordinary MVCC write-write conflict on the shared intent key +// rather than prepareAutoIncClaim's own staleness check (neither side has +// committed yet, so there is no base to be stale against), and that failure +// carries no retry base for ClaimRange's own internal loop to use - it +// returns the retryable MySQL 1205 (lock wait timeout) instead. Both calls +// succeeding here is therefore not "neither call ever errors": one or both +// sides legitimately receive 1205 first and claimRangeRetryingOnLockWait +// retries the whole call past it (up to maxLockWaitClientRetries times). See +// the helper's doc for the full mechanism. The convergence check below also +// only requires a QUORUM of nodes (2 of 3), not all three: the participant +// that lost the lock race on an earlier attempt is left out of that +// attempt's COMMIT and can end up a genuine straggler on this key - see +// waitForQuorumBase's doc. +// +// Mutation: a broken storedBase<=prevBase staleness check in +// prepareAutoIncClaim (db/replication_engine.go) would let a claimant retry +// from an already-superseded prevBase=0 and succeed instead of being +// rejected - the rangesDisjoint assertion below is what catches the +// resulting overlap. +func TestClaimRange_ConcurrentBoundary(t *testing.T) { + c := setupClaimCluster(t, "testdb", "concurrent") + + const size1, size2 = 30, 20 + ch := make(chan claimOutcome, 2) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + nb, g, err := claimRangeRetryingOnLockWait(context.Background(), c.nodes[1], "testdb", "concurrent", 0, size1) + ch <- claimOutcome{claimant: 1, newBase: nb, granted: g, err: err} + }() + go func() { + defer wg.Done() + nb, g, err := claimRangeRetryingOnLockWait(context.Background(), c.nodes[2], "testdb", "concurrent", 0, size2) + ch <- claimOutcome{claimant: 2, newBase: nb, granted: g, err: err} + }() + wg.Wait() + close(ch) + + results := make(map[uint64]claimOutcome, 2) + for r := range ch { + results[r.claimant] = r + } + require.Len(t, results, 2) + + r1, r2 := results[1], results[2] + require.NoError(t, r1.err, "node 1's concurrent claim must succeed") + require.NoError(t, r2.err, "node 2's concurrent claim must succeed") + require.Equal(t, uint64(size1), r1.granted) + require.Equal(t, uint64(size2), r2.granted) + + require.True(t, rangesDisjoint(r1.newBase, r1.granted, r2.newBase, r2.granted), + "concurrent claims must land on disjoint ranges: node1=[%d,%d] node2=[%d,%d]", + r1.newBase+1, r1.newBase+r1.granted, r2.newBase+1, r2.newBase+r2.granted) + + // The final stored base must equal the starting base (0) plus BOTH sizes - + // one range stacked above the other, never one silently dropped - on at + // least a QUORUM of nodes (2 of 3). See waitForQuorumBase's doc for why + // this test checks quorum rather than every node: true concurrent + // contention can legitimately leave the loser of one round's write-write + // conflict as an uncaught-up straggler on this key, which is a property + // of the write-coordinator's quorum-commit design (no synchronous + // straggler catch-up), not a claim-protocol defect. + allNodes := []*inprocNode{c.nodes[1], c.nodes[2], c.nodes[3]} + waitForQuorumBase(t, allNodes, "testdb", "concurrent", size1+size2, 2) +} + +// --------------------------------------------------------------------- +// Test 3: Restart recovers the committed base. +// --------------------------------------------------------------------- + +// TestClaimRange_RestartRecoversCommittedBase claims a range, then closes +// node 3's DatabaseManager and reopens it from the SAME data directory +// (db.NewDatabaseManager, db/database_manager.go:54), rebuilding its +// LocalReplicator and rewiring the fanout. It asserts the reopened node's +// stored base is the committed one, and that a subsequent claim (issued by +// node 1, so it must go through node 3 as a remote participant, proving node +// 3 itself - not some cached view - is voting correctly) starts strictly +// ABOVE it rather than reissuing the committed range. +// +// This covers only the GRACEFUL half of "stop then restart": dm.Close() runs +// SQLite's and Pebble's normal shutdown path, which a real process kill +// skips entirely. TestClaimRange_KillNineRecoversCommittedBase (below) +// covers that other half with a real SIGKILL, sent to a re-exec'd child +// process - there is no way to signal only part of one Go process. +// +// Mutation: the same broken storedBase<=prevBase staleness check that +// TestClaimRange_SequentialSafety pins fires here too, but the interesting +// failure mode this test adds is a regression that read node 3's base from +// something other than its own freshly reopened SQLite file (e.g. a cached +// in-memory value surviving the swap) - the subsequent claim's round-1 +// rejection carrying the wrong base would expose that. +func TestClaimRange_RestartRecoversCommittedBase(t *testing.T) { + c := setupClaimCluster(t, "testdb", "restart") + + newBase, granted, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "restart", 0, 40) + require.NoError(t, err) + require.Equal(t, uint64(0), newBase) + require.Equal(t, uint64(40), granted) + for _, id := range []uint64{1, 2, 3} { + waitForBase(t, c.nodes[id], "testdb", "restart", 40) + } + + c.reopenNode(3) + + base, err := c.nodes[3].claimStore().ReadBase("testdb", "restart") + require.NoError(t, err) + require.Equal(t, uint64(40), base, "reopened node 3 must read back the committed base from its own SQLite file") + + // A subsequent claim from node 1, issued with a stale prevBase=0, must be + // rejected (including by node 3, over the rewired fanout) and retry above + // 40 - never reissue [1,40]. + mark3 := c.fanout.prepareCallCount(3) + newBase2, granted2, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "restart", 0, 25) + require.NoError(t, err) + require.Equal(t, uint64(40), newBase2, "subsequent claim must start above the base node 3 recovered on reopen") + require.Equal(t, uint64(25), granted2) + + round1Node3 := c.fanout.prepareCallsSince(3, mark3) + require.NotEmpty(t, round1Node3) + require.False(t, round1Node3[0].resp.Success) + require.True(t, round1Node3[0].resp.Rejected) + require.Equal(t, uint64(40), round1Node3[0].resp.AutoIDStoredBase, "reopened node 3 must vote with its real recovered base") + + for _, id := range []uint64{1, 2, 3} { + waitForBase(t, c.nodes[id], "testdb", "restart", 65) + } +} + +// --------------------------------------------------------------------- +// Test 3b: a real kill -9, via a re-exec'd child process. +// --------------------------------------------------------------------- + +const ( + // killNineDataRootEnv, when set in the environment, marks the current + // process as the child TestClaimRange_KillNineRecoversCommittedBase + // re-execs, and names the directory its three node data directories + // live under. + killNineDataRootEnv = "MARMOT_KILLNINE_DATA_ROOT" + killNineDatabase = "testdb" + killNineTable = "killnine" + killNineClaimSize = 20 + // killNineChildTimeout bounds every wait on the child process, so a + // child that never reports a claim - or never dies after SIGKILL - + // cannot hang the suite. + killNineChildTimeout = 45 * time.Second +) + +// killNineNodeDir is the data directory a node with the given ID uses under +// root, shared by the parent (which creates it) and the child (which opens +// it) so both agree on the layout without passing more than one env var. +func killNineNodeDir(root string, id uint64) string { + return filepath.Join(root, fmt.Sprintf("node%d", id)) +} + +// TestClaimRange_KillNineRecoversCommittedBase covers the half +// TestClaimRange_RestartRecoversCommittedBase cannot: recovery after a real +// SIGKILL, not a graceful dm.Close(). There is no way to signal only part of +// one Go process, so this re-execs the current test binary as a child +// (TestClaimRangeKillNineChild), which builds its own 3-node in-process +// cluster over data directories this parent creates, commits one claim, +// reports it on stdout, then blocks forever. The parent waits for that +// report, sends SIGKILL, confirms the child died by signal, then reopens all +// three nodes from the same directories and asserts the committed base and +// the claim protocol's staleness rejection both survived the kill. +func TestClaimRange_KillNineRecoversCommittedBase(t *testing.T) { + root := t.TempDir() + dataDirs := make(map[uint64]string, 3) + for _, id := range []uint64{1, 2, 3} { + dir := killNineNodeDir(root, id) + require.NoError(t, os.MkdirAll(dir, 0755)) + dataDirs[id] = dir + } + + cmd := exec.Command(os.Args[0], "-test.run=^TestClaimRangeKillNineChild$", "-test.v") + cmd.Env = append(os.Environ(), killNineDataRootEnv+"="+root) + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + var stderr bytes.Buffer + cmd.Stderr = &stderr + require.NoError(t, cmd.Start(), "start child process") + + claimLine := make(chan string, 1) + scanDone := make(chan error, 1) + go func() { + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "CLAIMED ") { + claimLine <- line + return + } + } + scanDone <- scanner.Err() + }() + + var line string + select { + case line = <-claimLine: + case scanErr := <-scanDone: + _ = cmd.Process.Kill() + t.Fatalf("child exited before reporting a claim: %v\nstderr:\n%s", scanErr, stderr.String()) + case <-time.After(killNineChildTimeout): + _ = cmd.Process.Kill() + t.Fatalf("timed out waiting for child to report a claim\nstderr:\n%s", stderr.String()) + } + + var newBase, granted uint64 + if _, err := fmt.Sscanf(line, "CLAIMED %d %d", &newBase, &granted); err != nil { + _ = cmd.Process.Kill() + t.Fatalf("unparseable child output line %q: %v", line, err) + } + require.Equal(t, uint64(0), newBase) + require.Equal(t, uint64(killNineClaimSize), granted) + + require.NoError(t, cmd.Process.Kill(), "SIGKILL child") + + waitDone := make(chan error, 1) + go func() { waitDone <- cmd.Wait() }() + select { + case waitErr := <-waitDone: + var exitErr *exec.ExitError + require.ErrorAs(t, waitErr, &exitErr, "child must exit with a non-nil error after SIGKILL") + status, ok := exitErr.Sys().(syscall.WaitStatus) + require.True(t, ok, "unexpected process exit status type %T", exitErr.Sys()) + require.True(t, status.Signaled(), "child must have died by signal, exit status: %v", status) + require.Equal(t, syscall.SIGKILL, status.Signal(), "child must have died specifically from SIGKILL") + case <-time.After(killNineChildTimeout): + t.Fatalf("timed out waiting for killed child to exit\nstderr:\n%s", stderr.String()) + } + + c := newInprocClusterFromDirs(t, dataDirs) + for _, id := range []uint64{1, 2, 3} { + base, err := c.nodes[id].claimStore().ReadBase(killNineDatabase, killNineTable) + require.NoError(t, err) + require.Equal(t, newBase+granted, base, "node %d must recover the committed base after a real SIGKILL", id) + } + + // A stale claim (prevBase=0) must be rejected and retried above the + // recovered base, not reissue [1,20]. + newBase2, granted2, err := c.nodes[1].wc.ClaimRange(context.Background(), killNineDatabase, killNineTable, 0, 15) + require.NoError(t, err) + require.Equal(t, newBase+granted, newBase2, "post-kill claim must start above the recovered base, not reissue the same range") + require.Equal(t, uint64(15), granted2) +} + +// TestClaimRangeKillNineChild is the child TestClaimRange_KillNineRecoversCommittedBase +// re-execs to receive a real SIGKILL. It never participates in a normal test +// run: it skips unless killNineDataRootEnv is set, which only that parent +// test sets, on the child it starts. +func TestClaimRangeKillNineChild(t *testing.T) { + root := os.Getenv(killNineDataRootEnv) + if root == "" { + t.Skip("only runs as the re-exec'd child of TestClaimRange_KillNineRecoversCommittedBase") + } + + dataDirs := make(map[uint64]string, 3) + for _, id := range []uint64{1, 2, 3} { + dataDirs[id] = killNineNodeDir(root, id) + } + c := newInprocClusterFromDirs(t, dataDirs) + for _, id := range []uint64{1, 2, 3} { + require.NoError(t, c.nodes[id].dm.CreateDatabase(killNineDatabase)) + } + driveDDL(t, c.nodes[1], killNineDatabase, killNineTable, sprintfDDL(killNineTable)) + for _, id := range []uint64{1, 2, 3} { + waitForBase(t, c.nodes[id], killNineDatabase, killNineTable, 0) + } + + newBase, granted, err := c.nodes[1].wc.ClaimRange(context.Background(), killNineDatabase, killNineTable, 0, killNineClaimSize) + require.NoError(t, err) + for _, id := range []uint64{1, 2, 3} { + waitForBase(t, c.nodes[id], killNineDatabase, killNineTable, newBase+granted) + } + + // The parent watches stdout for exactly this line, then sends SIGKILL - + // so nothing after it may run any cleanup a graceful shutdown would. + fmt.Printf("CLAIMED %d %d\n", newBase, granted) + select {} +} + +// --------------------------------------------------------------------- +// Test 4: Meta-store wipe. +// --------------------------------------------------------------------- + +// metaPebbleDirFor returns the Pebble meta directory path for a database +// file, using the SAME derivation db.DatabaseManager itself uses: +// strings.TrimSuffix(, ".db") + "_meta.pebble" +// (db/database_manager.go:431, and identically at :851 for cleanupMetaStoreFiles). +func metaPebbleDirFor(dbFilePath string) string { + return strings.TrimSuffix(dbFilePath, ".db") + "_meta.pebble" +} + +// TestClaimRange_SurvivesMetaStoreWipe claims a range, then deletes ONLY +// node 3's Pebble meta directories - both the system database's +// (dataDir/__marmot_system_meta.pebble, derived from the system db path +// db/database_manager.go:90) and the user database's (derived from +// dm.GetDatabasePath, db/database_manager.go:649, before closing) - while +// leaving every .db file untouched, then reopens that node. Because the +// committed AUTO_INCREMENT base lives in a SQLite table in the system +// database (db/autoinc_claim.go:19-34's doc: "deliberately does NOT live in +// Pebble"), wiping Pebble state must not make node 3 forget it: it must +// still vote with its REAL base, rejecting a stale proposal. +// +// Mutation: a broken storedBase<=prevBase staleness check in +// prepareAutoIncClaim (db/replication_engine.go) would let node 2's stale +// round-1 proposal be accepted by node 3 instead of rejected - the +// assertion that node 3's round-1 PREPARE response is Rejected=true with +// AutoIDStoredBase=10 is what catches that, proving node 3 votes with its +// real base recovered from SQLite, not from the wiped Pebble state. +func TestClaimRange_SurvivesMetaStoreWipe(t *testing.T) { + c := setupClaimCluster(t, "testdb", "metawipe") + + newBase, granted, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "metawipe", 0, 10) + require.NoError(t, err) + require.Equal(t, uint64(0), newBase) + require.Equal(t, uint64(10), granted) + for _, id := range []uint64{1, 2, 3} { + waitForBase(t, c.nodes[id], "testdb", "metawipe", 10) + } + + node3 := c.nodes[3] + systemDBPath := filepath.Join(node3.dataDir, db.SystemDatabaseName+".db") + userDBPath, err := node3.dm.GetDatabasePath("testdb") + require.NoError(t, err) + systemMetaDir := metaPebbleDirFor(systemDBPath) + userMetaDir := metaPebbleDirFor(userDBPath) + + c.closeNode(3) + require.NoError(t, os.RemoveAll(systemMetaDir)) + require.NoError(t, os.RemoveAll(userMetaDir)) + // The .db files themselves must survive untouched. + require.FileExists(t, systemDBPath) + require.FileExists(t, userDBPath) + c.reopenNodeFresh(3) + // node3 was replaced in the cluster map by reopenNodeFresh; re-fetch it. + node3 = c.nodes[3] + + base, err := node3.claimStore().ReadBase("testdb", "metawipe") + require.NoError(t, err) + require.Equal(t, uint64(10), base, "node 3 must still read its real committed base after its Pebble meta store was wiped") + + // A stale proposal from node 2 must be rejected BY NODE 3 specifically, + // carrying its real base. + mark3 := c.fanout.prepareCallCount(3) + newBase2, granted2, err := c.nodes[2].wc.ClaimRange(context.Background(), "testdb", "metawipe", 0, 5) + require.NoError(t, err) + require.Equal(t, uint64(10), newBase2) + require.Equal(t, uint64(5), granted2) + + round1Node3 := c.fanout.prepareCallsSince(3, mark3) + require.NotEmpty(t, round1Node3) + require.False(t, round1Node3[0].resp.Success, "node 3 must reject the stale round-1 proposal after its meta store was wiped") + require.True(t, round1Node3[0].resp.Rejected) + require.Equal(t, uint64(10), round1Node3[0].resp.AutoIDStoredBase) + + for _, id := range []uint64{1, 2, 3} { + waitForBase(t, c.nodes[id], "testdb", "metawipe", 15) + } +} + +// --------------------------------------------------------------------- +// Test 5: Full-cluster snapshot restore. +// --------------------------------------------------------------------- + +// TestClaimRange_SurvivesFullClusterSnapshotRestore snapshots all three +// nodes (db.DatabaseManager.TakeSnapshotToDir, db/database_manager.go:989), +// stops all three, wipes their data directories entirely (including Pebble +// meta state), restores each node's files from its own snapshot, and +// restarts all three. It asserts the committed base survives on every node +// and the next claim starts above it - proving the base's deliberate +// placement in SQLite rather than Pebble (db/autoinc_claim.go:33's doc: "a +// restored node has no Pebble state, so a full-cluster restore would make +// every voter a yes-man at once") actually holds when every node is restored +// at once, not just one. +// +// Mutation: a broken storedBase<=prevBase staleness check in +// prepareAutoIncClaim (db/replication_engine.go) would let the post-restore +// stale claim (prevBase=0) reissue [1,20] instead of being rejected and +// retried above the restored base - the assertion that the retried claim +// starts at 20, not 0, is what catches that, on every node at once since a +// full-cluster restore leaves no node with pre-restore state to fall back +// on. +func TestClaimRange_SurvivesFullClusterSnapshotRestore(t *testing.T) { + c := setupClaimCluster(t, "testdb", "snaprestore") + + newBase, granted, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "snaprestore", 0, 20) + require.NoError(t, err) + require.Equal(t, uint64(0), newBase) + require.Equal(t, uint64(20), granted) + for _, id := range []uint64{1, 2, 3} { + waitForBase(t, c.nodes[id], "testdb", "snaprestore", 20) + } + + snapDirByNode := make(map[uint64]string, 3) + snapshotsByNode := make(map[uint64][]db.SnapshotInfo, 3) + + // Snapshot every node BEFORE any of them stop. + for _, id := range []uint64{1, 2, 3} { + snapDir := t.TempDir() + snapshots, _, _, err := c.nodes[id].dm.TakeSnapshotToDir(snapDir) + require.NoError(t, err) + require.NotEmpty(t, snapshots) + snapDirByNode[id] = snapDir + snapshotsByNode[id] = snapshots + } + + // Stop all three, then wipe each node's data directory entirely. + for _, id := range []uint64{1, 2, 3} { + c.closeNode(id) + } + for _, id := range []uint64{1, 2, 3} { + dataDir := c.nodes[id].dataDir + require.NoError(t, os.RemoveAll(dataDir)) + require.NoError(t, os.MkdirAll(dataDir, 0755)) + } + + // Restore each node's files into its fresh, empty directory through the + // SAME production restorer a real node uses to apply a snapshot + // (db/snapshot/restorer.go), including its system-database merge hook - + // not a fixture-only file copy. The manager is fully closed at this + // point, not live, exactly like a node coming up against a freshly + // provisioned data directory, so no ConnectionManager is needed. + for _, id := range []uint64{1, 2, 3} { + dataDir := c.nodes[id].dataDir + files := make([]snapshot.DatabaseFileInfo, 0, len(snapshotsByNode[id])) + for _, info := range snapshotsByNode[id] { + files = append(files, snapshot.DatabaseFileInfo{ + Name: info.Name, + Filename: info.Filename, + SizeBytes: info.Size, + SHA256Checksum: info.SHA256, + }) + } + r := snapshot.NewRestorer(dataDir, nil) + r.SetSystemDBMerge(db.RaiseAutoIncBasesFrom) + require.NoError(t, r.RestoreFiles(snapDirByNode[id], files)) + } + + // Restart all three. + for _, id := range []uint64{1, 2, 3} { + c.reopenNodeFresh(id) + } + + for _, id := range []uint64{1, 2, 3} { + base, err := c.nodes[id].claimStore().ReadBase("testdb", "snaprestore") + require.NoError(t, err) + require.Equal(t, uint64(20), base, "node %d must recover its committed base from the restored snapshot", id) + } + + // A stale claim, from node 1, must retry above the restored base rather + // than reissue [1,20]. + newBase2, granted2, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "snaprestore", 0, 15) + require.NoError(t, err) + require.Equal(t, uint64(20), newBase2, "post-restore claim must start above the restored base, not reissue [1,20]") + require.Equal(t, uint64(15), granted2) + + for _, id := range []uint64{1, 2, 3} { + waitForBase(t, c.nodes[id], "testdb", "snaprestore", 35) + } +} diff --git a/test/cluster_integration_test.go b/test/cluster_integration_test.go index d5441f79..4987d69b 100644 --- a/test/cluster_integration_test.go +++ b/test/cluster_integration_test.go @@ -84,7 +84,7 @@ func TestClusterReplication(t *testing.T) { nodeID := uint64(i + 1) node := &testNode{ nodeID: nodeID, - dataDir: filepath.Join(os.TempDir(), fmt.Sprintf("marmot-test-node-%d-%d", nodeID, time.Now().UnixNano())), + dataDir: filepath.Join(testDataRoot, fmt.Sprintf("marmot-test-node-%d-%d", nodeID, time.Now().UnixNano())), grpcPort: 18081 + i, mysqlPort: 13307 + i, } @@ -265,7 +265,7 @@ func TestClusterLoadDataLocalReplication(t *testing.T) { nodeID := uint64(i + 1) node := &testNode{ nodeID: nodeID, - dataDir: filepath.Join(os.TempDir(), fmt.Sprintf("marmot-load-test-node-%d-%d", nodeID, time.Now().UnixNano())), + dataDir: filepath.Join(testDataRoot, fmt.Sprintf("marmot-load-test-node-%d-%d", nodeID, time.Now().UnixNano())), grpcPort: 18181 + i, mysqlPort: 13407 + i, } @@ -422,6 +422,7 @@ func startNode(t *testing.T, node *testNode, seedNodes []string) { grpcServer.GetNodeRegistry(), seedNodes, ) + node.catchUpClient.SetDatabaseManager(dbMgr) node.deltaSync = marmotgrpc.NewDeltaSyncClient(marmotgrpc.DeltaSyncConfig{ NodeID: node.nodeID, @@ -433,13 +434,7 @@ func startNode(t *testing.T, node *testNode, seedNodes []string) { }) snapshotFunc := func(ctx context.Context, peerNodeID uint64, peerAddr string, database string) error { - if err := node.catchUpClient.CatchUpFromPeer(ctx, peerNodeID, peerAddr, database); err != nil { - return err - } - if err := dbMgr.ReopenDatabase(database); err != nil { - return fmt.Errorf("database reload failed after snapshot: %w", err) - } - return nil + return node.catchUpClient.CatchUpFromPeer(ctx, peerNodeID, peerAddr, database) } schemaVersionMgr = db.NewSchemaVersionManager(systemDB.GetMetaStore()) diff --git a/test/crash_recovery_test.go b/test/crash_recovery_test.go index e0a5b3c5..d8d0bd89 100644 --- a/test/crash_recovery_test.go +++ b/test/crash_recovery_test.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -106,7 +107,7 @@ type ClusterHarness struct { func NewClusterHarness(t *testing.T) *ClusterHarness { cleanupStalePorts() - baseDir := filepath.Join(os.TempDir(), fmt.Sprintf("marmot_crash_test_%d", time.Now().UnixNano())) + baseDir := filepath.Join(testDataRoot, fmt.Sprintf("marmot_crash_test_%d", time.Now().UnixNano())) marmotBin, err := getCrashHarnessBinary(t) if err != nil { t.Fatalf("Failed to build Marmot: %v", err) @@ -127,13 +128,32 @@ func NewClusterHarness(t *testing.T) *ClusterHarness { return harness } +// testDataRoot holds the integration tests' node data and binaries: Marmot's +// runtime files live under /tmp/marmot. +const testDataRoot = "/tmp/marmot/test" + +// testDataDir creates a fresh directory under testDataRoot. +func testDataDir(pattern string) (string, error) { + if err := os.MkdirAll(testDataRoot, 0o755); err != nil { + return "", err + } + return os.MkdirTemp(testDataRoot, pattern) +} + +// repoRoot is the module root: the directory above this test package, taken +// from this file's own path so the harness builds the tree it is compiled from. +func repoRoot() string { + _, file, _, _ := runtime.Caller(0) + return filepath.Dir(filepath.Dir(file)) +} + func getCrashHarnessBinary(t *testing.T) (string, error) { t.Helper() crashHarnessBuildOnce.Do(func() { t.Logf("Building shared Marmot binary for crash-recovery tests...") - buildDir, err := os.MkdirTemp("", "marmot_crash_bin_") + buildDir, err := testDataDir("marmot_crash_bin_") if err != nil { crashHarnessBuildErr = fmt.Errorf("failed to create build dir: %w", err) return @@ -141,7 +161,7 @@ func getCrashHarnessBinary(t *testing.T) (string, error) { binPath := filepath.Join(buildDir, "marmot") cmd := exec.Command("go", "build", "-tags", "sqlite_preupdate_hook sqlite_fts5 sqlite_json sqlite_math_functions sqlite_foreign_keys sqlite_stat4 sqlite_vacuum_incr", "-o", binPath, ".") - cmd.Dir = "/Users/zohaib/repos/marmot" + cmd.Dir = repoRoot() output, err := cmd.CombinedOutput() if err != nil { crashHarnessBuildErr = fmt.Errorf("build failed: %v\n%s", err, output) diff --git a/test/ddl_replication_test.go b/test/ddl_replication_test.go index 80529203..a9e13dcc 100644 --- a/test/ddl_replication_test.go +++ b/test/ddl_replication_test.go @@ -314,7 +314,7 @@ func TestDDLIdempotencyRewriter(t *testing.T) { // TestSchemaVersionManager validates schema version tracking per database func TestSchemaVersionManager(t *testing.T) { // Create temp directory for PebbleDB - tmpDir, err := os.MkdirTemp("", "schema-version-test-*") + tmpDir, err := testDataDir("schema-version-test-*") require.NoError(t, err) defer os.RemoveAll(tmpDir) @@ -475,7 +475,7 @@ func TestDDLReplicationBasic(t *testing.T) { dbMgr := NewDDLMockDatabaseManager() // Create temp directory for PebbleDB - tmpDir, err := os.MkdirTemp("", "ddl-replication-test-*") + tmpDir, err := testDataDir("ddl-replication-test-*") require.NoError(t, err) defer os.RemoveAll(tmpDir) @@ -572,7 +572,7 @@ func TestDDLWithConcurrentDML(t *testing.T) { dbMgr := NewDDLMockDatabaseManager() // Create temp directory for PebbleDB - tmpDir, err := os.MkdirTemp("", "ddl-concurrent-dml-test-*") + tmpDir, err := testDataDir("ddl-concurrent-dml-test-*") require.NoError(t, err) defer os.RemoveAll(tmpDir) From 7a665967b57081602f223f730444b9088a590135 Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Sat, 26 Sep 2026 10:45:59 -0500 Subject: [PATCH 05/17] feat(autoinc): mint narrow AUTO_INCREMENT ids from cluster-wide range claims Step 3c of the narrow AUTO_INCREMENT id design (issue #170). Inserts into an INT (or narrower) AUTO_INCREMENT column now take their ids from ranges the node claimed through a quorum (Step 3b), so ids fit the declared width and no two nodes hand out the same id while the membership is stable. Insert path. A per-(db, table) range allocator hands out ids from the node's claimed range and claims a new one when it is exhausted. An explicit id above the cluster base raises the base; one at or below it is refused with 1235 unless it falls in this node's own claimed range. Exhaustion returns 1062. A bound NULL or 0 is generated. Qualified table names are handled, prepared statements keep the client SQL, and a read-only replica forwards the client SQL rather than the transpiled one. Claim floors. Each claim row keeps three floors: committed (written only when this node applies a claim), seed (local data: DDL seeding, restore, rename) and merged (bases learned from peers). PREPARE votes against their max; COMMIT applies only if committed and seed are at or below the claim's start. Disjointness of the ranges a node ACKs rests on committed alone, so a base learned from a peer between PREPARE and COMMIT never refuses the COMMIT, while a DDL seed in that window still does. Existing system databases are migrated on open. A refused claim apply is reported as a typed error, not as node divergence. Membership. A node whose system database is new or lost holds its vote until it has merged bases from enough peers; cluster.standalone skips the hold for a single node. Every unheld node pulls the maximum bases from all alive peers every 10s, and an authenticated admin sync command does the same on demand and reports whether each node reached every member. The guarantee covers a stable membership with this procedure: change one node at a time, wait until it is ALIVE and unheld, run the sync until it reports complete, then make the next change. Claims carry the schema version they need, and every DDL path forgets cached cursors. Also fixed: - The schema version was written without sync, so a SIGKILL could lose it and leave the node unable to vote after restart. - INFORMATION_SCHEMA queries read through the read pool, so they no longer deadlock the write connection, and prepared IS queries match text ones. - A gossip reconnect now fires node-alive handling, so a node held while its seed was down releases when the seed returns. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- admin/autoinc.go | 93 ++++ admin/autoinc_test.go | 95 ++++ admin/middleware.go | 19 +- admin/routes.go | 5 + cfg/config.go | 20 +- cfg/config_test.go | 52 ++ common/internal_tables.go | 5 + config.toml.example | 28 +- coordinator/autoinc_claim.go | 98 +++- coordinator/autoinc_claim_bench_test.go | 2 +- coordinator/autoinc_claim_test.go | 109 +++- coordinator/commit_lost_pinned_state_test.go | 5 + coordinator/describe_columns.go | 12 +- coordinator/handler.go | 146 ++++- coordinator/narrow_autoinc_admit.go | 103 ++++ coordinator/narrow_autoinc_admit_test.go | 257 +++++++++ coordinator/narrow_autoinc_bench_test.go | 33 ++ coordinator/narrow_autoinc_insert_test.go | 163 ++++++ coordinator/noop_dml_test.go | 36 +- coordinator/write_coordinator.go | 37 +- .../write_coordinator_commitlocal_test.go | 50 ++ db/autoinc_claim.go | 369 +++++++++++-- db/autoinc_claim_test.go | 48 +- db/autoinc_commit_test.go | 101 ++++ db/autoinc_floors_test.go | 208 ++++++++ db/autoinc_prepare_test.go | 138 ++++- db/autoinc_restore_merge_test.go | 179 ++++++- db/autoinc_seed.go | 128 ++++- db/autoinc_seed_test.go | 42 +- db/autoinc_vote_hold.go | 233 ++++++++ db/autoinc_vote_hold_test.go | 159 ++++++ db/database_manager.go | 111 +++- db/ddl_apply.go | 18 +- db/ddl_prepare_validate.go | 41 +- db/ddl_prepare_validate_autoinc_code_test.go | 12 +- db/ddl_prepare_validate_test.go | 23 + db/meta_store_pebble.go | 20 +- db/prepared_intent_test.go | 10 +- db/replication_engine.go | 122 ++++- db/replication_engine_test.go | 4 + db/schema_change.go | 115 ++++ db/schema_change_test.go | 132 +++++ db/transaction.go | 60 ++- examples/run-single-node.sh | 1 + examples/wordpress/marmot-config.toml | 2 + grpc/autoinc_claim_handler_test.go | 12 +- grpc/autoinc_claim_schema_gate_test.go | 190 +++++++ grpc/autoinc_merge.go | 436 +++++++++++++++ grpc/autoinc_merge_test.go | 370 +++++++++++++ grpc/delta_sync.go | 11 +- grpc/marmot.pb.go | 497 ++++++++++++++---- grpc/marmot.proto | 41 ++ grpc/marmot_grpc.pb.go | 88 +++- grpc/node_registry.go | 44 +- grpc/node_registry_test.go | 38 ++ grpc/replication_handler.go | 16 +- grpc/replication_handler_test.go | 79 +++ grpc/replicator_adapter.go | 30 +- id/generator.go | 31 +- id/generator_test.go | 111 +++- id/range_allocator.go | 269 ++++++++++ id/range_allocator_test.go | 439 ++++++++++++++++ marmot.go | 20 + protocol/autoinc_claim.go | 27 +- protocol/autoinc_claim_test.go | 38 +- protocol/handlers/information_schema.go | 466 ++++++---------- protocol/handlers/information_schema_test.go | 208 ++++++++ protocol/handlers/metadata.go | 215 ++++---- protocol/handlers/metadata_marker_test.go | 2 +- protocol/handlers/metadata_readpool_test.go | 67 +++ protocol/information_schema.go | 98 ++++ protocol/information_schema_test.go | 191 +++++++ protocol/load_data_local.go | 57 +- protocol/load_data_local_test.go | 29 +- protocol/merge_exec_params_test.go | 21 + protocol/parser.go | 20 +- protocol/query/context.go | 18 +- protocol/query/parser.go | 85 ++- protocol/query/parser_test.go | 2 +- protocol/query/pipeline_autoincrement_test.go | 11 +- protocol/query/rules/autoincrement_id.go | 361 +++++++++++-- .../rules/autoincrement_id_narrow_test.go | 299 +++++++++++ protocol/query/rules/autoincrement_id_test.go | 33 +- protocol/query/transform/literal_extractor.go | 15 + protocol/query/transform/rule.go | 7 + protocol/query/transpiler.go | 8 +- protocol/server.go | 49 +- protocol/transaction.go | 19 +- replica/database_discovery_test.go | 16 + replica/handler.go | 12 +- replica/handler_test.go | 53 +- replica/replica.go | 2 +- test/autoinc_backstop_cluster_test.go | 128 +++++ test/autoinc_claim_cluster_test.go | 110 ++-- test/autoinc_claim_inprocess_test.go | 164 ++++-- test/autoinc_lifecycle_cluster_test.go | 344 ++++++++++++ test/autoinc_membership_cluster_test.go | 436 +++++++++++++++ test/autoinc_membership_inprocess_test.go | 107 ++++ test/autoinc_narrow_cluster_test.go | 457 ++++++++++++++++ test/cluster_integration_test.go | 3 + test/crash_recovery_test.go | 2 +- test/information_schema_cluster_test.go | 181 +++++++ test/schema_version_kill_cluster_test.go | 36 ++ test/verification_test.go | 6 + 104 files changed, 9630 insertions(+), 1139 deletions(-) create mode 100644 admin/autoinc.go create mode 100644 admin/autoinc_test.go create mode 100644 coordinator/narrow_autoinc_admit.go create mode 100644 coordinator/narrow_autoinc_admit_test.go create mode 100644 coordinator/narrow_autoinc_bench_test.go create mode 100644 coordinator/narrow_autoinc_insert_test.go create mode 100644 db/autoinc_floors_test.go create mode 100644 db/autoinc_vote_hold.go create mode 100644 db/autoinc_vote_hold_test.go create mode 100644 db/schema_change.go create mode 100644 db/schema_change_test.go create mode 100644 grpc/autoinc_claim_schema_gate_test.go create mode 100644 grpc/autoinc_merge.go create mode 100644 grpc/autoinc_merge_test.go create mode 100644 id/range_allocator.go create mode 100644 id/range_allocator_test.go create mode 100644 protocol/handlers/information_schema_test.go create mode 100644 protocol/handlers/metadata_readpool_test.go create mode 100644 protocol/information_schema.go create mode 100644 protocol/information_schema_test.go create mode 100644 protocol/query/rules/autoincrement_id_narrow_test.go create mode 100644 test/autoinc_backstop_cluster_test.go create mode 100644 test/autoinc_lifecycle_cluster_test.go create mode 100644 test/autoinc_membership_cluster_test.go create mode 100644 test/autoinc_membership_inprocess_test.go create mode 100644 test/autoinc_narrow_cluster_test.go create mode 100644 test/information_schema_cluster_test.go create mode 100644 test/schema_version_kill_cluster_test.go diff --git a/admin/autoinc.go b/admin/autoinc.go new file mode 100644 index 00000000..5b03473a --- /dev/null +++ b/admin/autoinc.go @@ -0,0 +1,93 @@ +package admin + +import ( + "net/http" + + marmotgrpc "github.com/maxpert/marmot/grpc" +) + +// handleAutoIncReleaseVotes handles POST +// /admin/cluster/autoinc/release-votes?accept_risk=duplicate-ids. +// +// It releases this node's held AUTO_INCREMENT claim votes after merging the +// bases of every peer that answers, without the safety condition the +// automatic merge waits for (grpc.ForceReleaseAutoIncVotes). It is for a +// cluster held beyond its failure model, and it can make the cluster issue +// the same id twice if a member holding a newer claim does not answer, so the +// caller must name that risk in accept_risk. +func (h *AdminHandlers) handleAutoIncReleaseVotes(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("accept_risk") != marmotgrpc.ForceReleaseRiskToken { + writeErrorResponse(w, http.StatusBadRequest, + "releasing held AUTO_INCREMENT claim votes without a safe merge can issue the same id twice "+ + "if a member holding a newer claim does not answer; pass accept_risk="+marmotgrpc.ForceReleaseRiskToken) + return + } + registry := h.resolveRegistryOrError(w) + if registry == nil { + return + } + client := h.server.GetGossipProtocol().GetClient() + if client == nil { + writeErrorResponse(w, http.StatusServiceUnavailable, "cluster client not started") + return + } + + answered, wasHeld, err := marmotgrpc.ForceReleaseAutoIncVotes(r.Context(), registry.GetLocalNodeID(), h.dbManager, registry, client) + if err != nil { + writeErrorResponse(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSONResponse(w, map[string]interface{}{ + "was_held": wasHeld, + "released": wasHeld, + "peers_answered": answered, + }, false, "") +} + +// handleAutoIncSync handles POST /admin/cluster/autoinc/sync. +// +// It runs the AUTO_INCREMENT claim base sync now on this node and on every +// member of its membership (grpc.SyncAutoIncBasesEverywhere), and reports +// per node whether it answered, whether its votes are held and whether it +// reached every member. It is the confirmation step of the membership-change +// procedure: add or remove one node, wait until it is ALIVE and unheld, run +// this - re-run until the report is complete - and make the next change only +// when "complete" is true. +func (h *AdminHandlers) handleAutoIncSync(w http.ResponseWriter, r *http.Request) { + registry := h.resolveRegistryOrError(w) + if registry == nil { + return + } + client := h.server.GetGossipProtocol().GetClient() + if client == nil { + writeErrorResponse(w, http.StatusServiceUnavailable, "cluster client not started") + return + } + outcomes := marmotgrpc.SyncAutoIncBasesEverywhere(r.Context(), registry.GetLocalNodeID(), h.dbManager, registry, client) + writeJSONResponse(w, autoIncSyncBody(outcomes), false, "") +} + +// autoIncSyncBody is the sync command's answer: one entry per member, and +// complete when every member answered, is unheld and reached every member. +func autoIncSyncBody(outcomes []marmotgrpc.AutoIncSyncOutcome) map[string]interface{} { + complete := len(outcomes) > 0 + nodes := make([]map[string]interface{}, 0, len(outcomes)) + for _, o := range outcomes { + node := map[string]interface{}{ + "node_id": o.NodeID, + "answered": o.Err == nil, + "complete": o.Complete(), + } + if o.Err != nil { + node["error"] = o.Err.Error() + } else { + node["votes_held"] = o.Report.VotesHeld + node["members"] = o.Report.Members + node["reached"] = o.Report.Reached + node["reached_every_member"] = o.Report.ReachedEveryMember() + } + complete = complete && o.Complete() + nodes = append(nodes, node) + } + return map[string]interface{}{"complete": complete, "nodes": nodes} +} diff --git a/admin/autoinc_test.go b/admin/autoinc_test.go new file mode 100644 index 00000000..059464d8 --- /dev/null +++ b/admin/autoinc_test.go @@ -0,0 +1,95 @@ +package admin + +import ( + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/maxpert/marmot/cfg" + marmotgrpc "github.com/maxpert/marmot/grpc" +) + +// TestAutoIncReleaseVotesRequiresTheRiskToken pins that the forced release is +// never reached by accident: without accept_risk naming the risk, or with any +// other value, it is refused before anything is touched, and the refusal +// states the risk. +// +// Mutation: drop the accept_risk check. The handler reaches the nil server +// and panics instead of answering 400. +func TestAutoIncReleaseVotesRequiresTheRiskToken(t *testing.T) { + h := &AdminHandlers{} + for _, query := range []string{"", "?accept_risk=yes"} { + rec := httptest.NewRecorder() + h.handleAutoIncReleaseVotes(rec, httptest.NewRequest(http.MethodPost, "/cluster/autoinc/release-votes"+query, nil)) + if rec.Code != http.StatusBadRequest { + t.Fatalf("%q: status %d, want 400", query, rec.Code) + } + if body := rec.Body.String(); !strings.Contains(body, "same id twice") || !strings.Contains(body, marmotgrpc.ForceReleaseRiskToken) { + t.Fatalf("%q: the refusal does not state the risk and the token: %s", query, body) + } + } +} + +// TestAutoIncEndpointsRequireTheClusterSecret pins R3c-15: the release and +// sync endpoints are refused on a cluster with no secret, and served only +// with the right one on a cluster that has one - unlike the other admin +// endpoints, which AuthMiddleware serves to anyone when no secret is set. +// +// Mutation: route the autoinc endpoints through chiAuthMiddleware only. The +// no-secret request reaches the handler and "served without a cluster +// secret" fires. +func TestAutoIncEndpointsRequireTheClusterSecret(t *testing.T) { + saved := cfg.Config.Cluster.ClusterSecret + t.Cleanup(func() { cfg.Config.Cluster.ClusterSecret = saved }) + + mux := http.NewServeMux() + RegisterRoutes(mux, &AdminHandlers{}) + do := func(path, secret string) int { + req := httptest.NewRequest(http.MethodPost, "/admin"+path, nil) + if secret != "" { + req.Header.Set("X-Marmot-Secret", secret) + } + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + return rec.Code + } + + for _, path := range []string{"/cluster/autoinc/sync", "/cluster/autoinc/release-votes?accept_risk=" + marmotgrpc.ForceReleaseRiskToken} { + cfg.Config.Cluster.ClusterSecret = "" + if code := do(path, ""); code != http.StatusForbidden { + t.Fatalf("%s: status %d, want 403: served without a cluster secret", path, code) + } + cfg.Config.Cluster.ClusterSecret = "s3cret" + if code := do(path, ""); code != http.StatusUnauthorized { + t.Fatalf("%s: status %d without the secret, want 401", path, code) + } + if code := do(path, "wrong"); code != http.StatusUnauthorized { + t.Fatalf("%s: status %d with a wrong secret, want 401", path, code) + } + } +} + +// TestAutoIncSyncBodyIsCompleteOnlyWhenEveryMemberIs pins the sync command's +// verdict: complete only when every member answered, is unheld and reached +// every member. +func TestAutoIncSyncBodyIsCompleteOnlyWhenEveryMemberIs(t *testing.T) { + ready := marmotgrpc.AutoIncSyncOutcome{NodeID: 1, Report: marmotgrpc.AutoIncSyncReport{ + NodeID: 1, Members: []uint64{1, 2}, Reached: []uint64{1, 2}}} + if body := autoIncSyncBody([]marmotgrpc.AutoIncSyncOutcome{ready, ready}); body["complete"] != true { + t.Fatalf("every member ready: %v", body) + } + down := marmotgrpc.AutoIncSyncOutcome{NodeID: 2, Err: errors.New("unreachable")} + body := autoIncSyncBody([]marmotgrpc.AutoIncSyncOutcome{ready, down}) + if body["complete"] != false { + t.Fatalf("an unreachable member reported complete: %v", body) + } + nodes := body["nodes"].([]map[string]interface{}) + if nodes[1]["answered"] != false || nodes[1]["error"] != "unreachable" { + t.Fatalf("the unreachable member's entry: %v", nodes[1]) + } + if autoIncSyncBody(nil)["complete"] != false { + t.Fatal("no outcome at all reported complete") + } +} diff --git a/admin/middleware.go b/admin/middleware.go index 27e7e8db..0cb0d0b4 100644 --- a/admin/middleware.go +++ b/admin/middleware.go @@ -1,6 +1,7 @@ package admin import ( + "crypto/subtle" "net/http" "strings" @@ -36,7 +37,7 @@ func AuthMiddleware(next http.Handler) http.Handler { providedSecret = parts[1] } - if providedSecret != secret { + if subtle.ConstantTimeCompare([]byte(providedSecret), []byte(secret)) != 1 { writeErrorResponse(w, http.StatusUnauthorized, "invalid secret") return } @@ -45,3 +46,19 @@ func AuthMiddleware(next http.Handler) http.Handler { next.ServeHTTP(w, r) }) } + +// RequireSecretMiddleware serves an endpoint only to a caller holding the +// cluster secret, and refuses it outright on a cluster with none. It guards +// the endpoints that change which AUTO_INCREMENT ids the cluster can issue, +// which AuthMiddleware alone would serve to anyone when no secret is set. +func RequireSecretMiddleware(next http.Handler) http.Handler { + authenticated := AuthMiddleware(next) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !cfg.IsClusterAuthEnabled() { + writeErrorResponse(w, http.StatusForbidden, + "this endpoint is never served unauthenticated: set cluster.cluster_secret to use it") + return + } + authenticated.ServeHTTP(w, r) + }) +} diff --git a/admin/routes.go b/admin/routes.go index 9c4ecbf4..2f2a0825 100644 --- a/admin/routes.go +++ b/admin/routes.go @@ -30,6 +30,11 @@ func RegisterRoutes(mux *http.ServeMux, handlers *AdminHandlers) { r.Post("/decommission/{nodeID}", handlers.handleDecommission) r.Get("/decommission/{nodeID}/status", handlers.handleDecommissionStatus) r.Post("/decommission/{nodeID}/cancel", handlers.handleDecommissionCancel) + r.Group(func(r chi.Router) { + r.Use(RequireSecretMiddleware) + r.Post("/autoinc/release-votes", handlers.handleAutoIncReleaseVotes) + r.Post("/autoinc/sync", handlers.handleAutoIncSync) + }) }) // Per-database transaction operations (outside metadata path) diff --git a/cfg/config.go b/cfg/config.go index 3b12d832..98f69fb3 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -21,10 +21,15 @@ type PromotionConfiguration struct { // ClusterConfiguration controls cluster membership and communication type ClusterConfiguration struct { - GRPCBindAddress string `toml:"grpc_bind_address"` - GRPCAdvertiseAddress string `toml:"grpc_advertise_address"` // Address other nodes use to connect (defaults to hostname:port) - GRPCPort int `toml:"grpc_port"` - SeedNodes []string `toml:"seed_nodes"` + GRPCBindAddress string `toml:"grpc_bind_address"` + GRPCAdvertiseAddress string `toml:"grpc_advertise_address"` // Address other nodes use to connect (defaults to hostname:port) + GRPCPort int `toml:"grpc_port"` + SeedNodes []string `toml:"seed_nodes"` + // Standalone marks a deliberate single-node deployment. Only such a node + // releases its held AUTO_INCREMENT claim votes while it is its own whole + // membership; any other node with a membership of one may simply not have + // learned its cluster yet. + Standalone bool `toml:"standalone"` ClusterSecret string `toml:"cluster_secret"` // PSK for cluster authentication (env: MARMOT_CLUSTER_SECRET) GossipIntervalMS int `toml:"gossip_interval_ms"` GossipFanout int `toml:"gossip_fanout"` @@ -549,6 +554,13 @@ func Validate() error { Msg("Auto-configured gRPC advertise address") } + if Config.Cluster.Standalone && len(Config.Cluster.SeedNodes) > 0 { + return fmt.Errorf("cluster.standalone is set but seed_nodes is not empty: a standalone node joins no cluster") + } + if Config.Cluster.Standalone && Config.Replica.Enabled { + return fmt.Errorf("cluster.standalone is set but replica.enabled is true: a read-only replica follows a cluster") + } + if Config.MySQL.Enabled && (Config.MySQL.Port < 1 || Config.MySQL.Port > 65535) { return fmt.Errorf("invalid MySQL port: %d", Config.MySQL.Port) } diff --git a/cfg/config_test.go b/cfg/config_test.go index dab88270..064ee50a 100644 --- a/cfg/config_test.go +++ b/cfg/config_test.go @@ -3,6 +3,7 @@ package cfg import ( "os" "path/filepath" + "strings" "testing" ) @@ -598,3 +599,54 @@ func TestReplicaConfig_ValidationRequiresFollowAddresses(t *testing.T) { t.Errorf("Expected no validation error with valid follow_addresses, got: %v", err) } } + +// TestValidate_StandaloneWithSeedsIsRefused: a standalone node joins no +// cluster, so naming seeds with it is a contradiction the node must refuse +// rather than release its AUTO_INCREMENT claim votes alone. +func TestValidate_StandaloneWithSeedsIsRefused(t *testing.T) { + original := Config + defer func() { Config = original }() + + Config = &Configuration{ + Cluster: ClusterConfiguration{ + GRPCPort: 8080, + Standalone: true, + SeedNodes: []string{"node1:8080"}, + }, + Replication: ReplicationConfiguration{ + DefaultWriteConsist: "QUORUM", + DefaultReadConsist: "LOCAL_ONE", + }, + } + if err := Validate(); err == nil || !strings.Contains(err.Error(), "standalone") { + t.Fatalf("Validate() = %v, want a standalone/seed_nodes refusal", err) + } +} + +// TestValidate_StandaloneReplicaIsRefused pins R3c-11's configuration half: +// a read-only replica follows a cluster, so it is never a standalone node. +// +// Mutation: drop the replica check. Validate refuses an unrelated field +// instead and the assertion names the missing refusal. +func TestValidate_StandaloneReplicaIsRefused(t *testing.T) { + original := Config + defer func() { Config = original }() + + Config = &Configuration{ + Cluster: ClusterConfiguration{ + GRPCPort: 8080, + Standalone: true, + }, + Replica: ReplicaConfiguration{ + Enabled: true, + FollowAddresses: []string{"node1:8080"}, + }, + Replication: ReplicationConfiguration{ + DefaultWriteConsist: "QUORUM", + DefaultReadConsist: "LOCAL_ONE", + }, + } + if err := Validate(); err == nil || !strings.Contains(err.Error(), "replica.enabled") { + t.Fatalf("Validate() = %v, want a standalone/replica refusal", err) + } +} diff --git a/common/internal_tables.go b/common/internal_tables.go index accc7278..eb309416 100644 --- a/common/internal_tables.go +++ b/common/internal_tables.go @@ -21,6 +21,11 @@ const InternalTablePrefix = "__marmot__" // this constant; see db/autoinc_claim.go for the schema and write path. const AutoIncClaimTableName = InternalTablePrefix + "autoinc" +// AutoIncHoldTableName is the hidden system-database table whose single row, +// while present, holds this node's AUTO_INCREMENT claim votes until it has +// merged claim bases from a majority of its peers. See db/autoinc_vote_hold.go. +const AutoIncHoldTableName = InternalTablePrefix + "autoinc_hold" + // IsInternalTableName reports whether name - compared case-insensitively, as // MySQL/SQLite identifiers are - names one of Marmot's own internal tables // rather than a client table. Used to refuse client SQL that targets Marmot's diff --git a/config.toml.example b/config.toml.example index 4efd1dd0..e8bab624 100644 --- a/config.toml.example +++ b/config.toml.example @@ -53,10 +53,36 @@ grpc_port = 8080 # Example: ["node1.example.com:8080", "node2.example.com:8080"] seed_nodes = [] +# Set to true only for a deliberate single-node deployment (seed_nodes empty). +# A node that initialises a new system database holds its AUTO_INCREMENT claim +# votes until it has merged claim bases from enough cluster members; only a +# standalone node may release them while it is its own whole membership. A +# first cluster node that others will join must leave this false. A +# standalone node that has ever seen another cluster member no longer +# releases alone (it logs why); validation refuses standalone together with +# seed_nodes. +standalone = false + # PSK for cluster authentication (or set MARMOT_CLUSTER_SECRET env var) -# Leave empty to disable authentication +# Leave empty to disable authentication. The AUTO_INCREMENT endpoints +# (/admin/cluster/autoinc/release-votes and /admin/cluster/autoinc/sync) are +# never served without it: on a cluster with no secret they answer 403. cluster_secret = "" +# Membership changes and narrow AUTO_INCREMENT ids. Ids of AUTO_INCREMENT +# columns narrower than BIGINT are unique for a stable membership. Every node +# also raises its claim bases to the maximum every alive peer reports, every +# 10 seconds, which closes the window a membership change opens. Change the +# membership one node at a time: +# 1. add or remove ONE node; +# 2. wait until it is ALIVE and its claim votes are released (its log says +# "AUTO_INCREMENT claim votes released", or it was never held); +# 3. POST /admin/cluster/autoinc/sync (with the cluster secret) on any node; +# 4. make the next change only when the answer's "complete" is true: every +# member answered, is unheld and reached every member. A sync gathers +# only from the peers a node sees ALIVE, so right after a change it can +# be incomplete: re-run until the report is complete. + # Gossip protocol interval (milliseconds) gossip_interval_ms = 1000 diff --git a/coordinator/autoinc_claim.go b/coordinator/autoinc_claim.go index b8187366..b27a9312 100644 --- a/coordinator/autoinc_claim.go +++ b/coordinator/autoinc_claim.go @@ -7,7 +7,10 @@ import ( "math/rand/v2" "time" + "github.com/maxpert/marmot/id" "github.com/maxpert/marmot/protocol" + "github.com/maxpert/marmot/protocol/mysqlcode" + "github.com/maxpert/marmot/protocol/query/transform" ) // maxClaimAttempts bounds how many PREPARE rounds ClaimRange spends on one @@ -31,9 +34,9 @@ const ( claimBackoffCap = 64 * time.Millisecond ) -// ClaimRange is the entry point the range allocator will call to obtain a narrow AUTO_INCREMENT range: it is not dead code -// despite having no production caller yet, since that allocator is a -// separate, later step. Its only callers today are the tests in this file. +// ClaimRange obtains a narrow AUTO_INCREMENT range for the range allocator +// (id.RangeAllocator): on success (newBase, newBase+granted] belongs to this +// node alone. // // It pushes one protocol.Statement through the existing 2PC machinery per // attempt: an AUTO_INCREMENT range claim, carrying AutoIDClaim=true and a @@ -43,6 +46,17 @@ const ( // storedBase <= prevBase && newBase == prevBase && size >= 1 && // newBase+size <= widthMax. // +// Every attempt carries this node's current view of the cluster's total +// membership, the denominator its quorum is computed over, and a participant +// whose own view differs rejects it (prepareAutoIncClaim): majorities of two +// different memberships need not intersect. +// +// size is evaluated afresh for every attempt against the base that attempt +// proposes, because a rejection moves the base and the range's size depends +// on where it starts (the endgame taper and the width ceiling). When size +// reports id.ErrRangeExhausted no claim is sent and ClaimRange returns that +// error. +// // WriteConsistency is pinned to ConsistencyQuorum unconditionally - it never // reads a caller-supplied level or the cluster's configured write default. // QuorumSize(ConsistencyOne, n) and QuorumSize(ConsistencyLocalOne, n) are @@ -70,39 +84,52 @@ const ( // fix by itself, so it ends the attempt loop immediately rather than spending // the full budget. // -// On exhausting maxClaimAttempts, or on hitting a failure with no usable -// base, ClaimRange returns protocol.ErrLockWaitTimeout() (MySQL 1205): the -// standard signal this codebase already uses for "retry me" (see -// runPreparePhase's handling of write-write conflicts). It is wrapped with -// %w around the last attempt's cause so a caller using errors.As can still -// recover the retryable code, and errors.Unwrap the underlying reason for -// logs. +// The two ways a claim can end are kept apart structurally, because a client +// must retry one and must never retry the other: +// - exhaustion - size found no room above the base, or a participant +// rejected the range as passing its own column ceiling (the rejection +// carries ER_DUP_ENTRY, see prepareAutoIncClaim) - returns an error +// wrapping id.ErrRangeExhausted; +// - anything else, on exhausting maxClaimAttempts or on a failure with no +// usable base, returns protocol.ErrLockWaitTimeout() (MySQL 1205), the +// standard "retry me" signal (see runPreparePhase's handling of +// write-write conflicts), wrapped with %w around the last attempt's cause. // // It never touches hookDB and never calls ExecuteLocalWithHooks: it only // builds a statement and drives it through WriteTransaction, exactly like // any other 2PC write. -func (wc *WriteCoordinator) ClaimRange(ctx context.Context, database, table string, prevBase, size uint64) (newBase uint64, granted uint64, err error) { - newBase = prevBase - +func (wc *WriteCoordinator) ClaimRange(ctx context.Context, database, table string, prevBase uint64, size id.RangeSizer) (newBase uint64, granted uint64, err error) { var lastErr error declinedRounds := 0 for attempt := 0; attempt < maxClaimAttempts; attempt++ { + claimSize, sizeErr := size(prevBase) + if sizeErr != nil { + return 0, 0, fmt.Errorf("auto-increment claim for %s.%s: %w", database, table, sizeErr) + } payload, encErr := protocol.EncodeAutoIncClaim(protocol.AutoIncClaim{ - Table: table, - PrevBase: prevBase, - NewBase: newBase, - Size: size, + Table: table, + PrevBase: prevBase, + NewBase: prevBase, + Size: claimSize, + Membership: uint32(wc.nodeProvider.GetTotalMembershipSize()), }) if encErr != nil { return 0, 0, fmt.Errorf("encode auto-increment claim for %s.%s: %w", database, table, encErr) } + // Read each attempt: a DDL applied between attempts raises it. + schemaVersion, versionErr := wc.claimSchemaVersion(database) + if versionErr != nil { + return 0, 0, fmt.Errorf("auto-increment claim for %s.%s: read schema version: %w", database, table, versionErr) + } + startTS := wc.clock.Now() txn := &Transaction{ - ID: startTS.ToTxnID(), - NodeID: wc.nodeID, - StartTS: startTS, - Database: database, + ID: startTS.ToTxnID(), + NodeID: wc.nodeID, + StartTS: startTS, + Database: database, + RequiredSchemaVersion: schemaVersion, // Pinned regardless of any caller or cluster default - see the // method doc for why a quorum of 1 cannot be trusted here. WriteConsistency: protocol.ConsistencyQuorum, @@ -118,13 +145,15 @@ func (wc *WriteCoordinator) ClaimRange(ctx context.Context, database, table stri attemptErr := wc.WriteTransaction(ctx, txn) if attemptErr == nil { - return newBase, size, nil + return prevBase, claimSize, nil } lastErr = attemptErr + if exhaustedByParticipant(attemptErr) { + return 0, 0, fmt.Errorf("auto-increment claim for %s.%s: %w (%v)", database, table, id.ErrRangeExhausted, attemptErr) + } if base, ok := autoIDStoredBaseFromRejection(attemptErr); ok && base > prevBase { prevBase = base - newBase = base continue } if !declinedByParticipant(attemptErr) || attempt == maxClaimAttempts-1 { @@ -141,6 +170,15 @@ func (wc *WriteCoordinator) ClaimRange(ctx context.Context, database, table stri database, table, protocol.ErrLockWaitTimeout(), lastErr) } +// exhaustedByParticipant reports whether a claim round was rejected because +// the range passed a participant's own column ceiling: prepareAutoIncClaim +// gives exactly that rejection ER_DUP_ENTRY, the code MySQL reports when an +// AUTO_INCREMENT column is full, and no other claim rejection carries it. +func exhaustedByParticipant(err error) bool { + var coded *transform.CodedError + return errors.As(err, &coded) && coded.Code == mysqlcode.ErrCodeDupEntry +} + // autoIDStoredBaseFromRejection extracts the highest base a rejecting // participant reported from either shape runPreparePhase can return: a // *RemotePrepareRejectedError (quorum failed to form and a remote rejection @@ -168,6 +206,20 @@ func declinedByParticipant(err error) bool { return errors.As(err, &quorumErr) && quorumErr.Phase == "prepare" && quorumErr.Declined > 0 } +// claimSchemaVersion is the schema version a claim for database carries: +// this node's own, so a participant that has not applied every DDL this node +// has declines the claim, exactly as it declines DML +// (grpc ReplicationHandler's schema version check). A base a lagging +// participant kept for a table name belongs to the incarnation this node +// may already have replaced, a rename's source for one, and voting with it +// could grant ids the new incarnation already holds. +func (wc *WriteCoordinator) claimSchemaVersion(database string) (uint64, error) { + if wc.schemaVersion == nil { + return 0, nil + } + return wc.schemaVersion(database) +} + // sleepClaimBackoff waits the k-th jittered claim backoff, or until ctx ends. func sleepClaimBackoff(ctx context.Context, k int) error { ceiling := claimBackoffBase << k diff --git a/coordinator/autoinc_claim_bench_test.go b/coordinator/autoinc_claim_bench_test.go index 19f1073c..0754d00b 100644 --- a/coordinator/autoinc_claim_bench_test.go +++ b/coordinator/autoinc_claim_bench_test.go @@ -27,7 +27,7 @@ func BenchmarkClaimRange(b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - if _, _, err := wc.ClaimRange(ctx, "testdb", "orders", uint64(i), 50); err != nil { + if _, _, err := wc.ClaimRange(ctx, "testdb", "orders", uint64(i), fixedClaimSize(50)); err != nil { b.Fatalf("ClaimRange: %v", err) } } diff --git a/coordinator/autoinc_claim_test.go b/coordinator/autoinc_claim_test.go index f7c52b44..d37a9810 100644 --- a/coordinator/autoinc_claim_test.go +++ b/coordinator/autoinc_claim_test.go @@ -8,7 +8,9 @@ import ( "time" "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/id" "github.com/maxpert/marmot/protocol" + "github.com/maxpert/marmot/protocol/mysqlcode" ) // claimStepReplicator is a minimal fake Replicator used only by the @@ -123,7 +125,7 @@ func TestClaimRange_SuccessfulClaimSingleRound(t *testing.T) { nodeProvider := newMockNodeProvider([]uint64{1}) wc := NewWriteCoordinator(1, nodeProvider, fake, fake, 200*time.Millisecond, hlc.NewClock(1)) - newBase, granted, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, 50) + newBase, granted, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, fixedClaimSize(50)) if err != nil { t.Fatalf("ClaimRange: %v", err) } @@ -173,7 +175,7 @@ func TestClaimRange_ConsistencyIsAlwaysQuorum(t *testing.T) { nodeProvider := newMockNodeProvider([]uint64{1, 2, 3}) wc := NewWriteCoordinator(1, nodeProvider, fake, fake, 100*time.Millisecond, hlc.NewClock(1)) - _, _, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, 50) + _, _, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, fixedClaimSize(50)) if err == nil { t.Fatal("expected ClaimRange to fail with only 1 of 3 nodes acking - it would succeed under a quorum of 1, proving consistency was not pinned to QUORUM") } @@ -198,7 +200,7 @@ func TestClaimRange_RetriesAboveRejectedBase(t *testing.T) { nodeProvider := newMockNodeProvider([]uint64{1}) wc := NewWriteCoordinator(1, nodeProvider, fake, fake, 200*time.Millisecond, hlc.NewClock(1)) - newBase, granted, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, 50) + newBase, granted, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, fixedClaimSize(50)) if err != nil { t.Fatalf("ClaimRange: %v", err) } @@ -234,7 +236,7 @@ func TestClaimRange_RetriesAboveMaximumRejectedBase(t *testing.T) { nodeProvider := newMockNodeProvider([]uint64{1, 2, 3}) wc := NewWriteCoordinator(1, nodeProvider, fake, fake, 200*time.Millisecond, hlc.NewClock(1)) - newBase, granted, err := wc.ClaimRange(context.Background(), "testdb", "orders", 40, 10) + newBase, granted, err := wc.ClaimRange(context.Background(), "testdb", "orders", 40, fixedClaimSize(10)) if err != nil { t.Fatalf("ClaimRange: %v", err) } @@ -273,7 +275,7 @@ func TestClaimRange_EightConsecutiveRejectionsReturn1205(t *testing.T) { nodeProvider := newMockNodeProvider([]uint64{1}) wc := NewWriteCoordinator(1, nodeProvider, fake, fake, 200*time.Millisecond, hlc.NewClock(1)) - _, _, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, 50) + _, _, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, fixedClaimSize(50)) if err == nil { t.Fatal("expected ClaimRange to give up after 8 rejections, got nil error") } @@ -333,7 +335,7 @@ func TestClaimRange_BacksOffWhileAnotherClaimHoldsTheKey(t *testing.T) { wc := NewWriteCoordinator(1, nodeProvider, holder, holder, 200*time.Millisecond, hlc.NewClock(1)) start := time.Now() - newBase, granted, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, 50) + newBase, granted, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, fixedClaimSize(50)) if err != nil { t.Fatalf("a claim gave up while another claim held the key: %v", err) } @@ -367,7 +369,7 @@ func TestClaimRange_QuorumNotAchievedReturns1205WithoutRetrying(t *testing.T) { nodeProvider := newMockNodeProvider([]uint64{1, 2, 3}) wc := NewWriteCoordinator(1, nodeProvider, fake, fake, 100*time.Millisecond, hlc.NewClock(1)) - _, _, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, 50) + _, _, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, fixedClaimSize(50)) if err == nil { t.Fatal("expected an error when quorum cannot be achieved") } @@ -383,3 +385,96 @@ func TestClaimRange_QuorumNotAchievedReturns1205WithoutRetrying(t *testing.T) { t.Fatalf("PREPARE calls to the coordinator's own node = %d, want exactly 1 (no retry spin on an unusable failure)", got) } } + +// fixedClaimSize is a RangeSizer asking for the same size above any base: +// these tests pin the claim protocol, not the allocator's sizing policy. +func fixedClaimSize(size uint64) id.RangeSizer { + return func(uint64) (uint64, error) { return size, nil } +} + +// TestClaimRange_SizerExhaustionIsTerminal pins the claimant-side half of +// exhaustion: when the sizer finds no room above the base a rejection taught +// it, ClaimRange stops without another round and reports +// id.ErrRangeExhausted - never the retryable 1205. +// +// Mutation: treat a sizer error as a retryable failure. The error is then +// the lock-wait timeout and "exhaustion reached the client as retryable" +// fires. +func TestClaimRange_SizerExhaustionIsTerminal(t *testing.T) { + InitTestTelemetry() + + fake := newClaimStepReplicator() + fake.push(1, rejectionWithBase(127)) + wc := NewWriteCoordinator(1, newMockNodeProvider([]uint64{1}), fake, fake, 200*time.Millisecond, hlc.NewClock(1)) + + sizer := func(base uint64) (uint64, error) { + if base >= 127 { + return 0, id.ErrRangeExhausted + } + return 1, nil + } + _, _, err := wc.ClaimRange(context.Background(), "testdb", "tiny", 120, sizer) + if !errors.Is(err, id.ErrRangeExhausted) { + t.Fatalf("err = %v, want id.ErrRangeExhausted", err) + } + var mysqlErr *protocol.MySQLError + if errors.As(err, &mysqlErr) { + t.Fatalf("exhaustion reached the client as retryable %d", mysqlErr.Code) + } + if got := fake.prepareCallCount(1); got != 1 { + t.Fatalf("PREPARE calls = %d, want 1: no round may be sent once the sizer finds no room", got) + } +} + +// TestClaimRange_ParticipantExhaustionIsTerminal pins the participant-side +// half: a rejection carrying ER_DUP_ENTRY - the one prepareAutoIncClaim gives +// a range past its own column ceiling - ends the claim as exhausted, while a +// rejection with no code and no usable base stays the retryable 1205. +// +// Mutation: drop the exhaustedByParticipant check. The participant's verdict +// then reaches the client as 1205 and "a full column was reported as +// retryable" fires. +func TestClaimRange_ParticipantExhaustionIsTerminal(t *testing.T) { + InitTestTelemetry() + + fake := newClaimStepReplicator() + fake.push(1, &ReplicationResponse{Rejected: true, Error: "auto-increment claim for tiny exhausts the column", + AutoIDStoredBase: 120, ErrorCode: mysqlcode.ErrCodeDupEntry}) + wc := NewWriteCoordinator(1, newMockNodeProvider([]uint64{1}), fake, fake, 200*time.Millisecond, hlc.NewClock(1)) + _, _, err := wc.ClaimRange(context.Background(), "testdb", "tiny", 120, fixedClaimSize(64)) + if !errors.Is(err, id.ErrRangeExhausted) { + t.Fatalf("a full column was reported as retryable: %v", err) + } + + unavailable := newClaimStepReplicator() + unavailable.push(1, &ReplicationResponse{Rejected: true, Error: "refused", AutoIDStoredBase: 120}) + wc = NewWriteCoordinator(1, newMockNodeProvider([]uint64{1}), unavailable, unavailable, 200*time.Millisecond, hlc.NewClock(1)) + _, _, err = wc.ClaimRange(context.Background(), "testdb", "tiny", 120, fixedClaimSize(64)) + if errors.Is(err, id.ErrRangeExhausted) { + t.Fatalf("a rejection without the exhaustion code was reported as exhaustion: %v", err) + } + var mysqlErr *protocol.MySQLError + if !errors.As(err, &mysqlErr) || mysqlErr.Code != protocol.ErrCodeLockTimeout { + t.Fatalf("err = %v, want the retryable 1205", err) + } +} + +// TestClaimRange_CarriesTheClaimantsMembership pins that every attempt +// carries the membership this node's quorum is computed over, so a +// participant that counts the cluster differently can refuse it. +// +// Mutation: leave Membership unset. "claim carries membership 0" fires. +func TestClaimRange_CarriesTheClaimantsMembership(t *testing.T) { + InitTestTelemetry() + + fake := newClaimStepReplicator() + wc := NewWriteCoordinator(1, newMockNodeProvider([]uint64{1, 2, 3}), fake, fake, 200*time.Millisecond, hlc.NewClock(1)) + _, _, err := wc.ClaimRange(context.Background(), "testdb", "orders", 0, fixedClaimSize(8)) + if err != nil { + t.Fatalf("ClaimRange: %v", err) + } + claim := decodePreparedClaim(t, fake.prepareCall(2, 0)) + if claim.Membership != 3 { + t.Fatalf("claim carries membership %d, want 3", claim.Membership) + } +} diff --git a/coordinator/commit_lost_pinned_state_test.go b/coordinator/commit_lost_pinned_state_test.go index ffb140b1..80ab71ff 100644 --- a/coordinator/commit_lost_pinned_state_test.go +++ b/coordinator/commit_lost_pinned_state_test.go @@ -52,6 +52,9 @@ func setupLostPinnedState(t *testing.T) *lostPinnedStateSetup { dbMgr, err := db.NewDatabaseManager(tmpDir, 1, clock) require.NoError(t, err) t.Cleanup(func() { dbMgr.Close() }) + // A single node releases the claim votes its new system database holds, + // as a standalone node's merge does. + require.NoError(t, dbMgr.MergeAutoIncBasesAndReleaseVotes(nil)) const dbName = "lostpinned" require.NoError(t, dbMgr.CreateDatabase(dbName)) @@ -61,6 +64,8 @@ func setupLostPinnedState(t *testing.T) *lostPinnedStateSetup { schemaVersionMgr := db.NewSchemaVersionManager(systemDB.GetMetaStore()) nodeProvider := coordinator.NewMockNodeProvider([]uint64{1}) + // A claim participant counts the cluster as the claimant does. + dbMgr.SetClusterMembership(nodeProvider.GetTotalMembershipSize) writeCoord := coordinator.NewWriteCoordinator( 1, nodeProvider, diff --git a/coordinator/describe_columns.go b/coordinator/describe_columns.go index 96499cf5..02f2f838 100644 --- a/coordinator/describe_columns.go +++ b/coordinator/describe_columns.go @@ -22,18 +22,20 @@ const describeColumnsTimeout = 5 * time.Second // far from its cause. // // Statements that return nothing, and statements the database cannot prepare, -// yield no columns; the caller then omits the definitions. -func (h *CoordinatorHandler) DescribeResultColumns(session *protocol.ConnectionSession, sql string) ([]protocol.ColumnDef, error) { - if session == nil || session.CurrentDatabase == "" { +// yield no columns; the caller then omits the definitions. The statement is +// described in database, the one it names or the session's: a qualified +// table's name has already been stripped from sql. +func (h *CoordinatorHandler) DescribeResultColumns(session *protocol.ConnectionSession, database, sql string) ([]protocol.ColumnDef, error) { + if session == nil || database == "" { return nil, nil } if h.dbManager == nil { return nil, nil } - replicatedDB, err := h.dbManager.GetReplicatedDatabase(session.CurrentDatabase) + replicatedDB, err := h.dbManager.GetReplicatedDatabase(database) if err != nil { - return nil, fmt.Errorf("failed to get database %s: %w", session.CurrentDatabase, err) + return nil, fmt.Errorf("failed to get database %s: %w", database, err) } ctx, cancel := context.WithTimeout(context.Background(), describeColumnsTimeout) diff --git a/coordinator/handler.go b/coordinator/handler.go index 03ccb5dd..aeafd9a9 100644 --- a/coordinator/handler.go +++ b/coordinator/handler.go @@ -15,10 +15,13 @@ import ( "github.com/maxpert/marmot/cfg" "github.com/maxpert/marmot/common" "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/id" "github.com/maxpert/marmot/modules/vecindex" "github.com/maxpert/marmot/protocol" "github.com/maxpert/marmot/protocol/determinism" "github.com/maxpert/marmot/protocol/handlers" + "github.com/maxpert/marmot/protocol/mysqlcode" + "github.com/maxpert/marmot/protocol/query/rules" "github.com/maxpert/marmot/protocol/query/transform" "github.com/maxpert/marmot/telemetry" "github.com/rs/zerolog/log" @@ -195,15 +198,19 @@ type PublisherRegistry interface { // CoordinatorHandler implements protocol.ConnectionHandler // It routes queries to the appropriate coordinator (Read or Write) type CoordinatorHandler struct { - nodeID uint64 - writeCoord *WriteCoordinator - readCoord *ReadCoordinator - clock *hlc.Clock - dbManager DatabaseManager - ddlLockMgr *DDLLockManager - schemaVersionMgr SchemaVersionManager - nodeRegistry NodeRegistry - metadata *handlers.MetadataHandler + nodeID uint64 + writeCoord *WriteCoordinator + readCoord *ReadCoordinator + clock *hlc.Clock + dbManager DatabaseManager + ddlLockMgr *DDLLockManager + schemaVersionMgr SchemaVersionManager + nodeRegistry NodeRegistry + metadata *handlers.MetadataHandler + // narrowIDs mints ids for AUTO_INCREMENT columns declared narrower than + // BIGINT, from ranges this node claims through writeCoord. Nil when the + // handler has no write coordinator to claim through. + narrowIDs *id.RangeAllocator recentTxnIDs sync.Map // txn_id -> conn_id for duplicate detection publisherRegistry PublisherRegistry publisherMu sync.RWMutex @@ -228,7 +235,15 @@ type CoordinatorHandler struct { // NewCoordinatorHandler creates a new handler func NewCoordinatorHandler(nodeID uint64, writeCoord *WriteCoordinator, readCoord *ReadCoordinator, clock *hlc.Clock, dbManager DatabaseManager, ddlLockMgr *DDLLockManager, schemaVersionMgr SchemaVersionManager, nodeRegistry NodeRegistry) *CoordinatorHandler { + // The narrow allocator claims through this handler's own write + // coordinator, so it is built here, beside it, rather than in the shared + // query pipeline. A claim gets the same bound as any other write. + var narrowIDs *id.RangeAllocator + if writeCoord != nil { + narrowIDs = id.NewRangeAllocator(writeCoord, getWriteTimeout()) + } return &CoordinatorHandler{ + narrowIDs: narrowIDs, nodeID: nodeID, writeCoord: writeCoord, readCoord: readCoord, @@ -241,6 +256,31 @@ func NewCoordinatorHandler(nodeID uint64, writeCoord *WriteCoordinator, readCoor } } +// narrowAllocator returns the narrow allocator as the interface the parser +// takes, keeping a nil allocator a nil interface. +func (h *CoordinatorHandler) narrowAllocator() rules.NarrowAllocator { + if h.narrowIDs == nil { + return nil + } + return h.narrowIDs +} + +// TableIncarnationEnded discards this node's in-memory id range for a table +// whose incarnation a DDL statement ended (db.AutoIncIncarnationListener). +func (h *CoordinatorHandler) TableIncarnationEnded(database, table string) { + if h.narrowIDs != nil { + h.narrowIDs.Forget(database, table) + } +} + +// DatabaseIncarnationEnded discards this node's in-memory id ranges for every +// table of a dropped database (db.AutoIncIncarnationListener). +func (h *CoordinatorHandler) DatabaseIncarnationEnded(database string) { + if h.narrowIDs != nil { + h.narrowIDs.ForgetDatabase(database) + } +} + // SetDraining marks the handler as draining (node is leaving the cluster). // When draining, new write queries are rejected with ER_SERVER_SHUTDOWN (1053). func (h *CoordinatorHandler) SetDraining(d bool) { @@ -291,19 +331,6 @@ func (h *CoordinatorHandler) HandleQuery(session *protocol.ConnectionSession, sq return h.handleMarmotCommand(session, sql) } - // Build schema lookup function for auto-increment ID injection. - // Uses cached schema via DatabaseManager - does NOT query SQLite PRAGMA. - var schemaLookup protocol.SchemaLookupFunc - if h.dbManager != nil && session.CurrentDatabase != "" { - dbName := session.CurrentDatabase - schemaLookup = func(table string) *transform.SchemaInfo { - info, err := h.dbManager.GetTranspilerSchema(dbName, table) - if err != nil { - return nil - } - return info - } - } schemaProvider := func(database, table string) *transform.SchemaInfo { if table == "" || h.dbManager == nil { return nil @@ -322,10 +349,18 @@ func (h *CoordinatorHandler) HandleQuery(session *protocol.ConnectionSession, sq } return schemaInfo } + // Auto-increment id injection looks tables up in the statement's own + // database when it names one, and in the session's otherwise. + var schemaLookup protocol.SchemaLookupFunc + if h.dbManager != nil { + schemaLookup = protocol.SchemaLookupFunc(schemaProvider) + } // Parse with options based on session state stmt := protocol.ParseStatementWithOptions(sql, protocol.ParseOptions{ SchemaLookup: schemaLookup, + NarrowIDs: h.narrowAllocator(), + BoundParams: params, SchemaProvider: schemaProvider, SkipTranspilation: !session.TranspilationEnabled, ExtractLiterals: true, // Enable literal extraction for parameterized execution @@ -528,8 +563,21 @@ func (h *CoordinatorHandler) HandleLoadData(session *protocol.ConnectionSession, // Preserve the original SQL text so replicated peers re-apply the same // LOAD DATA LOCAL INFILE semantics (parser normalization can drop LOCAL). stmt.SQL = sql - stmt.Database = session.CurrentDatabase stmt.LoadDataPayload = data + database, table, targetErr := protocol.LoadDataTarget(sql) + if database == "" { + database = session.CurrentDatabase + } + stmt.Database = database + + // A table with a narrow AUTO_INCREMENT column takes its rows as ordinary + // INSERTs through this handler, so their ids are generated or admitted + // like any other INSERT's (admitNarrowIDs). Replicated as a statement, + // every node would instead apply the file itself and let SQLite assign + // any missing key on its own. + if targetErr == nil && h.hasNarrowAutoIncrement(database, table) { + return h.loadNarrowData(session, database, sql, data) + } consistency, _ := protocol.ParseConsistencyLevel(cfg.Config.Replication.DefaultWriteConsist) if session.InTransaction() { @@ -538,6 +586,38 @@ func (h *CoordinatorHandler) HandleLoadData(session *protocol.ConnectionSession, return h.handleMutation(stmt, nil, consistency) } +// loadNarrowData runs a LOAD DATA into a narrow table in database as INSERTs +// through this handler. A target in another database than the session's is +// loaded as its own autocommit work on a session of that database, since an +// explicit transaction commits to the database it began in; inside an open +// transaction that is refused. +func (h *CoordinatorHandler) loadNarrowData(session *protocol.ConnectionSession, database, sql string, data []byte) (*protocol.ResultSet, error) { + if database == session.CurrentDatabase { + return protocol.ExecuteLoadDataLocal(session, h, sql, data) + } + if session.InTransaction() { + return nil, transform.NewCodedError(mysqlcode.ErrCodeNotSupportedYet, + "This version of MySQL doesn't yet support 'LOAD DATA into database `%s` inside a transaction begun in `%s`'", + database, session.CurrentDatabase) + } + target := &protocol.ConnectionSession{ + ConnID: session.ConnID, + CurrentDatabase: database, + TranspilationEnabled: session.TranspilationEnabled, + } + return protocol.ExecuteLoadDataLocal(target, h, sql, data) +} + +// hasNarrowAutoIncrement reports whether table has a narrow AUTO_INCREMENT +// column (rules.IsNarrow). +func (h *CoordinatorHandler) hasNarrowAutoIncrement(database, table string) bool { + if h.dbManager == nil || database == "" || table == "" { + return false + } + info, err := h.dbManager.GetTranspilerSchema(database, table) + return err == nil && rules.IsNarrow(info) +} + func (h *CoordinatorHandler) handleMutation(stmt protocol.Statement, params []interface{}, consistency protocol.ConsistencyLevel) (*protocol.ResultSet, error) { queryStart := time.Now() @@ -684,6 +764,16 @@ func (h *CoordinatorHandler) handleMutation(stmt protocol.Statement, params []in } }() + // Every id the statement put into a narrow AUTO_INCREMENT column must be + // admitted before it replicates; a refusal fails the statement and + // releases what its local execution left behind. + if err := h.admitNarrowIDs(stmt.Database, txn.Statements); err != nil { + h.writeCoord.abortTransaction(ctx, []uint64{h.nodeID}, txn.ID, txn.Database) + telemetry.QueriesTotal.With("dml", "failed").Inc() + telemetry.QueryDurationSeconds.With("dml").Observe(time.Since(queryStart).Seconds()) + return nil, err + } + // A DML that matched no rows leaves no statements to replicate. Running 2PC // for it would burn a cluster round trip to commit nothing. if len(txn.Statements) > 0 { @@ -1387,10 +1477,14 @@ func (h *CoordinatorHandler) handleCommit(session *protocol.ConnectionSession) ( ctx, cancel := context.WithTimeout(context.Background(), writeTimeoutForStatements(txn.Statements)) defer cancel() - // DML that matched no rows leaves nothing to replicate. A transaction whose + // Every id the transaction put into a narrow AUTO_INCREMENT column must be + // admitted before it replicates; a refusal fails the COMMIT. DML that + // matched no rows leaves nothing to replicate. A transaction whose // statements all collapsed that way is a no-op and skips 2PC. - var err error - if len(txn.Statements) > 0 { + err := h.admitNarrowIDs(txn.Database, txn.Statements) + if err != nil { + h.writeCoord.abortTransaction(ctx, []uint64{h.nodeID}, txn.ID, txn.Database) + } else if len(txn.Statements) > 0 { err = h.writeCoord.WriteTransaction(ctx, txn) } diff --git a/coordinator/narrow_autoinc_admit.go b/coordinator/narrow_autoinc_admit.go new file mode 100644 index 00000000..ac386400 --- /dev/null +++ b/coordinator/narrow_autoinc_admit.go @@ -0,0 +1,103 @@ +package coordinator + +import ( + "github.com/maxpert/marmot/encoding" + "github.com/maxpert/marmot/protocol" + "github.com/maxpert/marmot/protocol/mysqlcode" + "github.com/maxpert/marmot/protocol/query/rules" + "github.com/maxpert/marmot/protocol/query/transform" +) + +// admitNarrowIDs admits every id a write transaction's rows put into a narrow +// AUTO_INCREMENT column, through the same allocator rule that admits literal +// ids at parse time (id.RangeAllocator.Admit), before the transaction is +// replicated. It is what covers the ids no parser can see: bound parameters, +// INSERT ... SELECT, REPLACE, LOAD DATA, an UPDATE that changes the column, +// and a key SQLite assigned itself. +// +// An id this node's allocator issued, or already admitted, passes. An id +// above the cluster's allocation base raises the base past it. An id at or +// below the base that no range of this node holds is refused: replicated +// inserts apply as INSERT OR REPLACE (db/cdc_applier.go), so once the node +// whose range holds it issued it, one of the two rows would silently replace +// the other everywhere. +// +// defaultDB names the database of a statement that does not carry one. The +// error is the coded one the client must see. +func (h *CoordinatorHandler) admitNarrowIDs(defaultDB string, stmts []protocol.Statement) error { + if h.dbManager == nil { + return nil + } + var lookups narrowTableLookups + for i := range stmts { + stmt := &stmts[i] + if len(stmt.NewValues) == 0 { + continue + } + database := stmt.Database + if database == "" { + database = defaultDB + } + info := lookups.get(h.dbManager, database, stmt.TableName) + if info == nil { + continue + } + newID, ok := cdcRowID(stmt.NewValues, info.AutoIncrementColumn) + if !ok { + continue + } + if oldID, had := cdcRowID(stmt.OldValues, info.AutoIncrementColumn); had && oldID == newID { + // An UPDATE that left the column alone writes no new id. + continue + } + if h.narrowIDs == nil { + return transform.NewCodedError(mysqlcode.ErrCodeUnknown, + "no allocator for AUTO_INCREMENT column `%s` of table `%s`", info.AutoIncrementColumn, stmt.TableName) + } + widthMax := rules.WidthMax(info) + if err := h.narrowIDs.Admit(database, stmt.TableName, widthMax, newID); err != nil { + return rules.AdmissionError(err, stmt.TableName, info.AutoIncrementColumn, widthMax) + } + } + return nil +} + +// narrowTableLookups memoises, for one transaction, which of its tables have +// a narrow auto-increment column. Transactions touch few tables, so a slice +// beats a map. +type narrowTableLookups []narrowTableLookup + +type narrowTableLookup struct { + database string + table string + info *transform.SchemaInfo // nil: no narrow auto-increment column +} + +func (l *narrowTableLookups) get(dbManager DatabaseManager, database, table string) *transform.SchemaInfo { + for _, known := range *l { + if known.database == database && known.table == table { + return known.info + } + } + info, err := dbManager.GetTranspilerSchema(database, table) + if err != nil || !rules.IsNarrow(info) { + info = nil + } + *l = append(*l, narrowTableLookup{database: database, table: table, info: info}) + return info +} + +// cdcRowID decodes column's value from a CDC row image as a positive id. A +// missing, non-integer or non-positive value is not an id the allocator could +// ever issue, so it can never meet one. +func cdcRowID(values map[string][]byte, column string) (uint64, bool) { + raw, ok := values[column] + if !ok { + return 0, false + } + var id int64 + if err := encoding.Unmarshal(raw, &id); err != nil || id <= 0 { + return 0, false + } + return uint64(id), true +} diff --git a/coordinator/narrow_autoinc_admit_test.go b/coordinator/narrow_autoinc_admit_test.go new file mode 100644 index 00000000..6d9086e3 --- /dev/null +++ b/coordinator/narrow_autoinc_admit_test.go @@ -0,0 +1,257 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package coordinator_test + +import ( + "strings" + "testing" + + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// narrowTable creates g (INT AUTO_INCREMENT) and plain (a BIGINT source) on s. +func narrowTable(t *testing.T, s *noopDMLSetup) { + t.Helper() + for _, ddl := range []string{ + "CREATE TABLE g (id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)", + "CREATE TABLE plain (id BIGINT PRIMARY KEY, name TEXT)", + } { + _, err := s.handler.HandleQuery(s.session, ddl, nil) + require.NoError(t, err) + } +} + +// generatedID inserts one row with a generated id through h and returns it. +func generatedID(t *testing.T, s *noopDMLSetup) int64 { + t.Helper() + res, err := s.handler.HandleQuery(s.session, "INSERT INTO g (name) VALUES ('generated')", nil) + require.NoError(t, err) + return res.LastInsertId +} + +// TestNarrowIDsWrittenByEveryShapeRaiseTheBase pins R3c-1: whatever shape +// writes an id into a narrow AUTO_INCREMENT column - a literal, a bound +// parameter, INSERT ... SELECT, REPLACE, an UPDATE of the column, LOAD DATA - +// the allocator admits it before the write replicates, so the next generated +// id lands above it instead of colliding with it. +// +// Mutation: make admitNarrowIDs return nil without admitting. Every shape but +// the literal one then generates an id at or below the one it wrote, and +// "generated id ... is not above" fires (or the generated INSERT collides). +func TestNarrowIDsWrittenByEveryShapeRaiseTheBase(t *testing.T) { + cases := []struct { + name string + write func(t *testing.T, s *noopDMLSetup) + highest int64 + }{ + {"literal", func(t *testing.T, s *noopDMLSetup) { + _, err := s.handler.HandleQuery(s.session, "INSERT INTO g (id, name) VALUES (5000, 'x')", nil) + require.NoError(t, err) + }, 5000}, + {"bound parameter", func(t *testing.T, s *noopDMLSetup) { + _, err := s.handler.HandleQuery(s.session, "INSERT INTO g (id, name) VALUES (?, ?)", []interface{}{int64(6000), "x"}) + require.NoError(t, err) + }, 6000}, + {"insert select", func(t *testing.T, s *noopDMLSetup) { + for _, id := range []int{7000, 7100, 7200} { + _, err := s.handler.HandleQuery(s.session, "INSERT INTO plain (id, name) VALUES (?, 'x')", []interface{}{id}) + require.NoError(t, err) + } + _, err := s.handler.HandleQuery(s.session, "INSERT INTO g (id, name) SELECT id, name FROM plain", nil) + require.NoError(t, err) + }, 7200}, + {"replace", func(t *testing.T, s *noopDMLSetup) { + _, err := s.handler.HandleQuery(s.session, "REPLACE INTO g (id, name) VALUES (?, ?)", []interface{}{int64(8000), "x"}) + require.NoError(t, err) + }, 8000}, + {"update of the column", func(t *testing.T, s *noopDMLSetup) { + generatedID(t, s) + _, err := s.handler.HandleQuery(s.session, "UPDATE g SET id = ? WHERE name = 'generated'", []interface{}{int64(9000)}) + require.NoError(t, err) + }, 9000}, + {"load data", func(t *testing.T, s *noopDMLSetup) { + _, err := s.handler.HandleLoadData(s.session, + "LOAD DATA LOCAL INFILE 'rows.csv' INTO TABLE g FIELDS TERMINATED BY ',' (id, name)", + []byte("9500,a\n9600,b\n")) + require.NoError(t, err) + }, 9600}, + {"explicit transaction", func(t *testing.T, s *noopDMLSetup) { + _, err := s.handler.HandleQuery(s.session, "BEGIN", nil) + require.NoError(t, err) + _, err = s.handler.HandleQuery(s.session, "INSERT INTO g (id, name) VALUES (?, ?)", []interface{}{int64(9900), "x"}) + require.NoError(t, err) + _, err = s.handler.HandleQuery(s.session, "COMMIT", nil) + require.NoError(t, err) + }, 9900}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := setupNoopDML(t) + narrowTable(t, s) + tc.write(t, s) + for range 3 { + id := generatedID(t, s) + require.Greater(t, id, tc.highest, "generated id %d is not above the id %d the %s wrote", id, tc.highest, tc.name) + require.LessOrEqual(t, id, int64(int32Max)) + } + }) + } +} + +// TestNarrowIDsBelowTheBaseAreRefused pins R3c-2 on one node whose allocator +// state is gone (a restarted process): an id at or below the allocation base +// that lies in no range this process claimed may lie in another node's +// unissued range, so every shape that writes one is refused with 1235 and +// writes nothing, and a generated INSERT still succeeds afterwards. +// +// Mutation: admit any id at or below the base. The writes succeed and "was +// written" fires. +func TestNarrowIDsBelowTheBaseAreRefused(t *testing.T) { + s := setupNoopDML(t) + narrowTable(t, s) + first := generatedID(t, s) // claims 1..64 and issues 1 + require.Equal(t, int64(1), first) + _, err := s.handler.HandleQuery(s.session, "INSERT INTO plain (id, name) VALUES (30, 'x')", nil) + require.NoError(t, err) + + s.handler = s.restartHandler() + writes := map[string]func() error{ + "literal": func() error { + _, err := s.handler.HandleQuery(s.session, "INSERT INTO g (id, name) VALUES (10, 'x')", nil) + return err + }, + "bound parameter": func() error { + _, err := s.handler.HandleQuery(s.session, "INSERT INTO g (id, name) VALUES (?, ?)", []interface{}{int64(20), "x"}) + return err + }, + "insert select": func() error { + _, err := s.handler.HandleQuery(s.session, "INSERT INTO g (id, name) SELECT id, name FROM plain", nil) + return err + }, + "update of the column": func() error { + _, err := s.handler.HandleQuery(s.session, "UPDATE g SET id = 40 WHERE id = 1", nil) + return err + }, + "load data": func() error { + _, err := s.handler.HandleLoadData(s.session, + "LOAD DATA LOCAL INFILE 'rows.csv' INTO TABLE g FIELDS TERMINATED BY ',' (id, name)", []byte("50,x\n")) + return err + }, + } + for name, write := range writes { + err := write() + require.Error(t, err, "%s: an id below the base was written", name) + require.True(t, strings.Contains(err.Error(), "1235"), "%s: err = %v, want MySQL 1235", name, err) + } + + var count int + require.NoError(t, s.conn.QueryRow("SELECT COUNT(*) FROM g WHERE id IN (10, 20, 30, 40, 50)").Scan(&count)) + require.Zero(t, count, "a refused id was written") + + id := generatedID(t, s) + require.Greater(t, id, int64(64), "the restarted node must claim a fresh range") +} + +// TestNarrowLoadDataWithoutTheColumnGetsGeneratedIDs pins that LOAD DATA into +// a narrow table whose rows omit the id gets ids from the allocator - fitting +// the column and contiguous - rather than a key each node's SQLite assigns on +// its own. +// +// Mutation: replicate LOAD DATA into a narrow table as a statement again. +// The ids then come from SQLite and the first generated INSERT afterwards +// collides with one, and this fails. +func TestNarrowLoadDataWithoutTheColumnGetsGeneratedIDs(t *testing.T) { + s := setupNoopDML(t) + narrowTable(t, s) + _, err := s.handler.HandleLoadData(s.session, + "LOAD DATA LOCAL INFILE 'rows.csv' INTO TABLE g FIELDS TERMINATED BY ',' (name)", []byte("a\nb\nc\n")) + require.NoError(t, err) + ids := narrowIDs(t, s, "SELECT id FROM g ORDER BY id") + require.Len(t, ids, 3) + next := generatedID(t, s) + require.Greater(t, next, ids[2], "a generated id after LOAD DATA must lie above the ids it wrote") +} + +// TestNarrowRewritesOfExistingRowsAreAdmitted pins that R3c-2's refusal is +// about NEW ids only: after a restart, when no range below the base is this +// node's, statements that rewrite a row whose id already exists - an UPDATE +// of another column, an INSERT ... ON DUPLICATE KEY UPDATE - still succeed. +// The id was issued once already, so it lies in no node's unissued range. +// +// Mutation: admit the id of an UPDATE that left it unchanged. "rewriting an +// existing row was refused" fires on the UPDATE. +func TestNarrowRewritesOfExistingRowsAreAdmitted(t *testing.T) { + s := setupNoopDML(t) + narrowTable(t, s) + id := generatedID(t, s) + s.handler = s.restartHandler() + + for _, q := range []string{ + "UPDATE g SET name = 'updated' WHERE id = ?", + "INSERT INTO g (id, name) VALUES (?, 'upserted') ON DUPLICATE KEY UPDATE name = 'upserted'", + } { + _, err := s.handler.HandleQuery(s.session, q, []interface{}{id}) + require.NoError(t, err, "rewriting an existing row was refused: %s", q) + } + var name string + require.NoError(t, s.conn.QueryRow("SELECT name FROM g WHERE id = ?", id).Scan(&name)) + require.Equal(t, "upserted", name) +} + +// TestNarrowQualifiedTargetsUseTheirOwnDatabase is F4 and L3: an INSERT or a +// LOAD DATA naming its table as db.t, from a session whose current database +// is another one, looks the table up, allocates and admits in db - not in the +// session's database, where the table does not exist. Every form gets +// generated narrow ids and none is refused. +// +// The writes run after a restart, when this node owns no range below the +// base: an id the allocator did not generate (a SQLite rowid) is then refused +// with 1235, so the test sees whether the id was generated. +// +// Mutation: look tables up in the session's database only (drop the +// statement's qualifier in the handler's schema lookup). No id is generated, +// SQLite assigns rowid 4, the admission in db refuses it, and "a qualified +// INSERT was refused" fires. +func TestNarrowQualifiedTargetsUseTheirOwnDatabase(t *testing.T) { + s := setupNoopDML(t) + _, err := s.handler.HandleQuery(s.session, "CREATE DATABASE other", nil) + require.NoError(t, err) + other := &protocol.ConnectionSession{ConnID: 2, CurrentDatabase: "other", TranspilationEnabled: true} + _, err = s.handler.HandleQuery(other, "CREATE TABLE g (id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)", nil) + require.NoError(t, err) + for i := 0; i < 3; i++ { + _, err = s.handler.HandleQuery(other, "INSERT INTO g (name) VALUES ('first')", nil) + require.NoError(t, err) + } + s.handler = s.restartHandler() + + for _, q := range []struct { + sql string + params []interface{} + }{ + {"INSERT INTO other.g (name) VALUES ('text')", nil}, + {"INSERT INTO `other`.`g` (name) VALUES (?)", []interface{}{"bound"}}, + {"INSERT INTO other.g (id, name) VALUES (?, ?)", []interface{}{nil, "bound-null"}}, + } { + res, err := s.handler.HandleQuery(s.session, q.sql, q.params) + require.NoError(t, err, "a qualified INSERT was refused: %s", q.sql) + require.Greater(t, res.LastInsertId, int64(3), "%s did not get a generated id", q.sql) + require.LessOrEqual(t, res.LastInsertId, int64(int32Max), "%s got a wide id", q.sql) + } + _, err = s.handler.HandleLoadData(s.session, + "LOAD DATA LOCAL INFILE 'rows.csv' INTO TABLE `other`.g FIELDS TERMINATED BY ',' (name)", []byte("a\nb\n")) + require.NoError(t, err, "a qualified LOAD DATA into a narrow table was refused") + + conn, err := s.dbMgr.GetDatabaseConnection("other") + require.NoError(t, err) + var rows, distinct, wide int + require.NoError(t, conn.QueryRow("SELECT COUNT(*), COUNT(DISTINCT id), SUM(id > ?) FROM g", int32Max).Scan(&rows, &distinct, &wide)) + require.Equal(t, 8, rows, "rows landed outside other.g") + require.Equal(t, rows, distinct) + require.Zero(t, wide, "a qualified write got an id wider than the INT column") + var stray int + require.NoError(t, s.conn.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE name = 'g'").Scan(&stray)) + require.Zero(t, stray, "a qualified write created or used a table in the session's database") +} diff --git a/coordinator/narrow_autoinc_bench_test.go b/coordinator/narrow_autoinc_bench_test.go new file mode 100644 index 00000000..239e043a --- /dev/null +++ b/coordinator/narrow_autoinc_bench_test.go @@ -0,0 +1,33 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package coordinator_test + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// BenchmarkAutoIncrementInsert measures single-row autocommit INSERTs through +// the coordinator handler into an INT AUTO_INCREMENT table (narrow ids from +// claimed ranges) and into a BIGINT one (wide generated ids), in one session. +func BenchmarkAutoIncrementInsert(b *testing.B) { + for _, tc := range []struct{ name, ddl string }{ + {"narrow_int", "CREATE TABLE bench (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)"}, + {"wide_bigint", "CREATE TABLE bench (id BIGINT AUTO_INCREMENT PRIMARY KEY, v TEXT)"}, + } { + b.Run(tc.name, func(b *testing.B) { + s := setupNoopDML(b) + _, err := s.handler.HandleQuery(s.session, tc.ddl, nil) + require.NoError(b, err) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if _, err := s.handler.HandleQuery(s.session, "INSERT INTO bench (v) VALUES ('x')", nil); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/coordinator/narrow_autoinc_insert_test.go b/coordinator/narrow_autoinc_insert_test.go new file mode 100644 index 00000000..28fd101f --- /dev/null +++ b/coordinator/narrow_autoinc_insert_test.go @@ -0,0 +1,163 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package coordinator_test + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// int32Max is the ceiling of a signed MySQL INT. +const int32Max = 2147483647 + +// narrowIDs reads every id of a table, in id order. +func narrowIDs(t *testing.T, s *noopDMLSetup, query string) []int64 { + t.Helper() + rows, err := s.conn.Query(query) + require.NoError(t, err) + defer rows.Close() + var ids []int64 + for rows.Next() { + var id int64 + require.NoError(t, rows.Scan(&id)) + ids = append(ids, id) + } + require.NoError(t, rows.Err()) + return ids +} + +// TestNarrowAutoIncrementLastInsertIDAutocommit pins D5 on the autocommit +// path: a multi-row INSERT into an INT AUTO_INCREMENT table gets contiguous +// ids that fit 32 bits, and LAST_INSERT_ID is the FIRST of them, as MySQL +// reports it. +// +// Mutation: source LastInsertId from sqlite3_last_insert_rowid instead of the +// statement's first insert. It reports the third id and this fires. +func TestNarrowAutoIncrementLastInsertIDAutocommit(t *testing.T) { + s := setupNoopDML(t) + _, err := s.handler.HandleQuery(s.session, "CREATE TABLE g (group_id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)", nil) + require.NoError(t, err) + + res, err := s.handler.HandleQuery(s.session, "INSERT INTO g (name) VALUES ('a'), ('b'), ('c')", nil) + require.NoError(t, err) + require.Equal(t, int64(3), res.RowsAffected) + + ids := narrowIDs(t, s, "SELECT group_id FROM g ORDER BY group_id") + require.Len(t, ids, 3) + for i, id := range ids { + require.Positive(t, id) + require.LessOrEqual(t, id, int64(int32Max), "id %d does not fit the INT column", id) + require.Equal(t, ids[0]+int64(i), id, "a multi-row INSERT's ids must be contiguous: %v", ids) + } + require.Equal(t, ids[0], res.LastInsertId, "LAST_INSERT_ID must be the statement's first id") +} + +// TestNarrowAutoIncrementLLDAPTransactionShape pins D5 on the pinned +// explicit-transaction path, in the shape LLDAP boots with: one transaction +// inserting into two narrow tables, both of which need their first claim. +// It must complete, every id must fit 32 bits, and each statement must report +// its own first id - not the transaction's first insert. +// +// Mutation: report the transaction's first captured insert instead of the +// statement's. The groups INSERT then reports the users id and this fires. +func TestNarrowAutoIncrementLLDAPTransactionShape(t *testing.T) { + s := setupNoopDML(t) + for _, ddl := range []string{ + "CREATE TABLE users (user_id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)", + "CREATE TABLE `groups` (group_id INT AUTO_INCREMENT PRIMARY KEY, display_name TEXT)", + } { + _, err := s.handler.HandleQuery(s.session, ddl, nil) + require.NoError(t, err) + } + + _, err := s.handler.HandleQuery(s.session, "BEGIN", nil) + require.NoError(t, err) + usersRes, err := s.handler.HandleQuery(s.session, "INSERT INTO users (name) VALUES ('admin')", nil) + require.NoError(t, err) + groupsRes, err := s.handler.HandleQuery(s.session, + "INSERT INTO `groups` (display_name) VALUES (?), (?)", []interface{}{"lldap_admin", "lldap_password_manager"}) + require.NoError(t, err) + _, err = s.handler.HandleQuery(s.session, "COMMIT", nil) + require.NoError(t, err) + + userIDs := narrowIDs(t, s, "SELECT user_id FROM users") + groupIDs := narrowIDs(t, s, "SELECT group_id FROM `groups` ORDER BY group_id") + require.Len(t, userIDs, 1) + require.Len(t, groupIDs, 2) + for _, id := range append(userIDs, groupIDs...) { + require.LessOrEqual(t, id, int64(int32Max), "id %d does not fit the INT column", id) + } + require.Equal(t, userIDs[0], usersRes.LastInsertId) + require.Equal(t, groupIDs[0], groupsRes.LastInsertId, "the second statement must report its own first id") + require.Equal(t, groupIDs[0]+1, groupIDs[1], "a multi-row INSERT's ids must be contiguous") +} + +// TestNarrowAutoIncrementExhaustionReturnsDupEntry pins D4 end to end: a +// TINYINT AUTO_INCREMENT table filled to 127 refuses the next INSERT with +// ER_DUP_ENTRY on 127, which clients do not retry - never the retryable 1205. +// +// Mutation: map an exhausted claim to the lock-wait timeout. This fires on +// the error code. +func TestNarrowAutoIncrementExhaustionReturnsDupEntry(t *testing.T) { + s := setupNoopDML(t) + _, err := s.handler.HandleQuery(s.session, "CREATE TABLE tiny (id TINYINT AUTO_INCREMENT PRIMARY KEY, v TEXT)", nil) + require.NoError(t, err) + + for i := 1; i <= 127; i++ { + res, err := s.handler.HandleQuery(s.session, "INSERT INTO tiny (v) VALUES ('x')", nil) + require.NoError(t, err, "insert %d", i) + require.Equal(t, int64(i), res.LastInsertId, "a single writer must mint 1, 2, 3 with no gap") + } + + _, err = s.handler.HandleQuery(s.session, "INSERT INTO tiny (v) VALUES ('overflow')", nil) + require.Error(t, err) + require.Contains(t, err.Error(), "1062") + require.Contains(t, err.Error(), "Duplicate entry '127'") +} + +// TestNarrowPreparedInsertWithNullIDIsGenerated is F3 end to end: a prepared +// INSERT binding NULL (or 0) to a narrow id gets a generated id every time, +// as the literal NULL does. Before, SQLite assigned its own rowid (MAX+1), +// which the CDC admission refuses with 1235 whenever it lies at or below the +// cluster's base in no range of this node - here, after a restart, in the +// range the previous process claimed. 500 runs, no refusal, every id distinct +// and 32-bit, and each LAST_INSERT_ID is the row's own id. +// +// Mutation: do not pass the bound values to the parser +// (ParseOptions.BoundParams). SQLite assigns rowid 4, the admission refuses +// it, and "a prepared insert with a NULL id was refused" fires. +func TestNarrowPreparedInsertWithNullIDIsGenerated(t *testing.T) { + s := setupNoopDML(t) + _, err := s.handler.HandleQuery(s.session, "CREATE TABLE g (group_id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)", nil) + require.NoError(t, err) + // The first process claims a range and issues 1..3; the restarted one owns + // nothing below the base. + for i := 0; i < 3; i++ { + _, err = s.handler.HandleQuery(s.session, "INSERT INTO g (name) VALUES ('first')", nil) + require.NoError(t, err) + } + s.handler = s.restartHandler() + + const runs = 500 + seen := make(map[int64]bool, runs) + for i := 0; i < runs; i++ { + var bound interface{} + if i%2 == 1 { + bound = int64(0) + } + res, err := s.handler.HandleQuery(s.session, "INSERT INTO g (group_id, name) VALUES (?, ?)", []interface{}{bound, "n"}) + require.NoError(t, err, "a prepared insert with a NULL id was refused (run %d)", i) + id := res.LastInsertId + require.Positive(t, id, "a prepared insert with a NULL id reported no generated id") + require.LessOrEqual(t, id, int64(int32Max)) + require.False(t, seen[id], "id %d issued twice", id) + seen[id] = true + } + ids := narrowIDs(t, s, "SELECT group_id FROM g WHERE name = 'n' ORDER BY group_id") + require.Len(t, ids, runs) + for _, id := range ids { + require.True(t, seen[id], "stored id %d is not a LAST_INSERT_ID the client saw", id) + } +} diff --git a/coordinator/noop_dml_test.go b/coordinator/noop_dml_test.go index 8e12291f..ed977dfa 100644 --- a/coordinator/noop_dml_test.go +++ b/coordinator/noop_dml_test.go @@ -53,9 +53,13 @@ type noopDMLSetup struct { session *protocol.ConnectionSession replicator *countingReplicator conn *sql.DB + dbMgr *db.DatabaseManager + // restartHandler builds a fresh handler over the same node, as a + // restarted process would have: same data, no in-memory allocator state. + restartHandler func() *coordinator.CoordinatorHandler } -func setupNoopDML(t *testing.T) *noopDMLSetup { +func setupNoopDML(t testing.TB) *noopDMLSetup { t.Helper() tmpDir := t.TempDir() @@ -64,6 +68,9 @@ func setupNoopDML(t *testing.T) *noopDMLSetup { dbMgr, err := db.NewDatabaseManager(tmpDir, 1, clock) require.NoError(t, err) t.Cleanup(func() { dbMgr.Close() }) + // A single node releases the claim votes its new system database holds, + // as a standalone node's merge does. + require.NoError(t, dbMgr.MergeAutoIncBasesAndReleaseVotes(nil)) const dbName = "noopdml" require.NoError(t, dbMgr.CreateDatabase(dbName)) @@ -74,6 +81,8 @@ func setupNoopDML(t *testing.T) *noopDMLSetup { replicator := &countingReplicator{} nodeProvider := coordinator.NewMockNodeProvider([]uint64{1}) + // A claim participant counts the cluster as the claimant does. + dbMgr.SetClusterMembership(nodeProvider.GetTotalMembershipSize) writeCoord := coordinator.NewWriteCoordinator( 1, @@ -90,16 +99,19 @@ func setupNoopDML(t *testing.T) *noopDMLSetup { 10*time.Second, ) - handler := coordinator.NewCoordinatorHandler( - 1, - writeCoord, - readCoord, - clock, - dbMgr, - coordinator.NewDDLLockManager(30*time.Second), - schemaVersionMgr, - noopNodeRegistry{}, - ) + newHandler := func() *coordinator.CoordinatorHandler { + return coordinator.NewCoordinatorHandler( + 1, + writeCoord, + readCoord, + clock, + dbMgr, + coordinator.NewDDLLockManager(30*time.Second), + schemaVersionMgr, + noopNodeRegistry{}, + ) + } + handler := newHandler() // Real connections enable transpilation (protocol/server.go), and without it // BEGIN does not parse as transaction control, so a test meaning to exercise @@ -120,7 +132,7 @@ func setupNoopDML(t *testing.T) *noopDMLSetup { conn, err := dbMgr.GetDatabaseConnection(dbName) require.NoError(t, err) - return &noopDMLSetup{handler: handler, session: session, replicator: replicator, conn: conn} + return &noopDMLSetup{handler: handler, session: session, replicator: replicator, conn: conn, dbMgr: dbMgr, restartHandler: newHandler} } // TestNoopDMLSucceeds pins that a DML matching zero rows is a successful no-op. diff --git a/coordinator/write_coordinator.go b/coordinator/write_coordinator.go index 25765027..d3b0522a 100644 --- a/coordinator/write_coordinator.go +++ b/coordinator/write_coordinator.go @@ -29,6 +29,18 @@ type WriteCoordinator struct { localReplicator Replicator timeout time.Duration clock *hlc.Clock + // schemaVersion returns this node's schema version for a database. A + // claim carries it as RequiredSchemaVersion, so a participant that has + // not applied the DDL this node has declines the claim instead of voting + // with a base kept for an incarnation this node has already replaced. + // Nil stamps claims with 0, which no participant checks. + schemaVersion func(database string) (uint64, error) +} + +// SetSchemaVersionSource sets where claims read this node's schema version +// (WriteCoordinator.schemaVersion). Call it before the coordinator is used. +func (wc *WriteCoordinator) SetSchemaVersionSource(source func(database string) (uint64, error)) { + wc.schemaVersion = source } // Replicator sends replication requests to remote nodes @@ -107,15 +119,21 @@ type ReplicationResponse struct { // DDL SQLite cannot apply). Timeouts and storage failures leave it false so they // stay retryable missing ACKs rather than a final verdict. Rejected bool - // AutoIDStoredBase is this participant's own committed base for the table a - // rejected AUTO_INCREMENT range claim named, so the claimant can retry above - // it rather than spin. Zero on every response that is not such a rejection. + // AutoIDStoredBase is this participant's own base (the largest of its claim + // row's floors) for the table a rejected AUTO_INCREMENT range claim named, + // so the claimant can retry above it rather than spin. Zero on every + // response that is not such a rejection. AutoIDStoredBase uint64 // ErrorCode is the MySQL server error code the rejecting participant chose // for this refusal, or 0 when it named none. Error carries the message; // this carries the code, because a typed error cannot cross either the // local or the remote participant boundary. ErrorCode uint16 + // ClaimNotApplicable reports a COMMIT the participant refused because it + // could not apply the AUTO_INCREMENT claim the transaction carries + // (protocol.ErrAutoIncClaimNotApplicable). Set only by the local + // participant. + ClaimNotApplicable bool } // NewWriteCoordinator creates a new write coordinator for full database replication @@ -580,6 +598,19 @@ func (wc *WriteCoordinator) commitLocalAfterRemoteQuorum(ctx context.Context, re if localResp != nil { errMsg = localResp.Error } + if localErr == nil && localResp != nil && localResp.ClaimNotApplicable { + // Not a divergence: this node refused to apply an AUTO_INCREMENT + // claim the remote quorum committed, so it withholds its ACK and + // never issues from the range. The ids are burnt, which is safe. + log.Warn(). + Str("resp_error", errMsg). + Uint64("txn_id", txnID). + Msg("Local AUTO_INCREMENT claim apply refused after remote quorum committed it; its range is not used") + return nil, &PartialCommitError{ + IsLocal: true, + LocalError: fmt.Errorf("%w: %s", protocol.ErrAutoIncClaimNotApplicable, errMsg), + } + } log.Error(). Err(localErr). Str("resp_error", errMsg). diff --git a/coordinator/write_coordinator_commitlocal_test.go b/coordinator/write_coordinator_commitlocal_test.go index 6746f171..d6e32220 100644 --- a/coordinator/write_coordinator_commitlocal_test.go +++ b/coordinator/write_coordinator_commitlocal_test.go @@ -2,10 +2,12 @@ package coordinator import ( "context" + "errors" "testing" "time" "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" ) // TestCommitLocalAfterRemoteQuorum_SuccessfulLocalCommit tests the happy path @@ -137,6 +139,54 @@ func TestCommitLocalAfterRemoteQuorum_LocalCommitFailure_Bug(t *testing.T) { } } +// TestCommitLocalAfterRemoteQuorum_RefusedClaimApplyIsNotDivergence: the local +// participant refused to apply the AUTO_INCREMENT claim the remote quorum +// committed (db.ApplyClaims, protocol.ErrAutoIncClaimNotApplicable). That is +// the protocol withholding an ACK, not a node that diverged, so the error must +// carry the claim sentinel; any other local failure must not. +// +// Mutation: ignore ReplicationResponse.ClaimNotApplicable in +// commitLocalAfterRemoteQuorum. "a refused claim apply was reported as a +// failed local commit" fires. +func TestCommitLocalAfterRemoteQuorum_RefusedClaimApplyIsNotDivergence(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + resp *ReplicationResponse + refusal bool + }{ + {"claim not applicable", &ReplicationResponse{Error: "auto-increment claim apply failed", ClaimNotApplicable: true}, true}, + {"failed commit", CreateErrorResponse("disk full"), false}, + } { + t.Run(tc.name, func(t *testing.T) { + localReplicator := newMockReplicator() + wc := NewWriteCoordinator(1, newMockNodeProvider([]uint64{1, 2, 3}), nil, localReplicator, + 100*time.Millisecond, hlc.NewClock(1)) + txn := NewTxnBuilder(). + WithID(250). + WithNodeID(1). + WithDatabase("test"). + WithCDCStatement("users", map[string][]byte{"id": {1}}, map[string][]byte{"id": {2}}). + Build() + localReplicator.SetNodeCommitResponse(1, tc.resp) + + _, err := wc.commitLocalAfterRemoteQuorum(context.Background(), wc.buildCommitRequest(txn), txn.ID) + AssertPartialCommitError(t, err, true) + var pErr *PartialCommitError + if !errors.As(err, &pErr) { + t.Fatalf("expected PartialCommitError, got %T", err) + } + if got := errors.Is(pErr.LocalError, protocol.ErrAutoIncClaimNotApplicable); got != tc.refusal { + if tc.refusal { + t.Fatalf("a refused claim apply was reported as a failed local commit: %v", pErr.LocalError) + } + t.Fatalf("a failed local commit was reported as a refused claim apply: %v", pErr.LocalError) + } + }) + } +} + // TestCommitLocalAfterRemoteQuorum_LocalCommitNilResponse tests handling of nil response // from local replicator, which should also be treated as a failure func TestCommitLocalAfterRemoteQuorum_LocalCommitNilResponse(t *testing.T) { diff --git a/db/autoinc_claim.go b/db/autoinc_claim.go index cabe8f1d..abef149d 100644 --- a/db/autoinc_claim.go +++ b/db/autoinc_claim.go @@ -7,6 +7,7 @@ import ( "math" "os" "strings" + "sync/atomic" "time" "github.com/maxpert/marmot/common" @@ -56,15 +57,81 @@ const AutoIncClaimTable = common.AutoIncClaimTableName // addressed only by (db, tbl) and a rowid would be dead weight. The // composite primary key is what lets one system-database table serve every // user database at once. +// +// A row keeps three floors apart, because they differ in what a COMMIT may +// be refused for (ApplyClaims): +// - committed: the end of the last range this node applied. Only +// ApplyClaims advances it, and the per-node disjointness of ACKed ranges +// rests on it alone. +// - seed: ids this node's own rows hold - the DDL-time and restore-time +// MAX(id) seeds, the PREPARE-time backfill, and a rename's inheritance. +// - merged: bases peers reported - the vote-hold merge, the membership +// backstop, the sync command and a snapshot restore. +// +// A table's base, the value PREPARE votes against and every reader reports, +// is the largest of the three (autoIncBaseExpr). The column order is the one +// migrateAutoIncClaimTable leaves a table from before the split in. var autoIncClaimDDL = `CREATE TABLE IF NOT EXISTS ` + AutoIncClaimTable + ` ( db TEXT NOT NULL, tbl TEXT NOT NULL, - base INTEGER NOT NULL, + committed INTEGER NOT NULL, owner INTEGER NOT NULL, granted_at INTEGER NOT NULL, + seed INTEGER NOT NULL DEFAULT 0, + merged INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (db, tbl) ) WITHOUT ROWID` +// autoIncBaseExpr is a claim row's base: the largest of its three floors. +const autoIncBaseExpr = "MAX(committed, seed, merged)" + +// claimTableQuerier is what inspecting and migrating the claim table needs. +// *sql.DB and *sql.Tx both satisfy it. +type claimTableQuerier interface { + Exec(query string, args ...interface{}) (sql.Result, error) + QueryRow(query string, args ...interface{}) *sql.Row +} + +// autoIncClaimTableIsPreSplit reports whether the claim table in schema +// ("main", or an attached database's name) still has the single base column +// of a binary from before the split into committed, seed and merged. +func autoIncClaimTableIsPreSplit(q claimTableQuerier, schema string) (bool, error) { + var n int + if err := q.QueryRow("SELECT COUNT(*) FROM pragma_table_info(?, ?) WHERE name = 'base'", + AutoIncClaimTable, schema).Scan(&n); err != nil { + return false, fmt.Errorf("inspect %s.%s: %w", schema, AutoIncClaimTable, err) + } + return n > 0, nil +} + +// migrateAutoIncClaimTable splits a claim table from before the split: the +// base column becomes committed, and seed and merged start at that base. +// +// A pre-split base B was the largest of everything the three floors now keep +// apart, so B bounds each of them, and giving all three B keeps every check +// this node made before the upgrade exactly as strict: committed = B still +// covers every range the node applied, which is all the disjointness argument +// needs of it. The caller runs it inside the transaction that opens the +// table, so no crash leaves a half-migrated table. A split table is left as +// it is. +func migrateAutoIncClaimTable(q claimTableQuerier) error { + preSplit, err := autoIncClaimTableIsPreSplit(q, "main") + if err != nil || !preSplit { + return err + } + for _, stmt := range []string{ + "ALTER TABLE " + AutoIncClaimTable + " RENAME COLUMN base TO committed", + "ALTER TABLE " + AutoIncClaimTable + " ADD COLUMN seed INTEGER NOT NULL DEFAULT 0", + "ALTER TABLE " + AutoIncClaimTable + " ADD COLUMN merged INTEGER NOT NULL DEFAULT 0", + "UPDATE " + AutoIncClaimTable + " SET seed = committed, merged = committed", + } { + if _, err := q.Exec(stmt); err != nil { + return fmt.Errorf("split %s: %w", AutoIncClaimTable, err) + } + } + return nil +} + // AutoIncClaim is the payload a claim statement carries in its intent's // DataSnapshot. It is defined in protocol (protocol/autoinc_claim.go), which // coordinator also depends on, because db already imports coordinator (for @@ -75,25 +142,23 @@ type AutoIncClaim = protocol.AutoIncClaim // ErrAutoIncBaseAbsent reports that a table has no claim row. // // It is an error and never a zero, and the distinction is the protocol's whole -// safety argument. The natural implementation is "absent means 0 means yes", +// safety argument (the PREPARE handler backfills from the table itself rather +// than read absence as 0, see backfillAutoIncBase). The natural implementation is "absent means 0 means yes", // and an engineer will write that unless told not to: a cache that defaults to // 0 and accepts is precisely the thing that cannot cast the rejection that // would repair it. var ErrAutoIncBaseAbsent = errors.New("no auto-increment claim row for table") -// ErrAutoIncClaimNotApplicable reports a COMMIT whose claim this node cannot -// apply: the claim intent written at PREPARE is gone, or the stored base has -// moved past the claim's newBase since PREPARE. Either way this node did not -// hold the claim continuously from its vote to its commit, and ACKing would -// let it count toward a quorum for a range it may also have granted to -// another claimant. -var ErrAutoIncClaimNotApplicable = errors.New("auto-increment claim cannot be applied") +// ErrAutoIncClaimNotApplicable is protocol.ErrAutoIncClaimNotApplicable, +// which ApplyClaims returns for a COMMIT whose claim this node cannot apply. +var ErrAutoIncClaimNotApplicable = protocol.ErrAutoIncClaimNotApplicable // AutoIncClaimStore is the node's AUTO_INCREMENT base store: one row per // (database, table). The receiver is ALWAYS the SYSTEM database, never a user // database, and every method therefore names its database explicitly. type AutoIncClaimStore struct { - sys *ReplicatedDatabase + sys *ReplicatedDatabase + incarnation atomic.Pointer[AutoIncIncarnationListener] } // NewAutoIncClaimStore wraps the system database as a claim store. sys must @@ -102,7 +167,8 @@ func NewAutoIncClaimStore(sys *ReplicatedDatabase) *AutoIncClaimStore { return &AutoIncClaimStore{sys: sys} } -// ReadBase returns a table's committed base. +// ReadBase returns a table's base: the largest of its committed, seed and +// merged floors, which is what PREPARE votes against. // // It reads through the READ pool, not the write handle: writeDB is capped at a // single connection, so reading a base through it would put every claim behind @@ -115,7 +181,7 @@ func NewAutoIncClaimStore(sys *ReplicatedDatabase) *AutoIncClaimStore { func (s *AutoIncClaimStore) ReadBase(database, table string) (uint64, error) { var base int64 err := s.sys.GetReadDB().QueryRow( - "SELECT base FROM "+AutoIncClaimTable+" WHERE db = ? AND tbl = ?", database, table).Scan(&base) + "SELECT "+autoIncBaseExpr+" FROM "+AutoIncClaimTable+" WHERE db = ? AND tbl = ?", database, table).Scan(&base) switch { case err == sql.ErrNoRows: return 0, ErrAutoIncBaseAbsent @@ -128,7 +194,10 @@ func (s *AutoIncClaimStore) ReadBase(database, table string) (uint64, error) { return uint64(base), nil } -// Seed creates or RAISES a table's base at DDL time. Never lowers. +// Seed creates or RAISES a table's seed floor to floor, the largest id this +// node's own rows hold for it (or the declared floor). Never lowers. It runs +// at DDL time, when a restored database is reattached, and when PREPARE +// backfills an absent row. // // Tagging an existing table holding ids 1..1000 with an absent row would leave // the base at 0, so the first range would be [0,R) and the allocator would @@ -144,8 +213,8 @@ func (s *AutoIncClaimStore) Seed(database, table string, floor, owner uint64) er return fmt.Errorf("create %s: %w", AutoIncClaimTable, err) } _, err := writeDB.Exec( - "INSERT INTO "+AutoIncClaimTable+" (db, tbl, base, owner, granted_at) VALUES (?, ?, ?, ?, ?) "+ - "ON CONFLICT(db, tbl) DO UPDATE SET base = MAX(base, excluded.base), owner = excluded.owner, granted_at = excluded.granted_at", + "INSERT INTO "+AutoIncClaimTable+" (db, tbl, committed, seed, merged, owner, granted_at) VALUES (?, ?, 0, ?, 0, ?, ?) "+ + "ON CONFLICT(db, tbl) DO UPDATE SET seed = MAX(seed, excluded.seed), owner = excluded.owner, granted_at = excluded.granted_at", database, table, int64(floor), int64(owner), time.Now().UnixNano()) if err != nil { return fmt.Errorf("seed auto-increment base for %s.%s: %w", database, table, err) @@ -153,20 +222,116 @@ func (s *AutoIncClaimStore) Seed(database, table string, floor, owner uint64) er return nil } -// RaiseAutoIncBasesFrom raises every claim base in the system database file at -// incomingPath to at least the base the system database file at localPath -// holds for the same (database, table), and copies in any row only localPath -// has. It is run on a peer's system database before a restore installs it in -// place of this node's own. +// Inherit raises table's seed floor to at least the base of every table in +// from, creating its row if needed, and keeps the rows of from. It runs when +// one DDL statement created table while removing the tables in from (a +// RENAME): table may now hold their rows, and so ids granted under their +// names. Every range granted under a name ends at or below that name's base +// on a majority, so after Inherit every later grant for table lies above +// every id table can hold. Never lowers, and never deletes a row: a base is +// monotone per (database, table) name for the life of the cluster. +// +// It raises the seed floor, not the merged one: the rows came into this +// node's own table inside the DDL, exactly as the MAX(id) seed that runs +// after it records, so a claim on table PREPARED before the rename and +// committed after it is refused like one a DDL seed overtook. +func (s *AutoIncClaimStore) Inherit(database, table string, from []string, owner uint64) (err error) { + tx, err := s.sys.GetWriteDB().Begin() + if err != nil { + return fmt.Errorf("begin auto-increment base inheritance for %s.%s: %w", database, table, err) + } + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + now := time.Now().UnixNano() + for _, f := range from { + if _, err = tx.Exec("INSERT INTO "+AutoIncClaimTable+" (db, tbl, committed, seed, merged, owner, granted_at) "+ + "SELECT db, ?, 0, "+autoIncBaseExpr+", 0, ?, ? FROM "+AutoIncClaimTable+" WHERE db = ? AND tbl = ? "+ + "ON CONFLICT(db, tbl) DO UPDATE SET seed = excluded.seed, owner = excluded.owner, granted_at = excluded.granted_at "+ + "WHERE excluded.seed > seed", + table, int64(owner), now, database, f); err != nil { + return fmt.Errorf("raise auto-increment base of %s.%s to %s's: %w", database, table, f, err) + } + } + if err = tx.Commit(); err != nil { + return fmt.Errorf("commit auto-increment base inheritance for %s.%s: %w", database, table, err) + } + return nil +} + +// AutoIncIncarnationListener learns of every table incarnation a DDL +// statement ends on this node, before the new definitions become visible to +// queries: a table dropped, created, renamed from or to a name, or redefined +// (TableIncarnationEnded), and every table of a dropped database +// (DatabaseIncarnationEnded). The node's narrow allocator discards its +// in-memory ranges for them. Such a range was granted for the old +// incarnation; it is disjoint from every later grant under the same name, +// but the rows of a new incarnation may come from grants under another name +// (RENAME). +type AutoIncIncarnationListener interface { + TableIncarnationEnded(database, table string) + DatabaseIncarnationEnded(database string) +} + +// SetIncarnationListener registers l (AutoIncIncarnationListener). +func (s *AutoIncClaimStore) SetIncarnationListener(l AutoIncIncarnationListener) { + s.incarnation.Store(&l) +} + +// tableIncarnationEnded reports table to the registered listener, if any. +func (s *AutoIncClaimStore) tableIncarnationEnded(database, table string) { + if l := s.incarnation.Load(); l != nil { + (*l).TableIncarnationEnded(database, table) + } +} + +// databaseIncarnationEnded reports database to the registered listener, if +// any. +func (s *AutoIncClaimStore) databaseIncarnationEnded(database string) { + if l := s.incarnation.Load(); l != nil { + (*l).DatabaseIncarnationEnded(database) + } +} + +// RaiseAutoIncBasesFrom prepares the peer's system database file at +// incomingPath to become this node's: it keeps every floor the system +// database file at localPath holds for the same (database, table), and copies +// in any row only localPath has. It is run on a peer's system database before +// a restore installs it in place of this node's own. // // A restore must never lower a base this node committed. The peer may have // missed a claim this node was in the majority for; installing the peer's // lower base would let this node accept that range again, and majority -// intersection - the protocol's whole safety argument - would be gone. A -// missing localPath (a node with no prior state) leaves incomingPath as it is. +// intersection - the protocol's whole safety argument - would be gone. +// +// Where each floor lands (see autoIncClaimDDL): +// - the peer's committed and merged floors become merged: they record +// ranges other nodes applied, which is what a merge carries, and a merge +// never refuses a COMMIT; +// - the peer's seed stays seed: it counts ids the peer's rows hold, and the +// snapshot this file travels with makes those rows this node's own; +// - this node's own committed, seed and merged floors are kept, each as the +// larger of the two, so committed stays exactly this node's. +// +// A peer file from before the split holds one base B, which cannot be taken +// apart, so B becomes all three floors (migrateAutoIncClaimTable) and only its +// committed moves to merged: B, the peer's committed included, stays in seed. +// That is the conservative choice, and it costs liveness only: during a rolling +// upgrade, a claim this node holds PREPAREd across such a restore can have its +// COMMIT refused until the pending-transaction GC. +// +// The vote hold (AutoIncHoldTable) is this node's, never the peer's: the +// incoming file keeps the local file's hold. A missing localPath - a node with +// no claim history of its own, which may nonetheless have ACKed claims before +// it lost its data - adds nothing to the peer's floors and HOLDS this node's +// votes until it has merged bases from a majority. func RaiseAutoIncBasesFrom(incomingPath, localPath string) (err error) { + hasLocal := true if _, statErr := os.Stat(localPath); errors.Is(statErr, os.ErrNotExist) { - return nil + hasLocal = false } else if statErr != nil { return fmt.Errorf("stat local system database: %w", statErr) } @@ -187,10 +352,20 @@ func RaiseAutoIncBasesFrom(incomingPath, localPath string) (err error) { // ATTACH is per connection, so every statement below must run on one. conn.SetMaxOpenConns(1) - // A peer running a binary older than the claim table ships a system - // database without it; the local rows still have to land. - if _, err := conn.Exec(autoIncClaimDDL); err != nil { - return fmt.Errorf("create %s in incoming system database: %w", AutoIncClaimTable, err) + if err := adoptPeerClaimTable(conn); err != nil { + return err + } + if _, err := conn.Exec(autoIncHoldDDL); err != nil { + return fmt.Errorf("create %s in incoming system database: %w", AutoIncHoldTable, err) + } + if _, err := conn.Exec("DELETE FROM main." + AutoIncHoldTable); err != nil { + return fmt.Errorf("clear the peer's vote hold: %w", err) + } + if !hasLocal { + if _, err := conn.Exec(holdVotesSQL, time.Now().UnixNano()); err != nil { + return fmt.Errorf("hold auto-increment votes: %w", err) + } + return nil } if _, err := conn.Exec("ATTACH DATABASE ? AS local", localPath); err != nil { return fmt.Errorf("attach local system database: %w", err) @@ -200,14 +375,35 @@ func RaiseAutoIncBasesFrom(incomingPath, localPath string) (err error) { AutoIncClaimTable).Scan(&localHasTable); err != nil { return fmt.Errorf("inspect local system database: %w", err) } + var localHasHold int + if err := conn.QueryRow("SELECT COUNT(*) FROM local.sqlite_master WHERE type = 'table' AND name = ?", + AutoIncHoldTable).Scan(&localHasHold); err != nil { + return fmt.Errorf("inspect local system database: %w", err) + } + if localHasHold > 0 { + if _, err := conn.Exec("INSERT INTO main." + AutoIncHoldTable + " (id, since) SELECT id, since FROM local." + AutoIncHoldTable); err != nil { + return fmt.Errorf("keep this node's vote hold: %w", err) + } + } if localHasTable > 0 { + // The local file is only read: a binary from before the split wrote + // one base, which bounds all three floors (migrateAutoIncClaimTable). + localPreSplit, err := autoIncClaimTableIsPreSplit(conn, "local") + if err != nil { + return err + } + floors := "committed, seed, merged" + if localPreSplit { + floors = "base, base, base" + } // "WHERE true" disambiguates the upsert's ON CONFLICT from a join // constraint, as SQLite requires for INSERT ... SELECT. - if _, err := conn.Exec("INSERT INTO main." + AutoIncClaimTable + " (db, tbl, base, owner, granted_at) " + - "SELECT db, tbl, base, owner, granted_at FROM local." + AutoIncClaimTable + " WHERE true " + - "ON CONFLICT(db, tbl) DO UPDATE SET base = excluded.base, owner = excluded.owner, granted_at = excluded.granted_at " + - "WHERE excluded.base > base"); err != nil { - return fmt.Errorf("raise incoming auto-increment bases: %w", err) + if _, err := conn.Exec("INSERT INTO main." + AutoIncClaimTable + " (db, tbl, committed, seed, merged, owner, granted_at) " + + "SELECT db, tbl, " + floors + ", owner, granted_at FROM local." + AutoIncClaimTable + " WHERE true " + + "ON CONFLICT(db, tbl) DO UPDATE SET committed = MAX(committed, excluded.committed), " + + "seed = MAX(seed, excluded.seed), merged = MAX(merged, excluded.merged), " + + "owner = excluded.owner, granted_at = excluded.granted_at"); err != nil { + return fmt.Errorf("keep this node's auto-increment floors: %w", err) } } if _, err := conn.Exec("DETACH DATABASE local"); err != nil { @@ -219,6 +415,37 @@ func RaiseAutoIncBasesFrom(incomingPath, localPath string) (err error) { return nil } +// adoptPeerClaimTable readies the claim table of the peer's system database +// on conn to become this node's, in one transaction: it creates the table if +// the peer's binary predates it (the local rows still have to land), splits a +// table from before the split, and moves the peer's committed floor into +// merged. A pre-split table's base also stays in seed, as the split gives it +// to all three floors (RaiseAutoIncBasesFrom). +func adoptPeerClaimTable(conn *sql.DB) (err error) { + tx, err := conn.Begin() + if err != nil { + return fmt.Errorf("begin adopting the peer's %s: %w", AutoIncClaimTable, err) + } + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + if _, err = tx.Exec(autoIncClaimDDL); err != nil { + return fmt.Errorf("create %s in incoming system database: %w", AutoIncClaimTable, err) + } + if err = migrateAutoIncClaimTable(tx); err != nil { + return err + } + if _, err = tx.Exec("UPDATE " + AutoIncClaimTable + " SET merged = MAX(committed, merged), committed = 0"); err != nil { + return fmt.Errorf("record the peer's committed floors as merged: %w", err) + } + if err = tx.Commit(); err != nil { + return fmt.Errorf("commit adopting the peer's %s: %w", AutoIncClaimTable, err) + } + return nil +} + // AutoIncWidthMax returns the largest id the table's AUTO_INCREMENT column can // hold, derived from THIS node's own schema. // @@ -256,7 +483,7 @@ func (mdb *ReplicatedDatabase) AutoIncWidthMax(table string) (uint64, error) { return 0, fmt.Errorf("table %s auto-increment column %s is absent from its schema", table, autoIncCol) } -// ApplyClaims writes the committed base for every AUTO_INCREMENT range claim +// ApplyClaims writes the committed floor for every AUTO_INCREMENT range claim // prepared under txnID against database. It is called by the COMMIT handler, // on the user-database path, only for a COMMIT that carries the claim flag, // and before the transaction is committed. @@ -265,24 +492,47 @@ func (mdb *ReplicatedDatabase) AutoIncWidthMax(table string) (uint64, error) { // being read is against the user database, even though the write this method // performs lands in the system database. // -// The stored base becomes newBase + size. The claimant owns -// newBase+1 .. newBase+size, so newBase+size is both the -// last id handed out and the prevBase the next claimant proposes. That sum is -// the quantity the protocol advances monotonically. +// The committed floor becomes newBase + size (autoIncClaimDDL). The claimant +// owns newBase+1 .. newBase+size, so newBase+size is both the last id handed +// out and the prevBase the next claimant proposes. +// +// The write is conditional on the committed and seed floors both being at or +// below the claim's newBase, where its range starts, and ApplyClaims fails +// with ErrAutoIncClaimNotApplicable when either is not, or when the +// transaction holds no claim intent at all. The merged floor is not tested. +// +// Why the ranges one node ACKs are pairwise disjoint, from committed alone: +// 1. After the row is created (committed 0, or a pre-split base on upgrade), +// only this method writes committed, and a restore keeps this node's own +// committed (RaiseAutoIncBasesFrom). The write happens only where +// committed <= newBase and sets newBase+size > newBase, since size > 0: +// committed never decreases. +// 2. So once this node applied (n, n+s], committed >= n+s from then on. +// 3. The system database has one writer, so applies are serialised. A later +// apply of (n', n'+s'] needs committed <= n' at that moment, so +// n' >= n+s: it starts at or above the end of every earlier range. +// 4. Two overlapping ranges can therefore not both gather a commit majority: +// any two majorities share a node, and it would have applied both. +// +// That argument does not trust the claim key's row lock to have survived from +// PREPARE to COMMIT. The one way it does not - a stale-transaction abort +// racing a COMMIT that already read its intent - lives inside one call of +// this method, during which committed can only rise. While the lock does hold +// (a pending holder is never overwritten, and it persists across a restart), +// nothing else is applied for the table between this claim's PREPARE and its +// COMMIT, and PREPARE voted against the largest of all three floors. +// +// A merged floor is left out because it records ranges other nodes applied. +// A peer that committed this same claim first reports exactly newBase+size, +// so testing it would refuse the claim this node voted for; and a range +// overlapping this claim that another majority committed is excluded by 4. +// Merged does its work at PREPARE: a node that lost its claim state votes +// again only after its merged floor covers every claim it may have ACKed +// before (grpc autoIncMergeSafe), so every PREPARE it answers sees them. // -// The write is conditional on the stored base still being at or below the -// claim's newBase, where its range starts, and ApplyClaims fails with ErrAutoIncClaimNotApplicable -// when it is not, or when the transaction holds no claim intent at all. This -// is what makes a node's COMMIT ACKs safe without trusting the claim key's -// row lock to have survived from PREPARE to COMMIT: the system database has -// one writer, so the conditional writes a node applies are serialised, and -// each applies only if no range it already committed reaches past the new -// claim's newBase. The condition is on newBase rather than prevBase so that it -// holds on its own, not only because PREPARE admitted newBase == prevBase. The ranges one node ACKs are therefore pairwise disjoint, -// and two overlapping ranges cannot both gather a commit majority, because -// any two majorities share a node. The same condition keeps a base that a DDL -// seed raised between PREPARE and COMMIT from being lowered, and refuses a -// range that would sit below that raised floor. +// The seed floor counts ids this node's own rows hold. A seed raised between +// PREPARE and COMMIT - a DDL, a rename's inheritance, a reattach after +// restore - refuses a range that would sit below ids now in the table. // // Every claim in the transaction lands in ONE SQLite transaction against the // system database. A partially applied set would leave this node holding a @@ -296,9 +546,9 @@ func (mdb *ReplicatedDatabase) AutoIncWidthMax(table string) (uint64, error) { // rather than from the payload, so a claimant cannot misattribute a range to // another node. // -// The row is updated in place and never deleted: an absent row is a hard rejection at PREPARE, so any code that -// removed rows would create a window in which a live table re-mints ids it has -// already used. +// The row is updated in place and never deleted, by this or any DDL: a base +// is monotone per (database, table) name for the life of the cluster, and a +// removed row would let a name be granted a range it was granted before. func (s *AutoIncClaimStore) ApplyClaims(database string, txnID uint64, meta MetaStore) error { intents, err := meta.GetIntentsByTxn(txnID) if err != nil { @@ -330,8 +580,9 @@ func (s *AutoIncClaimStore) ApplyClaims(database string, txnID uint64, meta Meta return nil } -// applyClaimTx applies one claim intent inside tx, conditional on the stored -// base not having passed the claim's newBase (see ApplyClaims). +// applyClaimTx applies one claim intent inside tx, conditional on neither the +// committed nor the seed floor having passed the claim's newBase (see +// ApplyClaims). func applyClaimTx(tx *sql.Tx, database string, intent *WriteIntentRecord) error { claim, err := protocol.DecodeAutoIncClaim(intent.DataSnapshot) if err != nil { @@ -346,8 +597,10 @@ func applyClaimTx(tx *sql.Tx, database string, intent *WriteIntentRecord) error claim.Table, claim.Size, claim.NewBase) } res, err := tx.Exec( - "UPDATE "+AutoIncClaimTable+" SET base = ?, owner = ?, granted_at = ? WHERE db = ? AND tbl = ? AND base <= ?", - int64(claim.NewBase+claim.Size), int64(intent.NodeID), intent.TSWall, database, claim.Table, int64(claim.NewBase)) + "UPDATE "+AutoIncClaimTable+" SET committed = ?, owner = ?, granted_at = ? "+ + "WHERE db = ? AND tbl = ? AND committed <= ? AND seed <= ?", + int64(claim.NewBase+claim.Size), int64(intent.NodeID), intent.TSWall, database, claim.Table, + int64(claim.NewBase), int64(claim.NewBase)) if err != nil { return fmt.Errorf("write auto-increment base for %s.%s: %w", database, claim.Table, err) } @@ -356,7 +609,7 @@ func applyClaimTx(tx *sql.Tx, database string, intent *WriteIntentRecord) error return fmt.Errorf("write auto-increment base for %s.%s: %w", database, claim.Table, err) } if n != 1 { - return fmt.Errorf("%w: the stored base for %s.%s is absent or has passed the claim's newBase %d", + return fmt.Errorf("%w: the claim row for %s.%s is absent or its committed or seed floor has passed the claim's newBase %d", ErrAutoIncClaimNotApplicable, database, claim.Table, claim.NewBase) } return nil diff --git a/db/autoinc_claim_test.go b/db/autoinc_claim_test.go index d4b98192..400e0b32 100644 --- a/db/autoinc_claim_test.go +++ b/db/autoinc_claim_test.go @@ -121,7 +121,7 @@ func TestSeedAutoIncBaseOnlyRaises(t *testing.T) { // A later seed with a LOWER floor must not lower the base: re-applying a // replicated DDL on a node that has since committed claims would otherwise // wind the base backwards and remint over ids already handed out. - // Mutation: use excluded.base instead of MAX(base, excluded.base). + // Mutation: use excluded.seed instead of MAX(seed, excluded.seed). if err := store.Seed("testdb", "users", 5, 1); err != nil { t.Fatalf("Seed (lower): %v", err) } @@ -168,23 +168,25 @@ func TestClaimTableIsCDCSkipped(t *testing.T) { } entries := captureEntries(t, source, 7001, - "INSERT INTO "+AutoIncClaimTable+" (db, tbl, base, owner, granted_at) VALUES ('testdb', 'users', 64, 1, 0)") + "INSERT INTO "+AutoIncClaimTable+" (db, tbl, committed, owner, granted_at) VALUES ('testdb', 'users', 64, 1, 0)") if len(entries) != 0 { t.Errorf("writing the claim table produced %d CDC entries, want 0", len(entries)) } } -// TestDropDatabaseRemovesItsClaimRowsAndLeavesOthersIntact pins the DROP -// DATABASE side of the relocation: removing a database must take its claim -// rows with it (db/database_manager.go DropDatabase), but must not touch -// another database's rows in the same shared system-database table. This is -// the opposite of DROP TABLE (TestDroppedTableKeepsItsClaimRow, -// db/autoinc_commit_test.go), which deliberately leaves the row behind. +// TestDropDatabaseKeepsItsClaimRows pins that a claim base is monotone per +// (database, table) name for the life of the cluster: DROP DATABASE keeps its +// claim rows exactly as DROP TABLE does (TestDroppedTableKeepsItsClaimRow, +// db/autoinc_commit_test.go), and recreating the database and its table +// cannot lower them. A lowered base would grant a range some node still holds +// in memory from before the drop. // -// Mutation: scope the DELETE without a WHERE db = ? clause, or drop it -// entirely. Either the row survives its own database's removal, or a -// surviving database's row is deleted too, and the assertions below fire. -func TestDropDatabaseRemovesItsClaimRowsAndLeavesOthersIntact(t *testing.T) { +// The drop also ends the in-memory ranges of the database's tables on this +// node. +// +// Mutation: delete the database's claim rows in DropDatabase. The rows are +// gone and "DROP DATABASE deleted its claim row" fires. +func TestDropDatabaseKeepsItsClaimRows(t *testing.T) { _, dm, cleanup := setupTestReplicationEngine(t) defer cleanup() @@ -195,17 +197,23 @@ func TestDropDatabaseRemovesItsClaimRowsAndLeavesOthersIntact(t *testing.T) { require.NoError(t, store.Seed("dropme", "users", 1000, 1)) require.NoError(t, store.Seed("keepme", "users", 2000, 1)) + rec := &incarnationRecorder{} + dm.SetAutoIncIncarnationListener(rec) require.NoError(t, dm.DropDatabase("dropme")) + require.Equal(t, []string{"dropme"}, rec.databases, "DROP DATABASE did not end its tables' incarnations") + base, err := store.ReadBase("dropme", "users") + require.NoError(t, err, "DROP DATABASE deleted its claim row") + require.Equal(t, uint64(1000), base, "DROP DATABASE changed its claim base") - if _, err := store.ReadBase("dropme", "users"); !errors.Is(err, ErrAutoIncBaseAbsent) { - t.Errorf("dropme.users claim row survived DROP DATABASE: ReadBase returned %v, want ErrAutoIncBaseAbsent", err) - } + require.NoError(t, dm.CreateDatabase("dropme")) + require.NoError(t, store.Seed("dropme", "users", 0, 2)) + base, err = store.ReadBase("dropme", "users") + require.NoError(t, err) + require.Equal(t, uint64(1000), base, "recreating the database and its table lowered the claim base") - base, err := store.ReadBase("keepme", "users") - require.NoError(t, err, "DROP DATABASE of one database must not disturb another's claim rows") - if base != 2000 { - t.Errorf("keepme.users base = %d after dropping a different database, want 2000", base) - } + base, err = store.ReadBase("keepme", "users") + require.NoError(t, err) + require.Equal(t, uint64(2000), base, "DROP DATABASE of one database changed another's claim base") } // TestSystemDatabaseCommitsAreDurable: a claim COMMIT this node ACKs counts diff --git a/db/autoinc_commit_test.go b/db/autoinc_commit_test.go index 13a04f27..a9e57720 100644 --- a/db/autoinc_commit_test.go +++ b/db/autoinc_commit_test.go @@ -241,6 +241,107 @@ func TestDroppedTableKeepsItsClaimRow(t *testing.T) { require.Equal(t, uint64(1000), base) } +// applyDDLForTest runs one DDL statement through PREPARE and COMMIT, the +// replicated path every node applies DDL through. +func applyDDLForTest(t *testing.T, engine *ReplicationEngine, txnID uint64, table, ddl string) { + t.Helper() + stmt := protocol.Statement{Type: protocol.StatementDDL, Database: "testdb", TableName: table, SQL: ddl} + prep := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, NodeID: 1, StartTS: hlc.Timestamp{WallTime: int64(txnID)}, Database: "testdb", + Statements: []protocol.Statement{stmt}, + }) + require.True(t, prep.Success, "PREPARE of %q: %s", ddl, prep.Error) + res := engine.Commit(context.Background(), &CommitRequest{ + TxnID: txnID, Database: "testdb", Statements: []protocol.Statement{stmt}, + }) + require.True(t, res.Success, "COMMIT of %q: %s", ddl, res.Error) +} + +// incarnationRecorder records what a DDL apply reports to the node's +// AutoIncIncarnationListener. +type incarnationRecorder struct { + tables []string + databases []string +} + +func (r *incarnationRecorder) TableIncarnationEnded(database, table string) { + r.tables = append(r.tables, database+"."+table) +} + +func (r *incarnationRecorder) DatabaseIncarnationEnded(database string) { + r.databases = append(r.databases, database) +} + +// takeTables returns the table incarnations reported so far and resets them. +func (r *incarnationRecorder) takeTables() []string { + tables := r.tables + r.tables = nil + return tables +} + +// TestRenamedTableInheritsTheBasesOfTheNamesItReplaces holds the invariant +// that, with forget-on-incarnation-change, makes a node's stale in-memory +// ranges harmless: a base is monotone per name, and a table that takes +// another's place is raised to the base of the name its rows came from. +// +// Reviewer B's round trip: t has base 64, t RENAME TO t2, t2 grants up to +// 128, t2 RENAME TO t. Without the inheritance t2 would start from its +// backfill and t would stay at 64 below ids the table holds. Each rename ends +// the incarnations of both names, and the node's listener hears of both. +// +// Mutation: skip AutoIncClaimStore.Inherit in seedAutoIncBasesForDDL. t2's +// row is absent and "the rename did not raise t2" fires. +func TestRenamedTableInheritsTheBasesOfTheNamesItReplaces(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE t (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "t", 64, 1) + + rec := &incarnationRecorder{} + dm.SetAutoIncIncarnationListener(rec) + + applyDDLForTest(t, engine, 7410, "t", "ALTER TABLE t RENAME TO t2") + base, err := autoIncClaimStoreForTest(dm).ReadBase("testdb", "t2") + require.NoError(t, err, "the rename did not raise t2") + require.Equal(t, uint64(64), base, "the rename did not raise t2 to t's base") + require.Equal(t, uint64(64), readClaimBase(t, dm, "testdb", "t"), "the rename changed the source's row") + require.Equal(t, []string{"testdb.t", "testdb.t2"}, rec.takeTables(), "a rename did not end both names' incarnations") + + seedClaimBase(t, dm, "testdb", "t2", 128, 2) + applyDDLForTest(t, engine, 7411, "t2", "ALTER TABLE t2 RENAME TO t") + require.Equal(t, uint64(128), readClaimBase(t, dm, "testdb", "t"), "renaming back did not raise t above t2's grants") + require.Equal(t, uint64(128), readClaimBase(t, dm, "testdb", "t2"), "renaming back changed t2's row") + require.Equal(t, []string{"testdb.t", "testdb.t2"}, rec.takeTables(), "renaming back did not end both names' incarnations") +} + +// TestDDLEndsTheIncarnationsItChanges pins which DDL ends a name's +// incarnation: DROP TABLE, CREATE TABLE and any change to the table's +// definition do; an index does not. A recreated table keeps the base of the +// name it reuses (DROP TABLE keeps the row) and inherits nothing lower. +// +// Mutation: make endedIncarnations report only removed tables. The CREATE is +// not reported and "CREATE TABLE did not end t's incarnation" fires. +func TestDDLEndsTheIncarnationsItChanges(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE t (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "t", 127, 1) + + rec := &incarnationRecorder{} + dm.SetAutoIncIncarnationListener(rec) + + applyDDLForTest(t, engine, 7420, "t", "DROP TABLE t") + require.Equal(t, []string{"testdb.t"}, rec.takeTables(), "DROP TABLE did not end t's incarnation") + applyDDLForTest(t, engine, 7421, "t", "CREATE TABLE t (id INTEGER /*M:8a*/ PRIMARY KEY, v TEXT)") + require.Equal(t, []string{"testdb.t"}, rec.takeTables(), "CREATE TABLE did not end t's incarnation") + require.Equal(t, uint64(127), readClaimBase(t, dm, "testdb", "t"), "DROP + CREATE lowered the base") + + applyDDLForTest(t, engine, 7422, "t", "CREATE INDEX t_v ON t (v)") + require.Empty(t, rec.takeTables(), "an index ended the table's incarnation") + applyDDLForTest(t, engine, 7423, "t", "ALTER TABLE t ADD COLUMN w INTEGER") + require.Equal(t, []string{"testdb.t"}, rec.takeTables(), "redefining t did not end its incarnation") +} + // dmlLogInsertStatement builds a real client DML statement (not a claim) that // carries a CDC row image, exactly as PREPARE requires for the DML path // (createDMLIntent, db/replication_engine.go). Riding alongside a claim diff --git a/db/autoinc_floors_test.go b/db/autoinc_floors_test.go new file mode 100644 index 00000000..4e47aab3 --- /dev/null +++ b/db/autoinc_floors_test.go @@ -0,0 +1,208 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "database/sql" + "path/filepath" + "testing" + + "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" +) + +// readFloors reads a claim row's three floors from dm's system database. +func readFloors(t *testing.T, dm *DatabaseManager, database, table string) claimFloors { + t.Helper() + var f claimFloors + require.NoError(t, dm.GetSystemDatabase().GetReadDB().QueryRow( + "SELECT committed, seed, merged FROM "+AutoIncClaimTable+" WHERE db = ? AND tbl = ?", database, table). + Scan(&f.committed, &f.seed, &f.merged)) + return f +} + +// TestCommitAppliesAClaimAMergeRaiseOvertook: a peer of the same claim +// committed first, and this node's membership backstop (or the operator's +// sync command) pulled that peer's base, newBase+size, between this node's +// PREPARE and its COMMIT. The COMMIT must apply: refusing it left the claim's +// transaction prepared, holding the claim key until the pending-transaction +// GC, and this node could neither claim for the table nor vote on it. +// +// Mutation: test the merged floor in applyClaimTx's condition (or write merges +// to the committed floor). "a merge raise between PREPARE and COMMIT refused +// the claim" fires. +func TestCommitAppliesAClaimAMergeRaiseOvertook(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 1000, 1) + + const txnID = 9100 + require.True(t, prepareClaim(engine, t, txnID, 7, 1000, 64).Success) + require.NoError(t, dm.RaiseAutoIncBases([]AutoIncBase{{Database: "testdb", Table: "users", Base: 1064}})) + + res := commitClaim(engine, txnID) + require.True(t, res.Success, "a merge raise between PREPARE and COMMIT refused the claim: %s", res.Error) + require.Nil(t, mdb.GetTransactionManager().GetTransaction(txnID), "the claim's transaction is still pending after its COMMIT") + require.GreaterOrEqual(t, readClaimBase(t, dm, "testdb", "users"), uint64(1064)) + require.Equal(t, claimFloors{committed: 1064, seed: 1000, merged: 1064}, readFloors(t, dm, "testdb", "users")) + + next := prepareClaim(engine, t, txnID+1, 8, 1064, 64) + require.True(t, next.Success, "the next claim on the table could not PREPARE: %s", next.Error) + require.True(t, commitClaim(engine, txnID+1).Success) + require.Equal(t, uint64(1128), readClaimBase(t, dm, "testdb", "users")) +} + +// TestCommitAppliesAClaimAMergeRaisePassed: a merge may report a base past +// this claim's end, from later claims this node has not seen yet. The COMMIT +// still applies, and the base stays at the merged floor: the merged floor +// only ever refuses at PREPARE. +func TestCommitAppliesAClaimAMergeRaisePassed(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 1000, 1) + + const txnID = 9110 + require.True(t, prepareClaim(engine, t, txnID, 7, 1000, 64).Success) + require.NoError(t, dm.RaiseAutoIncBases([]AutoIncBase{{Database: "testdb", Table: "users", Base: 5000}})) + + res := commitClaim(engine, txnID) + require.True(t, res.Success, "a merge raise between PREPARE and COMMIT refused the claim: %s", res.Error) + require.Equal(t, uint64(5000), readClaimBase(t, dm, "testdb", "users")) + + stale := prepareClaim(engine, t, txnID+1, 8, 1064, 64) + require.False(t, stale.Success, "a claim below the merged floor was voted for") + require.Equal(t, uint64(5000), stale.AutoIDStoredBase) +} + +// TestClaimFloorsHaveOneWriterEach pins which operation writes which floor: +// a claim's apply the committed floor, the DDL/restore seed and a rename's +// inheritance the seed floor, and a merge the merged floor. Each is a raise. +func TestClaimFloorsHaveOneWriterEach(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + store := autoIncClaimStoreForTest(dm) + + require.NoError(t, store.Seed("testdb", "users", 100, 1)) + require.Equal(t, claimFloors{seed: 100}, readFloors(t, dm, "testdb", "users"), "Seed") + + require.NoError(t, dm.RaiseAutoIncBases([]AutoIncBase{{Database: "testdb", Table: "users", Base: 150}})) + require.Equal(t, claimFloors{seed: 100, merged: 150}, readFloors(t, dm, "testdb", "users"), "RaiseAutoIncBases") + + require.True(t, prepareClaim(engine, t, 9120, 7, 150, 10).Success) + require.True(t, commitClaim(engine, 9120).Success) + require.Equal(t, claimFloors{committed: 160, seed: 100, merged: 150}, readFloors(t, dm, "testdb", "users"), "ApplyClaims") + + require.NoError(t, store.Inherit("testdb", "renamed", []string{"users"}, 2)) + require.Equal(t, claimFloors{seed: 160}, readFloors(t, dm, "testdb", "renamed"), "Inherit") + + require.NoError(t, dm.MergeAutoIncBasesAndReleaseVotes([]AutoIncBase{{Database: "testdb", Table: "users", Base: 120}})) + require.NoError(t, store.Seed("testdb", "users", 50, 1)) + require.Equal(t, claimFloors{committed: 160, seed: 100, merged: 150}, readFloors(t, dm, "testdb", "users"), "a floor was lowered") +} + +// TestPreSplitSystemDatabaseIsSplitOnOpen opens a system database written by +// the binary at 541fc0f - one base column, no vote-hold table - and checks +// that the split keeps every check exactly as strict: all three floors start +// at the old base, the node is not held, PREPARE refuses below it, and a +// claim at it commits. +// +// Mutation: skip migrateAutoIncClaimTable in createAutoIncTables. The database +// still opens, as the claim DDL is IF NOT EXISTS, but the first floor read +// fails with "no such column: committed". +func TestPreSplitSystemDatabaseIsSplitOnOpen(t *testing.T) { + dir := t.TempDir() + clock := hlc.NewClock(1) + dm, err := NewDatabaseManager(dir, 1, clock) + require.NoError(t, err) + require.NoError(t, dm.MergeAutoIncBasesAndReleaseVotes(nil)) + markedTableDB(t, NewReplicationEngine(1, dm, clock), dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + require.NoError(t, dm.Close()) + + sys, err := sql.Open(SQLiteDriverName, filepath.Join(dir, SystemDatabaseName+".db")) + require.NoError(t, err) + for _, stmt := range []string{ + "DROP TABLE " + AutoIncClaimTable, + "DROP TABLE " + AutoIncHoldTable, + preSplitClaimDDL, + "INSERT INTO " + AutoIncClaimTable + " (db, tbl, base, owner, granted_at) VALUES ('testdb', 'users', 1000, 3, 1)", + } { + _, err = sys.Exec(stmt) + require.NoError(t, err, stmt) + } + require.NoError(t, sys.Close()) + + dm, err = NewDatabaseManager(dir, 1, clock) + require.NoError(t, err, "a 541fc0f system database did not open") + defer dm.Close() + dm.SetClusterMembership(func() int { return testClaimMembership }) + engine := NewReplicationEngine(1, dm, clock) + + require.Equal(t, claimFloors{committed: 1000, seed: 1000, merged: 1000}, readFloors(t, dm, "testdb", "users")) + held, err := dm.AutoIncVotesHeld() + require.NoError(t, err) + require.False(t, held, "a system database from a binary that granted claims was held") + + stale := prepareClaim(engine, t, 9130, 7, 500, 64) + require.False(t, stale.Success, "a claim below the pre-split base was voted for") + require.Equal(t, uint64(1000), stale.AutoIDStoredBase) + + require.True(t, prepareClaim(engine, t, 9131, 7, 1000, 64).Success) + require.True(t, commitClaim(engine, 9131).Success) + require.Equal(t, claimFloors{committed: 1064, seed: 1000, merged: 1000}, readFloors(t, dm, "testdb", "users")) +} + +// TestRaiseAutoIncBasesFromPlacesEachFloor pins where a restore puts each +// floor: the peer's committed and merged floors become merged, the peer's +// seed stays seed, and this node keeps its own three floors, each as the +// larger of the two - so committed is exactly this node's. +// +// Mutation: carry the peer's committed floor into committed. "the peer's +// committed floor became this node's" fires. +func TestRaiseAutoIncBasesFromPlacesEachFloor(t *testing.T) { + dir := t.TempDir() + incoming := filepath.Join(dir, "incoming.db") + local := filepath.Join(dir, "local.db") + newClaimFileFloors(t, incoming, true, map[string]claimFloors{ + "both": {committed: 100, seed: 300, merged: 200}, + "peer_only": {committed: 700, seed: 5}, + }) + newClaimFileFloors(t, local, true, map[string]claimFloors{ + "both": {committed: 50, seed: 10, merged: 20}, + "local_only": {committed: 40, seed: 30, merged: 60}, + }) + + require.NoError(t, RaiseAutoIncBasesFrom(incoming, local)) + + got := claimFloorsIn(t, incoming) + require.Equal(t, claimFloors{committed: 50, seed: 300, merged: 200}, got["both"], "the peer's committed floor became this node's") + require.Equal(t, claimFloors{seed: 5, merged: 700}, got["peer_only"], "the peer's committed floor became this node's") + require.Equal(t, claimFloors{committed: 40, seed: 30, merged: 60}, got["local_only"], "a row only this node held was changed") +} + +// TestRaiseAutoIncBasesFromPreSplitFiles: either side of a restore may come +// from the binary at 541fc0f. The peer's file is split before it is adopted; +// this node's file is read as it is, one base standing for all three floors, +// and is never written. +func TestRaiseAutoIncBasesFromPreSplitFiles(t *testing.T) { + dir := t.TempDir() + incoming := filepath.Join(dir, "incoming.db") + local := filepath.Join(dir, "local.db") + newPreSplitClaimFile(t, incoming, map[string]int64{"users": 500, "peer_only": 300}) + newPreSplitClaimFile(t, local, map[string]int64{"users": 900, "local_only": 50}) + + require.NoError(t, RaiseAutoIncBasesFrom(incoming, local)) + + got := claimFloorsIn(t, incoming) + require.Equal(t, claimFloors{committed: 900, seed: 900, merged: 900}, got["users"]) + require.Equal(t, claimFloors{seed: 300, merged: 300}, got["peer_only"]) + require.Equal(t, claimFloors{committed: 50, seed: 50, merged: 50}, got["local_only"]) + + var preSplit int + require.NoError(t, loneCopy(t, local).QueryRow( + "SELECT COUNT(*) FROM pragma_table_info(?) WHERE name = 'base'", AutoIncClaimTable).Scan(&preSplit)) + require.Equal(t, 1, preSplit, "the restore rewrote this node's own system database") +} diff --git a/db/autoinc_prepare_test.go b/db/autoinc_prepare_test.go index 81d21b8f..f8b9ade0 100644 --- a/db/autoinc_prepare_test.go +++ b/db/autoinc_prepare_test.go @@ -5,6 +5,7 @@ package db import ( "context" + "errors" "strings" "testing" @@ -13,12 +14,16 @@ import ( "github.com/stretchr/testify/require" ) +// testClaimMembership is the cluster size every claim in these tests counts, +// and the view markedTableDB installs on the participant. +const testClaimMembership = 1 + // claimStatement builds the wire shape a range claim takes: an existing DML // statement type, a non-empty intent key, the payload, and no row image. func claimStatement(t *testing.T, database, table string, prevBase, newBase, size uint64) protocol.Statement { t.Helper() payload, err := protocol.EncodeAutoIncClaim(AutoIncClaim{ - Table: table, PrevBase: prevBase, NewBase: newBase, Size: size, + Table: table, PrevBase: prevBase, NewBase: newBase, Size: size, Membership: testClaimMembership, }) require.NoError(t, err) return protocol.Statement{ @@ -31,9 +36,11 @@ func claimStatement(t *testing.T, database, table string, prevBase, newBase, siz } } -// markedTableDB sets up a database holding one marked AUTO_INCREMENT table. +// markedTableDB sets up a database holding one marked AUTO_INCREMENT table, +// on a participant whose view of the cluster is testClaimMembership nodes. func markedTableDB(t *testing.T, engine *ReplicationEngine, dm *DatabaseManager, ddl string) *ReplicatedDatabase { t.Helper() + dm.SetClusterMembership(func() int { return testClaimMembership }) require.NoError(t, dm.CreateDatabase("testdb")) mdb, err := dm.GetDatabase("testdb") require.NoError(t, err) @@ -69,35 +76,126 @@ func readClaimBase(t *testing.T, dm *DatabaseManager, database, table string) ui return base } -// TestPrepareClaimRejectsAbsentBase is the fail-closed clause, and it is the one -// an implementer gets wrong: the natural implementation is "absent means 0 -// means yes", and a cache that defaults to 0 and accepts is precisely the thing -// that cannot cast the rejection that would repair it. +// claimRequest wraps one claim statement in a PREPARE request. +func claimRequest(txnID uint64, stmt protocol.Statement) *PrepareRequest { + return &PrepareRequest{ + TxnID: txnID, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: int64(txnID)}, + Database: "testdb", + Statements: []protocol.Statement{stmt}, + } +} + +// TestPrepareClaimBackfillsAnAbsentBaseFromTheTable pins the absent-row rule. +// "Absent means 0 means yes" is the mistake the protocol exists to prevent: a +// table whose ids reach 50 must not grant a range starting at 0. A node whose +// votes are not held backfills the base the DDL-time seed would have written +// - MAX(id) - and votes on that. // // Mutation: treat ErrAutoIncBaseAbsent as base 0 and fall through to the // comparison. The claim is accepted and this fires. -func TestPrepareClaimRejectsAbsentBase(t *testing.T) { +func TestPrepareClaimBackfillsAnAbsentBaseFromTheTable(t *testing.T) { engine, dm, cleanup := setupTestReplicationEngine(t) defer cleanup() - markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + mdb := markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + for id := 1; id <= 50; id++ { + _, err := mdb.GetWriteDB().Exec("INSERT INTO users (id, v) VALUES (?, 'x')", id) + require.NoError(t, err) + } // No claim row has ever been written for this table. - result := engine.Prepare(context.Background(), &PrepareRequest{ - TxnID: 5001, - NodeID: 1, - StartTS: hlc.Timestamp{WallTime: 1}, - Database: "testdb", - Statements: []protocol.Statement{claimStatement(t, "testdb", "users", 0, 0, 64)}, - }) + result := engine.Prepare(context.Background(), claimRequest(5001, claimStatement(t, "testdb", "users", 0, 0, 64))) + if result.Success { + t.Fatal("a participant with no stored base ACKed a range starting at 0 over ids up to 50; absent must never read as 0") + } + if !result.Rejected || result.AutoIDStoredBase != 50 { + t.Fatalf("rejection = {rejected:%v base:%d}, want a rejection carrying the backfilled base 50", result.Rejected, result.AutoIDStoredBase) + } + if base := readClaimBase(t, dm, "testdb", "users"); base != 50 { + t.Fatalf("stored base after backfill = %d, want 50", base) + } + result = engine.Prepare(context.Background(), claimRequest(5002, claimStatement(t, "testdb", "users", 50, 50, 64))) + require.True(t, result.Success, "a claim above the backfilled base must be accepted: %s", result.Error) +} + +// TestPrepareClaimRefusesWhenThereIsNothingToBackfill pins that the backfill +// applies only to a narrow auto-increment column: anything else stays absent +// and refused. +// +// Mutation: backfill every absent row with 0. The claim is accepted and this +// fires. +func TestPrepareClaimRefusesWhenThereIsNothingToBackfill(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + + result := engine.Prepare(context.Background(), claimRequest(5001, claimStatement(t, "testdb", "ghost", 0, 0, 64))) + if result.Success { + t.Fatal("a claim for a table this node does not have was ACKed") + } + if !result.Rejected || !strings.Contains(result.Error, "ghost") { + t.Fatalf("refusal = {rejected:%v error:%q}, want a rejection naming the table", result.Rejected, result.Error) + } + if _, err := autoIncClaimStoreForTest(dm).ReadBase("testdb", "ghost"); !errors.Is(err, ErrAutoIncBaseAbsent) { + t.Fatalf("ReadBase after the refusal = %v, want the row still absent", err) + } +} + +// TestPrepareClaimDeclinesWhileVotesAreHeld pins the vote hold: a node whose +// bases may be missing claims it or a majority granted declines every claim, +// without a verdict and without backfilling, until it has merged bases. +// +// Mutation: ignore the hold in prepareAutoIncClaim. The claim is accepted +// and this fires. +func TestPrepareClaimDeclinesWhileVotesAreHeld(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 0, 1) + _, err := dm.GetSystemDatabase().GetWriteDB().Exec(holdVotesSQL, 1) + require.NoError(t, err) + + result := engine.Prepare(context.Background(), claimRequest(5001, claimStatement(t, "testdb", "users", 0, 0, 64))) if result.Success { - t.Fatal("a participant with no stored base ACKed a range claim; absent must reject") + t.Fatal("a node whose claim votes are held ACKed a claim") } - if !result.Rejected { - t.Error("the refusal was not marked Rejected, so the coordinator would retry it as a timeout") + if result.Rejected { + t.Fatal("a held node cast a verdict; it must only decline, so the claim is decided by the rest of the cluster") } - if !strings.Contains(result.Error, "users") { - t.Errorf("rejection %q does not name the table", result.Error) + + require.NoError(t, dm.MergeAutoIncBasesAndReleaseVotes(nil)) + result = engine.Prepare(context.Background(), claimRequest(5002, claimStatement(t, "testdb", "users", 0, 0, 64))) + require.True(t, result.Success, "after the merge released the hold the claim must be accepted: %s", result.Error) +} + +// TestPrepareClaimRejectsAnotherMembershipCount pins the membership check: a +// claim is voted on only by a node that counts the cluster the way the +// claimant did, since majorities of two memberships need not intersect. A +// claimant that predates the field sends 0 and is refused, not read as a size. +// +// Mutation: drop the membership comparison. The mismatched claim is accepted +// and this fires. +func TestPrepareClaimRejectsAnotherMembershipCount(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 30, 1) + + for i, membership := range []uint32{testClaimMembership + 2, 0} { + stmt := claimStatement(t, "testdb", "users", 30, 30, 64) + payload, err := protocol.EncodeAutoIncClaim(AutoIncClaim{Table: "users", PrevBase: 30, NewBase: 30, Size: 64, Membership: membership}) + require.NoError(t, err) + stmt.AutoIDClaimPayload = payload + + result := engine.Prepare(context.Background(), claimRequest(uint64(5001+i), stmt)) + if result.Success { + t.Fatalf("a claim counting %d members was ACKed by a node counting %d", membership, testClaimMembership) + } + if !result.Rejected || result.AutoIDStoredBase != 30 { + t.Fatalf("membership %d: refusal = {rejected:%v base:%d}, want a rejection carrying base 30", membership, result.Rejected, result.AutoIDStoredBase) + } } } diff --git a/db/autoinc_restore_merge_test.go b/db/autoinc_restore_merge_test.go index a6dfb99d..1f6ccd9f 100644 --- a/db/autoinc_restore_merge_test.go +++ b/db/autoinc_restore_merge_test.go @@ -12,21 +12,58 @@ import ( "github.com/stretchr/testify/require" ) +// claimFloors is one claim row's three floors (autoIncClaimDDL). +type claimFloors struct { + committed, seed, merged int64 +} + // newClaimFile creates a system-database file holding the given claim rows -// for database "d", in WAL mode as the system database runs. +// for database "d", each base as the row's committed floor, in WAL mode as +// the system database runs. func newClaimFile(t *testing.T, path string, withTable bool, bases map[string]int64) { t.Helper() - conn, err := sql.Open(SQLiteDriverName, path) - require.NoError(t, err) + rows := make(map[string]claimFloors, len(bases)) + for tbl, base := range bases { + rows[tbl] = claimFloors{committed: base} + } + newClaimFileFloors(t, path, withTable, rows) +} + +// newClaimFileFloors is newClaimFile with every floor of every row given. +func newClaimFileFloors(t *testing.T, path string, withTable bool, rows map[string]claimFloors) { + t.Helper() + conn := newSystemFile(t, path) defer func() { require.NoError(t, conn.Close()) }() - _, err = conn.Exec("PRAGMA journal_mode=WAL") - require.NoError(t, err) - _, err = conn.Exec("CREATE TABLE registry (name TEXT)") - require.NoError(t, err) if !withTable { return } - _, err = conn.Exec(autoIncClaimDDL) + _, err := conn.Exec(autoIncClaimDDL) + require.NoError(t, err) + for tbl, f := range rows { + _, err = conn.Exec("INSERT INTO "+AutoIncClaimTable+" (db, tbl, committed, seed, merged, owner, granted_at) "+ + "VALUES ('d', ?, ?, ?, ?, 1, 1)", tbl, f.committed, f.seed, f.merged) + require.NoError(t, err) + } +} + +// preSplitClaimDDL is the claim table as the binary at 541fc0f created it, +// before the base was split into committed, seed and merged floors. +const preSplitClaimDDL = `CREATE TABLE IF NOT EXISTS ` + AutoIncClaimTable + ` ( + db TEXT NOT NULL, + tbl TEXT NOT NULL, + base INTEGER NOT NULL, + owner INTEGER NOT NULL, + granted_at INTEGER NOT NULL, + PRIMARY KEY (db, tbl) +) WITHOUT ROWID` + +// newPreSplitClaimFile creates a system-database file whose claim table has +// the 541fc0f shape, holding the given bases for database "d". +func newPreSplitClaimFile(t *testing.T, path string, bases map[string]int64) { + t.Helper() + conn := newSystemFile(t, path) + defer func() { require.NoError(t, conn.Close()) }() + _, err := conn.Exec(preSplitClaimDDL) require.NoError(t, err) for tbl, base := range bases { _, err = conn.Exec("INSERT INTO "+AutoIncClaimTable+" (db, tbl, base, owner, granted_at) VALUES ('d', ?, ?, 1, 1)", tbl, base) @@ -34,30 +71,60 @@ func newClaimFile(t *testing.T, path string, withTable bool, bases map[string]in } } -// claimBasesIn reads every claim base for database "d" from a lone .db file, -// copied away from any -wal/-shm beside it, as the restorer moves it. -func claimBasesIn(t *testing.T, path string) map[string]int64 { +// newSystemFile creates a system-database file in WAL mode, as the system +// database runs, with a table beside the claim table. +func newSystemFile(t *testing.T, path string) *sql.DB { + t.Helper() + conn, err := sql.Open(SQLiteDriverName, path) + require.NoError(t, err) + _, err = conn.Exec("PRAGMA journal_mode=WAL") + require.NoError(t, err) + _, err = conn.Exec("CREATE TABLE registry (name TEXT)") + require.NoError(t, err) + return conn +} + +// loneCopy opens a copy of the .db file at path, away from any -wal/-shm +// beside it, as the restorer moves it. +func loneCopy(t *testing.T, path string) *sql.DB { t.Helper() data, err := os.ReadFile(path) require.NoError(t, err) lone := filepath.Join(t.TempDir(), "lone.db") require.NoError(t, os.WriteFile(lone, data, 0o644)) - conn, err := sql.Open(SQLiteDriverName, lone) require.NoError(t, err) - defer func() { require.NoError(t, conn.Close()) }() - rows, err := conn.Query("SELECT tbl, base FROM " + AutoIncClaimTable + " WHERE db = 'd'") + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + return conn +} + +// claimBasesIn reads every claim base for database "d" from a lone copy of +// the .db file at path. +func claimBasesIn(t *testing.T, path string) map[string]int64 { + t.Helper() + bases := map[string]int64{} + for tbl, f := range claimFloorsIn(t, path) { + bases[tbl] = max(f.committed, f.seed, f.merged) + } + return bases +} + +// claimFloorsIn reads every claim row's floors for database "d" from a lone +// copy of the .db file at path. +func claimFloorsIn(t *testing.T, path string) map[string]claimFloors { + t.Helper() + rows, err := loneCopy(t, path).Query("SELECT tbl, committed, seed, merged FROM " + AutoIncClaimTable + " WHERE db = 'd'") require.NoError(t, err) defer rows.Close() - bases := map[string]int64{} + floors := map[string]claimFloors{} for rows.Next() { var tbl string - var base int64 - require.NoError(t, rows.Scan(&tbl, &base)) - bases[tbl] = base + var f claimFloors + require.NoError(t, rows.Scan(&tbl, &f.committed, &f.seed, &f.merged)) + floors[tbl] = f } require.NoError(t, rows.Err()) - return bases + return floors } // TestRaiseAutoIncBasesFromNeverLowersABase: a restore installs a peer's system @@ -65,9 +132,9 @@ func claimBasesIn(t *testing.T, path string) map[string]int64 { // missed a claim this node committed as part of a majority, and a lowered base // would let this node accept that range again. // -// Mutation: drop "WHERE excluded.base > base" from the merge's upsert. The -// local copy then overwrites a higher incoming base and "a restore lowered a -// base" fires. +// Mutation: set merged to excluded.merged instead of the larger of the two in +// the merge's upsert. The local copy then overwrites a higher incoming base +// and "a restore lowered a base" fires. func TestRaiseAutoIncBasesFromNeverLowersABase(t *testing.T) { dir := t.TempDir() incoming := filepath.Join(dir, "incoming.db") @@ -162,3 +229,71 @@ func pragmaInt(t *testing.T, conn *sqlite3.SQLiteConn, pragma string) int64 { require.True(t, ok, "PRAGMA %s returned %T", pragma, dest[0]) return v } + +// holdIn reports whether a lone copy of the .db file at path carries the +// AUTO_INCREMENT vote hold. +func holdIn(t *testing.T, path string) bool { + t.Helper() + conn := loneCopy(t, path) + var tables, rows int + require.NoError(t, conn.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE name = ?", AutoIncHoldTable).Scan(&tables)) + if tables == 0 { + return false + } + require.NoError(t, conn.QueryRow("SELECT COUNT(*) FROM "+AutoIncHoldTable).Scan(&rows)) + return rows > 0 +} + +// placeHold writes the vote hold into the system-database file at path. +func placeHold(t *testing.T, path string) { + t.Helper() + conn, err := sql.Open(SQLiteDriverName, path) + require.NoError(t, err) + defer func() { require.NoError(t, conn.Close()) }() + _, err = conn.Exec(autoIncHoldDDL) + require.NoError(t, err) + _, err = conn.Exec(holdVotesSQL, 1) + require.NoError(t, err) +} + +// TestRaiseAutoIncBasesFromNoLocalStateHoldsVotes: a node installing a peer's +// system database with none of its own may have ACKed claims whose rows the +// peer lacks (a node rebuilt from one lagging peer). It must not vote until it +// has merged bases from a majority. +// +// Mutation: skip the hold when there is no local file. "a node rebuilt from +// one peer's system database may vote at once" fires. +func TestRaiseAutoIncBasesFromNoLocalStateHoldsVotes(t *testing.T) { + dir := t.TempDir() + incoming := filepath.Join(dir, "incoming.db") + newClaimFile(t, incoming, true, map[string]int64{"users": 7}) + + require.NoError(t, RaiseAutoIncBasesFrom(incoming, filepath.Join(dir, "absent.db"))) + require.True(t, holdIn(t, incoming), "a node rebuilt from one peer's system database may vote at once") +} + +// TestRaiseAutoIncBasesFromKeepsThisNodesHold: the hold belongs to the node, +// not to the file a peer sent. A peer's hold must not travel to this node, and +// this node's own hold must survive the install. +// +// Mutation: skip clearing the incoming hold, or skip copying the local one. +// One of the two assertions fires. +func TestRaiseAutoIncBasesFromKeepsThisNodesHold(t *testing.T) { + dir := t.TempDir() + + incoming := filepath.Join(dir, "held-peer.db") + local := filepath.Join(dir, "local.db") + newClaimFile(t, incoming, true, map[string]int64{"users": 7}) + placeHold(t, incoming) + newClaimFile(t, local, true, map[string]int64{"users": 9}) + require.NoError(t, RaiseAutoIncBasesFrom(incoming, local)) + require.False(t, holdIn(t, incoming), "a peer's vote hold travelled to this node through its snapshot") + + incoming = filepath.Join(dir, "peer.db") + heldLocal := filepath.Join(dir, "held-local.db") + newClaimFile(t, incoming, true, map[string]int64{"users": 7}) + newClaimFile(t, heldLocal, true, map[string]int64{"users": 9}) + placeHold(t, heldLocal) + require.NoError(t, RaiseAutoIncBasesFrom(incoming, heldLocal)) + require.True(t, holdIn(t, incoming), "this node's vote hold was lost when a peer's system database was installed") +} diff --git a/db/autoinc_seed.go b/db/autoinc_seed.go index 0892891a..923035c0 100644 --- a/db/autoinc_seed.go +++ b/db/autoinc_seed.go @@ -1,8 +1,10 @@ package db import ( + "context" "database/sql" "fmt" + "sort" "github.com/maxpert/marmot/protocol/query/transform/intmarker" ) @@ -44,9 +46,14 @@ func autoIncMarkedColumn(q rowQuerier, table string) (col string, attrs intmarke } // autoIncSeedFloor computes the base floor a table's DDL-declared -// AUTO_INCREMENT column implies right now: max(MAX() over the table's -// existing rows, the marker's own declared floor). ok is false when the table -// carries no explicitly declared AUTO_INCREMENT marker. +// AUTO_INCREMENT column implies right now: max(the marker's own declared +// floor, the largest existing id the column's width can hold). ok is false +// when the table carries no explicitly declared AUTO_INCREMENT marker. +// +// Ids above the width are left out. The allocator can never issue one, so no +// base has to clear them, and counting them would put the base past the +// ceiling and make every later insert fail as exhausted - the table the width +// check exempts (checkAutoIncWidthCeilings) would be unusable instead. func autoIncSeedFloor(q rowQuerier, table string) (floor uint64, ok bool, err error) { col, attrs, marked, err := autoIncMarkedColumn(q, table) if err != nil || !marked { @@ -54,8 +61,8 @@ func autoIncSeedFloor(q rowQuerier, table string) (floor uint64, ok bool, err er } var max sql.NullInt64 - query := fmt.Sprintf("SELECT MAX(%s) FROM %s", quoteIdent(col), quoteIdent(table)) - if scanErr := q.QueryRow(query).Scan(&max); scanErr != nil { + query := fmt.Sprintf("SELECT MAX(%s) FROM %s WHERE %s <= ?", quoteIdent(col), quoteIdent(table), quoteIdent(col)) + if scanErr := q.QueryRow(query, int64(attrs.WidthMax())).Scan(&max); scanErr != nil { return 0, false, fmt.Errorf("compute existing max for %s.%s: %w", table, col, scanErr) } @@ -69,6 +76,90 @@ func autoIncSeedFloor(q rowQuerier, table string) (floor uint64, ok bool, err er return floor, true, nil } +// schemaQuerier is the read surface tableDefinitions needs: *sql.DB, +// *sql.Conn and *sql.Tx all satisfy it. +type schemaQuerier interface { + QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) +} + +// tableDefinitions returns every table in q's schema, by name, with the +// CREATE statement sqlite_master holds for it. +func tableDefinitions(ctx context.Context, q schemaQuerier) (map[string]string, error) { + rows, err := q.QueryContext(ctx, "SELECT name, sql FROM sqlite_master WHERE type = 'table'") + if err != nil { + return nil, fmt.Errorf("list tables: %w", err) + } + defer rows.Close() + + defs := make(map[string]string) + for rows.Next() { + var name string + var def sql.NullString + if err := rows.Scan(&name, &def); err != nil { + return nil, fmt.Errorf("list tables: %w", err) + } + defs[name] = def.String + } + return defs, rows.Err() +} + +// endedIncarnations returns, sorted, every table name whose incarnation one DDL +// statement ended: a table it removed, created or redefined. A redefinition +// includes re-declaring the AUTO_INCREMENT column's width; any other change +// to the table's definition ends the incarnation too, which costs the +// unissued part of a range and nothing else. +func endedIncarnations(before, after map[string]string) []string { + var changed []string + for name, def := range before { + if newDef, ok := after[name]; !ok || newDef != def { + changed = append(changed, name) + } + } + for name := range after { + if _, ok := before[name]; !ok { + changed = append(changed, name) + } + } + sort.Strings(changed) + return changed +} + +// autoIncInheritance names a table that one DDL statement brought into being +// while it removed others: a RENAME, or any statement with the same effect. +// The table may hold the removed tables' rows, and with them ids granted +// under their names, so its claim base must reach theirs (AutoIncClaimStore.Inherit). +type autoIncInheritance struct { + table string + from []string + owner uint64 +} + +// inheritedTables compares a database's tables before and after one DDL +// statement. When the statement removed tables, every table it created is +// treated as having inherited their rows. It is conservative on purpose: a +// statement that dropped one table and created an unrelated one only raises +// the new table's base, which burns ids but can never reissue one, while a +// missed rename could. +func inheritedTables(before, after map[string]string, owner uint64) []autoIncInheritance { + var removed []string + for name := range before { + if _, ok := after[name]; !ok { + removed = append(removed, name) + } + } + if len(removed) == 0 { + return nil + } + sort.Strings(removed) + var inherited []autoIncInheritance + for name := range after { + if _, ok := before[name]; !ok { + inherited = append(inherited, autoIncInheritance{table: name, from: removed, owner: owner}) + } + } + return inherited +} + // ddlTableOwner names one table a DDL intent in the current transaction // touched, paired with the node that authored that intent. The owner comes // from the intent record's own NodeID - identical on every node applying the @@ -80,13 +171,15 @@ type ddlTableOwner struct { owner uint64 } -// seedAutoIncBasesForDDL seeds __marmot__autoinc, in the system database, for -// every table named by a DDL intent in this transaction that carries an -// explicitly declared AUTO_INCREMENT marker. It is a -// no-op for a table with no such marker. +// seedAutoIncBasesForDDL first raises every table in inherited to the bases of +// the tables it took the place of (AutoIncClaimStore.Inherit), then seeds +// __marmot__autoinc, in the system database, for every table named by a DDL +// intent in this transaction that carries an explicitly declared +// AUTO_INCREMENT marker. Both only ever raise a base. Seeding is a no-op for a +// table with no such marker. // // The seed floor is derived from THIS node's own sqlite_master and MAX(id) on -// the USER database (tm.db), read after the DDL has executed - not by parsing +// the USER database, read through q after the DDL has executed - not by parsing // the DDL statement text - so the same logic uniformly covers CREATE TABLE and // every ALTER shape that can leave a marker in sqlite_master. That derivation // is unchanged by the claim table's relocation to the system database; only @@ -97,16 +190,25 @@ type ddlTableOwner struct { // backed by the system database (db/autoinc_claim.go). A marked table found // with no store wired is a configuration error, not a "nothing to do" case, // so it fails fast rather than silently skipping the seed. -func (tm *TransactionManager) seedAutoIncBasesForDDL(tables []ddlTableOwner) error { - seen := make(map[string]bool, len(tables)) +func (tm *TransactionManager) seedAutoIncBasesForDDL(q rowQuerier, inherited []autoIncInheritance, tables []ddlTableOwner) error { claimStore, dbName := tm.autoIncClaimStoreAndDatabaseName() + for _, in := range inherited { + if claimStore == nil { + return fmt.Errorf("inherit auto-increment base for %s.%s: no auto-increment claim store wired", dbName, in.table) + } + if err := claimStore.Inherit(dbName, in.table, in.from, in.owner); err != nil { + return err + } + } + + seen := make(map[string]bool, len(tables)) for _, t := range tables { if t.table == "" || seen[t.table] { continue } seen[t.table] = true - floor, ok, err := autoIncSeedFloor(tm.db, t.table) + floor, ok, err := autoIncSeedFloor(q, t.table) if err != nil { return fmt.Errorf("derive auto-increment seed for %s: %w", t.table, err) } diff --git a/db/autoinc_seed_test.go b/db/autoinc_seed_test.go index 403b8e42..70ddb261 100644 --- a/db/autoinc_seed_test.go +++ b/db/autoinc_seed_test.go @@ -68,7 +68,7 @@ func TestSeedAutoIncBasesForDDL_NewMarkedTableSeedsZero(t *testing.T) { tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) tm.SetAutoIncClaimStore(store) tm.SetDatabaseName("testdb") - require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 7}})) + require.NoError(t, tm.seedAutoIncBasesForDDL(tm.db, nil, []ddlTableOwner{{table: "t", owner: 7}})) base, owner, ok := readClaimRow(t, store, "testdb", "t") require.True(t, ok, "expected a claim row to be created") @@ -92,7 +92,7 @@ func TestSeedAutoIncBasesForDDL_ExistingRowsRaiseTheFloor(t *testing.T) { tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) tm.SetAutoIncClaimStore(store) tm.SetDatabaseName("testdb") - require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}})) + require.NoError(t, tm.seedAutoIncBasesForDDL(tm.db, nil, []ddlTableOwner{{table: "t", owner: 1}})) base, _, ok := readClaimRow(t, store, "testdb", "t") require.True(t, ok) @@ -117,7 +117,7 @@ func TestSeedAutoIncBasesForDDL_DeclaredFloorWins(t *testing.T) { tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) tm.SetAutoIncClaimStore(store) tm.SetDatabaseName("testdb") - require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}})) + require.NoError(t, tm.seedAutoIncBasesForDDL(tm.db, nil, []ddlTableOwner{{table: "t", owner: 1}})) base, _, ok := readClaimRow(t, store, "testdb", "t") require.True(t, ok) @@ -140,7 +140,7 @@ func TestSeedAutoIncBasesForDDL_OnlyRaises(t *testing.T) { tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) tm.SetAutoIncClaimStore(store) tm.SetDatabaseName("testdb") - require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}})) + require.NoError(t, tm.seedAutoIncBasesForDDL(tm.db, nil, []ddlTableOwner{{table: "t", owner: 1}})) base1, _, ok := readClaimRow(t, store, "testdb", "t") require.True(t, ok) require.GreaterOrEqual(t, base1, int64(1000)) @@ -150,7 +150,7 @@ func TestSeedAutoIncBasesForDDL_OnlyRaises(t *testing.T) { // move down to it. _, err = db.Exec(`DELETE FROM t WHERE id > 5`) require.NoError(t, err) - require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}})) + require.NoError(t, tm.seedAutoIncBasesForDDL(tm.db, nil, []ddlTableOwner{{table: "t", owner: 1}})) base2, _, ok := readClaimRow(t, store, "testdb", "t") require.True(t, ok) @@ -172,7 +172,7 @@ func TestSeedAutoIncBasesForDDL_NoMarkerSeedsNothing(t *testing.T) { tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) tm.SetAutoIncClaimStore(store) tm.SetDatabaseName("testdb") - require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}})) + require.NoError(t, tm.seedAutoIncBasesForDDL(tm.db, nil, []ddlTableOwner{{table: "t", owner: 1}})) _, _, ok := readClaimRow(t, store, "testdb", "t") require.False(t, ok, "a table with no AUTO_INCREMENT marker must get no claim row") @@ -190,7 +190,7 @@ func TestSeedAutoIncBasesForDDL_MarkedButNotExplicitAutoIncSeedsNothing(t *testi tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) tm.SetAutoIncClaimStore(store) tm.SetDatabaseName("testdb") - require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}})) + require.NoError(t, tm.seedAutoIncBasesForDDL(tm.db, nil, []ddlTableOwner{{table: "t", owner: 1}})) _, _, ok := readClaimRow(t, store, "testdb", "t") require.False(t, ok) @@ -202,7 +202,7 @@ func TestSeedAutoIncBasesForDDL_MarkedButNotExplicitAutoIncSeedsNothing(t *testi func TestSeedAutoIncBasesForDDL_MissingTableIsANoOp(t *testing.T) { db := newSeedTestDB(t) tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) - require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "gone", owner: 1}})) + require.NoError(t, tm.seedAutoIncBasesForDDL(tm.db, nil, []ddlTableOwner{{table: "gone", owner: 1}})) } // TestSeedAutoIncBasesForDDL_DedupesRepeatedTable pins that a table named by @@ -217,7 +217,7 @@ func TestSeedAutoIncBasesForDDL_DedupesRepeatedTable(t *testing.T) { tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) tm.SetAutoIncClaimStore(store) tm.SetDatabaseName("testdb") - require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{ + require.NoError(t, tm.seedAutoIncBasesForDDL(tm.db, nil, []ddlTableOwner{ {table: "t", owner: 42}, {table: "t", owner: 99}, })) @@ -298,7 +298,7 @@ func TestSeedAutoIncBasesForDDL_NoStoreWiredFailsFast(t *testing.T) { tm.SetDatabaseName("testdb") // Deliberately no SetAutoIncClaimStore call. - err = tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}}) + err = tm.seedAutoIncBasesForDDL(tm.db, nil, []ddlTableOwner{{table: "t", owner: 1}}) require.Error(t, err, "seeding a marked table with no store wired must fail, not silently skip") require.Contains(t, err.Error(), "no auto-increment claim store wired") require.Contains(t, err.Error(), "t", "the error should name the table it failed to seed") @@ -326,9 +326,29 @@ func TestSeedAutoIncBasesForDDL_NegativeIdsSeedZero(t *testing.T) { tm := NewTransactionManager(db, nil, hlc.NewClock(1), nil) tm.SetAutoIncClaimStore(store) tm.SetDatabaseName("testdb") - require.NoError(t, tm.seedAutoIncBasesForDDL([]ddlTableOwner{{table: "t", owner: 1}})) + require.NoError(t, tm.seedAutoIncBasesForDDL(tm.db, nil, []ddlTableOwner{{table: "t", owner: 1}})) base, err := store.ReadBase("testdb", "t") require.NoError(t, err, "a table with only negative ids must seed base 0") require.Equal(t, uint64(0), base, "a table with only negative ids must seed base 0") } + +// TestAutoIncSeedFloorIgnoresIDsAboveTheWidth pins the seed on a table +// holding ids its width cannot: only the ids the allocator could ever issue +// raise the base. Counting the wide ones would put the base past the ceiling, +// and every later insert would fail as exhausted. +// +// Mutation: drop the "<= widthMax" filter from autoIncSeedFloor. The floor +// becomes 5000000000000 and this fires. +func TestAutoIncSeedFloorIgnoresIDsAboveTheWidth(t *testing.T) { + conn := newSeedTestDB(t) + _, err := conn.Exec(`CREATE TABLE legacy (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + _, err = conn.Exec(`INSERT INTO legacy (id, v) VALUES (41, 'narrow'), (5000000000000, 'wide')`) + require.NoError(t, err) + + floor, ok, err := autoIncSeedFloor(conn, "legacy") + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(41), floor, "an id above the column's width raised the base past the ceiling") +} diff --git a/db/autoinc_vote_hold.go b/db/autoinc_vote_hold.go new file mode 100644 index 00000000..a3d484fa --- /dev/null +++ b/db/autoinc_vote_hold.go @@ -0,0 +1,233 @@ +package db + +import ( + "database/sql" + "fmt" + "math" + "time" + + "github.com/maxpert/marmot/common" +) + +// AutoIncHoldTable holds this node's AUTO_INCREMENT claim votes. While its +// single row is present the node declines every claim at PREPARE, without a +// verdict, until it has raised its claim bases to the maximum reported by a +// majority of its peers (MergeAndReleaseVotes). +// +// The claim protocol's safety rests on majority intersection: every +// committed claim reached a majority, and each of those nodes keeps a base at +// or above the claim's end for as long as it votes. A node that lost its +// claim state breaks that premise, and every way of losing it is held: +// - a system database this process initialises (createAutoIncTables): the +// node is new, or lost its data directory or system database, and cannot +// tell which; +// - a system database that arrived from a peer with no local file to merge +// (RaiseAutoIncBasesFrom): the node may have ACKed claims whose rows the +// peer lacks. +// +// Claims never enter CDC, so catching up gives such a node no base above its +// DDL-time seed; only the merge (grpc.RunAutoIncBaseMerge) does. +// +// The row lives in the system database beside the bases it qualifies, so a +// hold survives a restart until the merge completes, and the merge that +// raises the bases releases the hold in the same transaction. +const AutoIncHoldTable = common.AutoIncHoldTableName + +var autoIncHoldDDL = `CREATE TABLE IF NOT EXISTS ` + AutoIncHoldTable + ` ( + id INTEGER PRIMARY KEY CHECK (id = 1), + since INTEGER NOT NULL +)` + +// holdVotesSQL places the hold; it is idempotent. +var holdVotesSQL = "INSERT OR IGNORE INTO " + AutoIncHoldTable + " (id, since) VALUES (1, ?)" + +// AutoIncBase is one table's allocation base: the largest of its committed, +// seed and merged floors (autoIncClaimDDL). +type AutoIncBase struct { + Database string + Table string + Base uint64 +} + +// VotesHeld reports whether this node's claim votes are held. +func (s *AutoIncClaimStore) VotesHeld() (bool, error) { + var n int + if err := s.sys.GetReadDB().QueryRow("SELECT COUNT(*) FROM " + AutoIncHoldTable).Scan(&n); err != nil { + return false, fmt.Errorf("read %s: %w", AutoIncHoldTable, err) + } + return n > 0, nil +} + +// Bases returns every base this node holds, as a peer's merge reads them. +// +// A base is the largest of the three floors, not committed alone. The merge's +// proof (grpc autoIncMergeSafe) needs an unheld peer's answer to cover every +// claim it ACKed - its committed floor does - or every claim the merge that +// released it covered - its merged floor does. The seed floor only raises the +// answer, which burns ids and never reissues one. +func (s *AutoIncClaimStore) Bases() ([]AutoIncBase, error) { + rows, err := s.sys.GetReadDB().Query("SELECT db, tbl, " + autoIncBaseExpr + " FROM " + AutoIncClaimTable) + if err != nil { + return nil, fmt.Errorf("read %s: %w", AutoIncClaimTable, err) + } + defer rows.Close() + + var bases []AutoIncBase + for rows.Next() { + var b AutoIncBase + var base int64 + if err := rows.Scan(&b.Database, &b.Table, &base); err != nil { + return nil, fmt.Errorf("scan %s: %w", AutoIncClaimTable, err) + } + if base < 0 { + return nil, fmt.Errorf("auto-increment base for %s.%s is negative (%d)", b.Database, b.Table, base) + } + b.Base = uint64(base) + bases = append(bases, b) + } + return bases, rows.Err() +} + +// MergeAndReleaseVotes raises every merged floor to at least the base bases +// reports for the same table, adds any table only bases names, and releases +// the vote hold, all in one transaction: the node votes again only with the +// merged bases durably in place. A floor is never lowered. +func (s *AutoIncClaimStore) MergeAndReleaseVotes(bases []AutoIncBase) error { + return s.raiseBases(bases, true) +} + +// RaiseBases raises every merged floor to at least the base bases reports +// for the same table, and adds any table only bases names, in one +// transaction. It never lowers a floor and never touches the vote hold. +// Raising is always safe: a higher merged floor only makes this node refuse +// more claims at PREPARE, and makes the claims it grants start higher. It +// never refuses a COMMIT: ApplyClaims does not test the merged floor, so a +// raise that lands between a claim's PREPARE and its COMMIT - a peer that +// committed the same claim first reports exactly its end - leaves that +// COMMIT to apply. +func (s *AutoIncClaimStore) RaiseBases(bases []AutoIncBase) error { + return s.raiseBases(bases, false) +} + +// raiseBases is RaiseBases, also releasing the vote hold in the same +// transaction when release is set. +func (s *AutoIncClaimStore) raiseBases(bases []AutoIncBase, release bool) (err error) { + tx, err := s.sys.GetWriteDB().Begin() + if err != nil { + return fmt.Errorf("begin auto-increment base merge: %w", err) + } + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + now := time.Now().UnixNano() + for _, b := range bases { + if b.Base > math.MaxInt64 { + return fmt.Errorf("merged auto-increment base for %s.%s is out of range (%d)", b.Database, b.Table, b.Base) + } + // owner 0 on a new row: it was created by a merge, not granted to a + // node. An existing row keeps the owner of its last applied range. + if _, err = tx.Exec("INSERT INTO "+AutoIncClaimTable+" (db, tbl, committed, seed, merged, owner, granted_at) VALUES (?, ?, 0, 0, ?, 0, ?) "+ + "ON CONFLICT(db, tbl) DO UPDATE SET merged = excluded.merged "+ + "WHERE excluded.merged > merged", + b.Database, b.Table, int64(b.Base), now); err != nil { + return fmt.Errorf("merge auto-increment base for %s.%s: %w", b.Database, b.Table, err) + } + } + if release { + if _, err = tx.Exec("DELETE FROM " + AutoIncHoldTable); err != nil { + return fmt.Errorf("release auto-increment vote hold: %w", err) + } + } + if err = tx.Commit(); err != nil { + return fmt.Errorf("commit auto-increment base merge: %w", err) + } + return nil +} + +// createAutoIncTables creates the claim and hold tables on the system +// database, and holds this node's votes when the system database has never +// been initialised: it has no claim table and no registered database. +// +// Such a node is new, or lost its data directory or system database, or +// crashed during its first initialisation; it cannot tell which, and a node +// that lost its state may have ACKed claims it no longer has a base for. The +// hold therefore does not depend on seeds, on the catch-up strategy or on +// whether deciding one failed. A system database that has registered +// databases but no claim table was written by a binary that predates claims +// and so never voted; it is not held, which keeps a rolling upgrade from +// holding every node at once. The hold is placed in the same transaction +// that creates the claim table, so no crash can leave a claim table without +// the hold it was created with. A claim table from before the split into +// committed, seed and merged floors is split in the same transaction +// (migrateAutoIncClaimTable). +func createAutoIncTables(sys *sql.DB) (err error) { + tx, err := sys.Begin() + if err != nil { + return fmt.Errorf("begin auto-increment table setup: %w", err) + } + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + var existing, registered int + if err = tx.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?", + AutoIncClaimTable).Scan(&existing); err != nil { + return fmt.Errorf("inspect system database: %w", err) + } + if err = tx.QueryRow("SELECT COUNT(*) FROM __marmot_databases").Scan(®istered); err != nil { + return fmt.Errorf("inspect system database: %w", err) + } + if _, err = tx.Exec(autoIncClaimDDL); err != nil { + return fmt.Errorf("create %s: %w", AutoIncClaimTable, err) + } + if err = migrateAutoIncClaimTable(tx); err != nil { + return err + } + if _, err = tx.Exec(autoIncHoldDDL); err != nil { + return fmt.Errorf("create %s: %w", AutoIncHoldTable, err) + } + if existing == 0 && registered == 0 { + if _, err = tx.Exec(holdVotesSQL, time.Now().UnixNano()); err != nil { + return fmt.Errorf("hold auto-increment votes: %w", err) + } + } + return tx.Commit() +} + +// AutoIncVotesHeld reports whether this node's claim votes are held. +func (dm *DatabaseManager) AutoIncVotesHeld() (bool, error) { + return NewAutoIncClaimStore(dm.systemDB).VotesHeld() +} + +// AutoIncBases returns every claim base this node holds (Bases), and +// whether its own votes are held - in which case a merging peer must not +// count it. +func (dm *DatabaseManager) AutoIncBases() ([]AutoIncBase, bool, error) { + store := NewAutoIncClaimStore(dm.systemDB) + held, err := store.VotesHeld() + if err != nil { + return nil, false, err + } + bases, err := store.Bases() + if err != nil { + return nil, false, err + } + return bases, held, nil +} + +// MergeAutoIncBasesAndReleaseVotes raises this node's merged floors to at +// least bases and releases its vote hold, atomically. +func (dm *DatabaseManager) MergeAutoIncBasesAndReleaseVotes(bases []AutoIncBase) error { + return NewAutoIncClaimStore(dm.systemDB).MergeAndReleaseVotes(bases) +} + +// RaiseAutoIncBases raises this node's merged floors to at least bases, in +// one transaction, without touching its vote hold (RaiseBases). +func (dm *DatabaseManager) RaiseAutoIncBases(bases []AutoIncBase) error { + return NewAutoIncClaimStore(dm.systemDB).RaiseBases(bases) +} diff --git a/db/autoinc_vote_hold_test.go b/db/autoinc_vote_hold_test.go new file mode 100644 index 00000000..d24ef584 --- /dev/null +++ b/db/autoinc_vote_hold_test.go @@ -0,0 +1,159 @@ +package db + +import ( + "database/sql" + "path/filepath" + "testing" + + "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" +) + +// TestUninitialisedSystemDatabaseHoldsVotes pins which starts hold the +// node's claim votes. Any start that initialises its system database holds, +// whatever the node's seeds or catch-up strategy: that is a new node, or one +// that lost its data directory (reviewer B's C2 (a) seedless seed node, (b) +// strategy error, (c) NO_CATCHUP), and they look the same. A crash during that +// first initialisation holds again. A node restarting over its own claim +// history, and a system database written by a binary that predates claims +// (it never voted), are not held. +// +// Mutation: hold only when the claim table is new, ignoring the registry +// (the old rule). "a system database from a binary without claims was held" +// fires. +func TestUninitialisedSystemDatabaseHoldsVotes(t *testing.T) { + requireHeld := func(t *testing.T, dm *DatabaseManager, want bool, msg string) { + t.Helper() + held, err := dm.AutoIncVotesHeld() + require.NoError(t, err) + require.Equal(t, want, held, msg) + } + + dir := t.TempDir() + dm, err := NewDatabaseManager(dir, 1, hlc.NewClock(1)) + require.NoError(t, err) + requireHeld(t, dm, true, "a node that initialised its system database may vote before merging claim bases") + require.NoError(t, dm.MergeAutoIncBasesAndReleaseVotes([]AutoIncBase{{Database: "d", Table: "t", Base: 40}})) + requireHeld(t, dm, false, "the merge did not release the hold") + require.NoError(t, dm.Close()) + + restarted, err := NewDatabaseManager(dir, 1, hlc.NewClock(1)) + require.NoError(t, err) + requireHeld(t, restarted, false, "a node restarting over its own claim history was held again") + require.NoError(t, restarted.Close()) + + // A crash after the system database file was created but before its + // first initialisation committed: no claim table, nothing registered. + interrupted := t.TempDir() + writeSystemDatabase(t, interrupted, false) + dm, err = NewDatabaseManager(interrupted, 1, hlc.NewClock(1)) + require.NoError(t, err) + requireHeld(t, dm, true, "an interrupted first initialisation was not held") + require.NoError(t, dm.Close()) + + upgraded := t.TempDir() + writeSystemDatabase(t, upgraded, true) + dm, err = NewDatabaseManager(upgraded, 1, hlc.NewClock(1)) + require.NoError(t, err) + requireHeld(t, dm, false, "a system database from a binary without claims was held") + require.NoError(t, dm.Close()) +} + +// writeSystemDatabase writes the system database a binary without claim +// tables leaves behind: the database registry, holding the default database +// when registered is set, and no claim table. +func writeSystemDatabase(t *testing.T, dataDir string, registered bool) { + t.Helper() + conn, err := sql.Open(SQLiteDriverName, filepath.Join(dataDir, SystemDatabaseName+".db")) + require.NoError(t, err) + defer conn.Close() + _, err = conn.Exec(`CREATE TABLE __marmot_databases (name TEXT PRIMARY KEY, created_at INTEGER NOT NULL, path TEXT NOT NULL)`) + require.NoError(t, err) + if registered { + _, err = conn.Exec(`INSERT INTO __marmot_databases (name, created_at, path) VALUES (?, 1, ?)`, + DefaultDatabaseName, filepath.Join("databases", DefaultDatabaseName+".db")) + require.NoError(t, err) + } +} + +// TestMergeAndReleaseVotesOnlyRaises pins the merge: every base ends at the +// maximum of this node's and the merged one, a table only the merge names is +// added, and nothing is lowered - the merged floor an earlier merge raised +// included. +// +// Mutation: drop "WHERE excluded.merged > merged" from the merge's upsert. +// "the merge lowered a base" fires. +func TestMergeAndReleaseVotesOnlyRaises(t *testing.T) { + dm, err := NewDatabaseManager(t.TempDir(), 1, hlc.NewClock(1)) + require.NoError(t, err) + defer dm.Close() + store := NewAutoIncClaimStore(dm.GetSystemDatabase()) + require.NoError(t, dm.RaiseAutoIncBases([]AutoIncBase{{Database: "d", Table: "ahead_here", Base: 900}})) + require.NoError(t, store.Seed("d", "behind_here", 100, 1)) + + require.NoError(t, dm.MergeAutoIncBasesAndReleaseVotes([]AutoIncBase{ + {Database: "d", Table: "ahead_here", Base: 500}, + {Database: "d", Table: "behind_here", Base: 700}, + {Database: "d", Table: "only_merged", Base: 60}, + })) + + bases, held, err := dm.AutoIncBases() + require.NoError(t, err) + require.False(t, held) + got := map[string]uint64{} + for _, b := range bases { + got[b.Table] = b.Base + } + require.Equal(t, uint64(900), got["ahead_here"], "the merge lowered a base") + require.Equal(t, uint64(700), got["behind_here"], "the merge did not raise a base") + require.Equal(t, uint64(60), got["only_merged"], "a table only a peer held was not added") +} + +// TestRaiseAutoIncBasesNeverTouchesTheHold pins the membership backstop's +// store operation (R3c-8a): it raises bases exactly as the merge does, in one +// transaction, and leaves the vote hold as it found it - a held node stays +// held until its own merge, an unheld node stays unheld. +// +// Mutation: raiseBases deletes the hold row whatever release says. "the +// backstop released a held node's votes" fires. +func TestRaiseAutoIncBasesNeverTouchesTheHold(t *testing.T) { + dm, err := NewDatabaseManager(t.TempDir(), 1, hlc.NewClock(1)) + require.NoError(t, err) + defer dm.Close() + store := NewAutoIncClaimStore(dm.GetSystemDatabase()) + require.NoError(t, store.Seed("d", "ahead_here", 900, 1)) + + raise := []AutoIncBase{ + {Database: "d", Table: "ahead_here", Base: 500}, + {Database: "d", Table: "only_raised", Base: 60}, + } + require.NoError(t, dm.RaiseAutoIncBases(raise)) + bases, held, err := dm.AutoIncBases() + require.NoError(t, err) + require.True(t, held, "the backstop released a held node's votes") + got := map[string]uint64{} + for _, b := range bases { + got[b.Table] = b.Base + } + require.Equal(t, uint64(900), got["ahead_here"], "the backstop lowered a base") + require.Equal(t, uint64(60), got["only_raised"], "a table only a peer held was not added") + + require.NoError(t, dm.MergeAutoIncBasesAndReleaseVotes(nil)) + require.NoError(t, dm.RaiseAutoIncBases([]AutoIncBase{{Database: "d", Table: "only_raised", Base: 90}})) + _, held, err = dm.AutoIncBases() + require.NoError(t, err) + require.False(t, held, "the backstop placed a hold") + base, err := store.ReadBase("d", "only_raised") + require.NoError(t, err) + require.Equal(t, uint64(90), base) + + // One transaction: a batch with one out-of-range base changes nothing. + err = dm.RaiseAutoIncBases([]AutoIncBase{ + {Database: "d", Table: "only_raised", Base: 95}, + {Database: "d", Table: "broken", Base: 1 << 63}, + }) + require.Error(t, err) + base, err = store.ReadBase("d", "only_raised") + require.NoError(t, err) + require.Equal(t, uint64(90), base, "a failed backstop round applied part of its bases") +} diff --git a/db/database_manager.go b/db/database_manager.go index 1f392332..8db0b21c 100644 --- a/db/database_manager.go +++ b/db/database_manager.go @@ -12,6 +12,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" "github.com/maxpert/marmot/coordinator" @@ -29,6 +30,9 @@ const ( // DatabaseProvider interface for accessing databases type DatabaseProvider interface { GetDatabase(name string) (*ReplicatedDatabase, error) + // ClusterMembership returns this node's current view of the cluster's + // total membership, the denominator of every quorum it computes. + ClusterMembership() (int, error) } // DatabaseManager manages multiple MVCC databases. @@ -53,6 +57,24 @@ type DatabaseManager struct { cdcHub CDCHub // CDC notification hub, can be nil vecIndexMgr *VectorIndexManager // Optional vector index manager autoIncClaimStore *AutoIncClaimStore // AUTO_INCREMENT claim store, backed by systemDB + membershipView atomic.Pointer[func() int] // this node's view of total cluster membership +} + +// SetClusterMembership installs view as the source of this node's view of +// the cluster's total membership (ClusterMembership). +func (dm *DatabaseManager) SetClusterMembership(view func() int) { + dm.membershipView.Store(&view) +} + +// ClusterMembership returns this node's current view of the cluster's total +// membership. An AUTO_INCREMENT claim participant compares it with the +// claimant's, so it fails rather than guess when no view is installed. +func (dm *DatabaseManager) ClusterMembership() (int, error) { + view := dm.membershipView.Load() + if view == nil { + return 0, errors.New("no cluster membership view installed") + } + return (*view)(), nil } // DatabaseMetadata represents database registry information @@ -139,11 +161,12 @@ func (dm *DatabaseManager) initSystemDatabase() error { return fmt.Errorf("failed to create database registry table: %w", err) } - // Create the AUTO_INCREMENT claim table (db/autoinc_claim.go). It lives - // here, in the system database, rather than in each user database: see - // AutoIncClaimTable's doc comment for why. - if _, err = systemDB.GetDB().Exec(autoIncClaimDDL); err != nil { - return fmt.Errorf("failed to create auto-increment claim table: %w", err) + // Create the AUTO_INCREMENT claim table (db/autoinc_claim.go) and its vote + // hold (db/autoinc_vote_hold.go). They live here, in the system database, + // rather than in each user database: see AutoIncClaimTable's doc comment + // for why. + if err = createAutoIncTables(systemDB.GetDB()); err != nil { + return fmt.Errorf("failed to create auto-increment tables: %w", err) } // Wire up GC coordination for system database @@ -246,6 +269,17 @@ func (dm *DatabaseManager) wireGCCoordination(mdb *ReplicatedDatabase, dbName st } } +// SetAutoIncIncarnationListener registers l to learn of every table +// incarnation a DDL statement ends on this node (AutoIncIncarnationListener). +func (dm *DatabaseManager) SetAutoIncIncarnationListener(l AutoIncIncarnationListener) { + dm.mu.Lock() + defer dm.mu.Unlock() + if dm.autoIncClaimStore == nil { + dm.autoIncClaimStore = NewAutoIncClaimStore(dm.systemDB) + } + dm.autoIncClaimStore.SetIncarnationListener(l) +} + // SetRefreshReplicationStatesFunc sets the callback for refreshing peer replication states // This is called by GC before Phase 2 to ensure fresh watermarks before deletion decisions // The callback is wired to all existing and future TransactionManagers @@ -437,12 +471,16 @@ func (dm *DatabaseManager) DropDatabase(name string) error { return fmt.Errorf("failed to remove database from registry: %w", err) } - // Remove this database's AUTO_INCREMENT claim rows (db/autoinc_claim.go). - // DROP TABLE deliberately does NOT reach this path and leaves its row in - // place; only DROP DATABASE removes claim rows. - _, err = dm.systemDB.GetDB().Exec("DELETE FROM "+AutoIncClaimTable+" WHERE db = ?", name) - if err != nil { - return fmt.Errorf("failed to remove auto-increment claims for database %s: %w", name, err) + // This database's AUTO_INCREMENT claim rows (db/autoinc_claim.go) stay, + // as DROP TABLE's do: a base is monotone per (database, table) name for + // the life of the cluster, so a range granted before the drop is never + // granted again after the database is recreated. The in-memory ranges of + // its tables end with it. + dm.mu.RLock() + claimStore := dm.autoIncClaimStore + dm.mu.RUnlock() + if claimStore != nil { + claimStore.databaseIncarnationEnded(name) } if detached != nil { @@ -628,6 +666,9 @@ type detachedDatabase struct { restoreFailed bool // AttachDatabase failed; the next restore of it may take it over dropped bool // DROP DATABASE arrived during the restore; AttachDatabase completes it dropPath string // the dropped database's file, removed by AttachDatabase + // tables are the replaced file's table definitions, which AttachDatabase + // compares with the restored file's (SchemaChange). + tables map[string]string } // DetachDatabase takes a user database out of service so a snapshot restore @@ -655,6 +696,18 @@ func (dm *DatabaseManager) DetachDatabase(ctx context.Context, name string) erro if err != nil || mdb == nil { return err } + // The file's tables, read before the drain closes its pools, are what the + // restored file's schema is compared with when it is attached. A DDL + // commit still in flight here reports its own change as it applies. + tables, err := tableDefinitions(ctx, mdb.GetReadDB()) + if err != nil { + log.Warn().Err(err).Str("name", name).Msg("Could not read the detached database's tables; its restore ends every table's incarnation without inheritance") + } + dm.mu.Lock() + if detached := dm.detached[name]; detached != nil { + detached.tables = tables + } + dm.mu.Unlock() // Drained outside both locks (see DatabaseManager): the detached entry // fences Create and Drop. if err := mdb.drainSQLite(ctx); err != nil { @@ -730,6 +783,18 @@ func (dm *DatabaseManager) AttachDatabase(name string) error { } dm.mu.Lock() dm.wireGCCoordination(mdb, name) + claimStore := dm.autoIncClaimStore + dm.mu.Unlock() + if err := restoredSchemaChange(mdb, name, detached.tables, claimStore); err != nil { + if closeErr := mdb.closeSQLite(); closeErr != nil { + log.Warn().Err(closeErr).Str("name", name).Msg("Error closing a restored database that failed to reattach") + } + dm.mu.Lock() + detached.restoreFailed = true + dm.mu.Unlock() + return fmt.Errorf("failed to reattach database %s: %w", name, err) + } + dm.mu.Lock() delete(dm.detached, name) dm.databases[name] = mdb dm.mu.Unlock() @@ -737,6 +802,28 @@ func (dm *DatabaseManager) AttachDatabase(name string) error { return nil } +// restoredSchemaChange applies to a restored database what any DDL applied +// on this node would (SchemaChange), before the database is back in service: +// every table in it may be a new incarnation, so the whole database's ranges +// are forgotten, as DROP DATABASE's are; every table that took the place of +// one the replaced file had inherits its base; and every table is seeded from +// its own MAX(id). +func restoredSchemaChange(mdb *ReplicatedDatabase, name string, before map[string]string, claimStore *AutoIncClaimStore) error { + after, err := tableDefinitions(context.Background(), mdb.GetDB()) + if err != nil { + return fmt.Errorf("read restored tables: %w", err) + } + var change SchemaChange + change.record(before, after, 0) + for table := range after { + change.tables = append(change.tables, ddlTableOwner{table: table}) + } + if claimStore != nil { + claimStore.databaseIncarnationEnded(name) + } + return mdb.txnMgr.seedSchemaChange(mdb.GetDB(), &change) +} + // openDetached opens the file registered for a detached database over the // meta store it kept. func (dm *DatabaseManager) openDetached(name string, metaStore MetaStore) (*ReplicatedDatabase, error) { @@ -1668,6 +1755,7 @@ func (dm *DatabaseManager) GetTranspilerSchema(database, table string) (*transfo info := &transform.SchemaInfo{ AutoIncrementColumn: autoIncCol, AutoIncrementOrdinal: columnOrdinal(schema.Columns, autoIncCol), + Database: database, } // The declared width comes from the marker the transpiler wrote into the // CREATE TABLE text; a column without one keeps the 64-bit path. @@ -1675,6 +1763,7 @@ func (dm *DatabaseManager) GetTranspilerSchema(database, table string) (*transfo if autoIncCol != "" && strings.EqualFold(col.Name, autoIncCol) { info.AutoIncrementWidth = col.DeclaredWidth info.AutoIncrementUnsigned = col.Unsigned + info.AutoIncrementExplicit = col.ExplicitAutoInc break } } diff --git a/db/ddl_apply.go b/db/ddl_apply.go index 43983855..61c9af9c 100644 --- a/db/ddl_apply.go +++ b/db/ddl_apply.go @@ -9,7 +9,10 @@ import ( ) // ApplyDDLSQLInTx rewrites DDL for idempotent replay and executes it in the -// provided transaction. +// provided transaction. It is for a read-only replica, which issues no +// AUTO_INCREMENT ids; a cluster node replays DDL through +// ReplicatedDatabase.ApplyReplayedDDL, which also records the change to its +// tables (SchemaChange). func ApplyDDLSQLInTx(ctx context.Context, tx *sql.Tx, ddlSQL string) error { if tx == nil { return fmt.Errorf("transaction is nil") @@ -20,16 +23,3 @@ func ApplyDDLSQLInTx(ctx context.Context, tx *sql.Tx, ddlSQL string) error { } return nil } - -// ApplyDDLSQL rewrites DDL for idempotent replay and executes it directly -// against a DB connection. -func ApplyDDLSQL(dbConn *sql.DB, ddlSQL string) error { - if dbConn == nil { - return fmt.Errorf("database handle is nil") - } - idempotentSQL := protocol.RewriteDDLForIdempotency(ddlSQL) - if _, err := dbConn.ExecContext(context.Background(), idempotentSQL); err != nil { - return err - } - return nil -} diff --git a/db/ddl_prepare_validate.go b/db/ddl_prepare_validate.go index 9cf1b310..949027ff 100644 --- a/db/ddl_prepare_validate.go +++ b/db/ddl_prepare_validate.go @@ -5,10 +5,12 @@ import ( "database/sql" "errors" "fmt" + "strings" "github.com/mattn/go-sqlite3" "github.com/maxpert/marmot/protocol/mysqlcode" "github.com/maxpert/marmot/protocol/query/transform" + "github.com/maxpert/marmot/protocol/query/transform/intmarker" ) // isContextError reports whether err was caused by the context being cancelled or @@ -169,7 +171,7 @@ func ValidateDDLStatements(ctx context.Context, dbConn *sql.DB, statements []str // - derived from the schema itself, not by parsing the statement text - so // a CREATE TABLE for an unrelated table is never rejected over a // pre-existing condition on some other table it never touched. - return checkAutoIncWidthCeilings(ctx, tx, changedTables(before, after)) + return checkAutoIncWidthCeilings(ctx, tx, before, changedTables(before, after)) } // tableSchemaSnapshot reads every table's CREATE TABLE text from @@ -207,18 +209,41 @@ func changedTables(before, after map[string]string) []string { return touched } -// checkAutoIncWidthCeilings rejects the DDL when a table's explicitly -// declared AUTO_INCREMENT column can no longer hold the ids the table already -// contains. It runs inside the validation transaction, so it sees the NEW -// schema (this DDL's own effect) against the OLD rows the DDL has not -// touched, and is always rolled back with the rest of validation. -func checkAutoIncWidthCeilings(ctx context.Context, tx *sql.Tx, tables []string) error { +// widthUnchanged reports whether beforeSQL already declared col with the +// width and signedness attrs records. +func widthUnchanged(beforeSQL, col string, attrs intmarker.Attributes) bool { + for name, prior := range intmarker.Decode(beforeSQL) { + if strings.EqualFold(name, col) { + return prior.Bits == attrs.Bits && prior.Unsigned == attrs.Unsigned + } + } + return false +} + +// checkAutoIncWidthCeilings rejects the DDL when it declares a table's +// explicitly declared AUTO_INCREMENT column (autoIncMarkedColumn) with a width +// that cannot hold +// the ids the table already contains. It runs inside the validation +// transaction, so it sees the NEW schema (this DDL's own effect) against the +// OLD rows the DDL has not touched, and is always rolled back with the rest +// of validation. +// +// A DDL that leaves the column's declared width as it was is not checked. +// The check exists to refuse declaring a width the data does not fit; a +// table whose ids already exceed a width it was declared with earlier - one +// filled with wide ids before narrow allocation existed - would otherwise +// refuse every later DDL, including the ones that repair it. Such a table +// keeps working: the allocator issues ids at or below the width, which can +// never meet the wider ones, and its base is seeded from the ids the width +// can hold (autoIncSeedFloor). before is sqlite_master's CREATE text per +// table ahead of this DDL. +func checkAutoIncWidthCeilings(ctx context.Context, tx *sql.Tx, before map[string]string, tables []string) error { for _, table := range tables { col, attrs, ok, err := autoIncMarkedColumn(tx, table) if err != nil { return fmt.Errorf("derive auto-increment column for %s: %w", table, err) } - if !ok { + if !ok || widthUnchanged(before[table], col, attrs) { continue } diff --git a/db/ddl_prepare_validate_autoinc_code_test.go b/db/ddl_prepare_validate_autoinc_code_test.go index abd90775..f6b36adb 100644 --- a/db/ddl_prepare_validate_autoinc_code_test.go +++ b/db/ddl_prepare_validate_autoinc_code_test.go @@ -75,9 +75,9 @@ func TestPrepareRejectionCarriesTheMySQLCode(t *testing.T) { } // TestPrepareOfARefusedDDLCarriesTheMySQLCode drives the DDL rejection site in -// ReplicationEngine.Prepare itself: a DDL that would leave a TINYINT -// AUTO_INCREMENT column unable to hold the id 200 it already contains is -// refused, and the PrepareResult carries 1264 for the coordinator. +// ReplicationEngine.Prepare itself: a DDL that declares a TINYINT +// AUTO_INCREMENT column over a value (200) it cannot hold is refused, and the +// PrepareResult carries 1264 for the coordinator. // // Mutation: drop ErrorCode from the PrepareResult literal at the DDL rejection // site in prepareRegularTransaction. "the DDL rejection lost its MySQL code" @@ -85,14 +85,14 @@ func TestPrepareRejectionCarriesTheMySQLCode(t *testing.T) { func TestPrepareOfARefusedDDLCarriesTheMySQLCode(t *testing.T) { engine, dm, cleanup := setupTestReplicationEngine(t) defer cleanup() - mdb := markedTableDB(t, engine, dm, "CREATE TABLE t (id INTEGER /*M:8a*/ PRIMARY KEY, v TEXT)") - _, err := mdb.GetWriteDB().Exec("INSERT INTO t (id, v) VALUES (200, 'x')") + mdb := markedTableDB(t, engine, dm, "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)") + _, err := mdb.GetWriteDB().Exec("INSERT INTO t (id, v) VALUES (1, 'x')") require.NoError(t, err) result := engine.Prepare(context.Background(), &PrepareRequest{ TxnID: 7600, NodeID: 1, StartTS: hlc.Timestamp{WallTime: 14}, Database: "testdb", Statements: []protocol.Statement{{Type: protocol.StatementDDL, Database: "testdb", TableName: "t", - SQL: "ALTER TABLE t ADD COLUMN extra INTEGER"}}, + SQL: "ALTER TABLE t ADD COLUMN seq INTEGER /*M:8a*/ DEFAULT 200"}}, }) require.False(t, result.Success) require.True(t, result.Rejected) diff --git a/db/ddl_prepare_validate_test.go b/db/ddl_prepare_validate_test.go index fccd7c43..4bbb5ced 100644 --- a/db/ddl_prepare_validate_test.go +++ b/db/ddl_prepare_validate_test.go @@ -409,3 +409,26 @@ func TestValidateDDLStatements_UnrelatedTableIsNotChecked(t *testing.T) { `CREATE TABLE unrelated (id INTEGER PRIMARY KEY)`, })) } + +// TestValidateDDLStatements_OverWidthTableKeepsAcceptingDDL is the path back +// for a table filled with wide ids before narrow allocation existed: its ids +// already exceed its declared width, and a DDL that leaves that width alone +// must not be refused, or the table could never be altered again - including +// by the DDL that repairs it. +// +// Mutation: check every changed table regardless of whether its width +// changed. The ALTER is refused with AutoIncWidthExceededError and this +// fires. +func TestValidateDDLStatements_OverWidthTableKeepsAcceptingDDL(t *testing.T) { + t.Parallel() + db := openDDLValidationDB(t) + + _, err := db.Exec(`CREATE TABLE legacy (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO legacy (id, v) VALUES (5000000000000, 'wide id from before narrow allocation')`) + require.NoError(t, err) + + require.NoError(t, ValidateDDLStatements(context.Background(), db, []string{ + `ALTER TABLE legacy ADD COLUMN note TEXT`, + }), "a DDL that does not re-declare the column's width was refused over ids the width never held") +} diff --git a/db/meta_store_pebble.go b/db/meta_store_pebble.go index 44183760..39d34b08 100644 --- a/db/meta_store_pebble.go +++ b/db/meta_store_pebble.go @@ -88,6 +88,11 @@ type PebbleMetaStore struct { // Prepared transactions whose row locks recoverPreparedCDCState restored // at open, held until the memory tier takes them (takeRecoveredPrepared). recoveredPrepared []TxnImmutableRecord + + // syncWrite is pebble.Sync, or pebble.NoSync for a store opened without + // a WAL (PebbleMetaStoreOptions.DisableWAL, tests only), which has no + // durable write to offer and refuses a synced one. + syncWrite *pebble.WriteOptions } func cdcPrepareSyncStrict() bool { @@ -228,6 +233,10 @@ func NewPebbleMetaStore(path string, opts PebbleMetaStoreOptions) (*PebbleMetaSt dmlIntents: xsync.NewMapOf[uint64, *xsync.MapOf[string, *WriteIntentRecord]](), persistedIntentTxns: xsync.NewMapOf[uint64, struct{}](), cdcLocks: NewXsyncCDCLockStore(), + syncWrite: pebble.Sync, + } + if opts.DisableWAL { + store.syncWrite = pebble.NoSync } // Initialize persistent counters @@ -1713,7 +1722,14 @@ func (s *PebbleMetaStore) GetSchemaVersion(dbName string) (int64, error) { return rec.Version, nil } -// UpdateSchemaVersion updates the schema version for a database +// UpdateSchemaVersion updates the schema version for a database. +// +// Unlike the transaction records, the write is synced. The DDL it counts is +// already durable in SQLite, and a version lost to a crash is not healed by +// replication: the node reports a lower version than the schema it runs, and +// every peer's transaction that requires the newer version is refused on it. +// Every schema-version write goes through here - a DDL's increment, catch-up +// and a snapshot restore - and DDL is rare, so the sync stays off the hot path. func (s *PebbleMetaStore) UpdateSchemaVersion(dbName string, version int64, ddlSQL string, txnID uint64) error { rec := &pebbleSchemaVersionRecord{ Version: version, @@ -1727,7 +1743,7 @@ func (s *PebbleMetaStore) UpdateSchemaVersion(dbName string, version int64, ddlS return err } - return s.db.Set(pebbleSchemaKey(dbName), data, pebble.NoSync) + return s.db.Set(pebbleSchemaKey(dbName), data, s.syncWrite) } // GetAllSchemaVersions returns all schema versions indexed by database name diff --git a/db/prepared_intent_test.go b/db/prepared_intent_test.go index df7781c4..3c194a08 100644 --- a/db/prepared_intent_test.go +++ b/db/prepared_intent_test.go @@ -176,6 +176,7 @@ func TestCommitRefusesAClaimWhoseIntentIsGone(t *testing.T) { res := commitClaim(engine, a) require.False(t, res.Success, "a claim COMMIT was ACKed with no claim intent to apply") require.Contains(t, res.Error, ErrAutoIncClaimNotApplicable.Error()) + require.True(t, res.ToCoordinatorResponse().ClaimNotApplicable, "the refusal reached the coordinator unclassified") require.Equal(t, uint64(500), readClaimBase(t, dm, "testdb", "users")) } @@ -185,8 +186,8 @@ func TestCommitRefusesAClaimWhoseIntentIsGone(t *testing.T) { // new floor, so the COMMIT must be refused and the base left where the seed // put it. // -// Mutation: drop "AND base <= ?" from applyClaimTx's UPDATE. The COMMIT is -// ACKed and "the claim lowered a base a seed had raised" fires. +// Mutation: drop "AND seed <= ?" from applyClaimTx's UPDATE. The COMMIT is +// ACKed and "a claim below the raised floor was ACKed" fires. func TestCommitRefusesAClaimOnceTheBaseMovedPastItsPrevBase(t *testing.T) { engine, dm, cleanup := setupTestReplicationEngine(t) defer cleanup() @@ -198,9 +199,10 @@ func TestCommitRefusesAClaimOnceTheBaseMovedPastItsPrevBase(t *testing.T) { seedClaimBase(t, dm, "testdb", "users", 800, 2) res := commitClaim(engine, a) - require.Equal(t, uint64(800), readClaimBase(t, dm, "testdb", "users"), "the claim lowered a base a seed had raised") require.False(t, res.Success, "a claim below the raised floor was ACKed") + require.Equal(t, uint64(800), readClaimBase(t, dm, "testdb", "users"), "the claim lowered a base a seed had raised") require.Contains(t, res.Error, ErrAutoIncClaimNotApplicable.Error()) + require.True(t, res.ClaimNotApplicable, "the refusal was not classified as a claim that cannot be applied") } // intentsFrom is a MetaStore whose GetIntentsByTxn replays intents captured @@ -222,7 +224,7 @@ func (m intentsFrom) GetIntentsByTxn(uint64) ([]*WriteIntentRecord, error) { // must now be refused, so this node ACKs one of the two and never both; two // majorities that each ACKed one would have to share a node that ACKed both. // -// Mutation: drop "AND base <= ?" from applyClaimTx's UPDATE. A's apply +// Mutation: drop "AND committed <= ?" from applyClaimTx's UPDATE. A's apply // succeeds and "this node applied two overlapping claims" fires. func TestApplyClaimsACKsAtMostOneOfTwoOverlappingClaims(t *testing.T) { engine, dm, cleanup := setupTestReplicationEngine(t) diff --git a/db/replication_engine.go b/db/replication_engine.go index 5f76af45..59f506a9 100644 --- a/db/replication_engine.go +++ b/db/replication_engine.go @@ -2,12 +2,14 @@ package db import ( "context" + "errors" "fmt" "github.com/maxpert/marmot/coordinator" "github.com/maxpert/marmot/hlc" "github.com/maxpert/marmot/protocol" "github.com/maxpert/marmot/protocol/filter" + "github.com/maxpert/marmot/protocol/mysqlcode" "github.com/rs/zerolog/log" ) @@ -38,9 +40,10 @@ type PrepareResult struct { // SQLite cannot apply. Infrastructure failures (timeouts, storage errors) leave // it false so the coordinator keeps treating them as a missing ACK. Rejected bool - // AutoIDStoredBase is this participant's own committed base for the table a - // rejected AUTO_INCREMENT range claim named, so the claimant can retry above - // it rather than spin. Zero on every response that is not such a rejection. + // AutoIDStoredBase is this participant's own base (the largest of its claim + // row's floors) for the table a rejected AUTO_INCREMENT range claim named, + // so the claimant can retry above it rather than spin. Zero on every + // response that is not such a rejection. AutoIDStoredBase uint64 // ErrorCode is the MySQL server error code this rejection must reach the // client with, when the underlying error named one (a @@ -65,6 +68,9 @@ type CommitResult struct { Success bool Error string DDLSQL string // SQL for DDL statements (if any) + // ClaimNotApplicable reports a COMMIT refused because this node could not + // apply the AUTO_INCREMENT claim it carries (ErrAutoIncClaimNotApplicable). + ClaimNotApplicable bool } // AbortRequest contains parameters for the abort phase @@ -299,11 +305,17 @@ func (re *ReplicationEngine) autoIncClaimStore() (*AutoIncClaimStore, error) { // which is what keeps a single-node cluster minting 1, 2, 3 with no jump at any // range boundary. // -// and an ABSENT stored base REJECTS. That last clause is the protocol's whole -// safety argument and it is the one an implementer will get wrong: the natural -// implementation is "absent means 0 means yes", and a cache that defaults to 0 -// and accepts is precisely the thing that cannot cast the rejection that would -// repair it. +// and an ABSENT stored base is never read as 0. That clause is the one an +// implementer will get wrong: the natural implementation is "absent means 0 +// means yes", and a cache that defaults to 0 and accepts is precisely the +// thing that cannot cast the rejection that would repair it. An absent row is +// instead backfilled from the table itself (backfillAutoIncBase) on a node +// whose votes are not held, and refused otherwise. +// +// Two more gates come before the condition. A node whose votes are held +// (AutoIncHoldTable) declines without a verdict. A claim whose Membership is +// not this node's own count of the cluster is rejected: majorities of two +// different memberships need not intersect. // // widthMax is derived from THIS node's own sqlite_master, never taken from the // claimant: the width marker rides in the replicated DDL text so every node @@ -328,8 +340,10 @@ func (re *ReplicationEngine) autoIncClaimStore() (*AutoIncClaimStore, error) { // it marks the transaction committed (Commit's ApplyClaims call runs ahead of // txnMgr.CommitTransaction), so the base is durable while the lock is still // held. ApplyClaims also refuses a COMMIT whose claim intent is gone or whose -// prevBase the stored base has passed, so an ACK never depends on the lock -// alone (see ApplyClaims for the per-node disjointness argument). +// newBase this node's committed or seed floor has passed, so an ACK never +// depends on the lock alone (see ApplyClaims for the per-node disjointness +// argument). The base read below is the largest of the row's committed, seed +// and merged floors (AutoIncClaimStore.ReadBase). // // The intent is therefore written FIRST and the base read SECOND. Read first // and the read is unlocked: two claimants can both read the pre-commit base, @@ -385,13 +399,44 @@ func (re *ReplicationEngine) prepareAutoIncClaim( Error: fmt.Sprintf("auto-increment claim for %s refused: %v", claim.Table, err)} } + held, err := claimStore.VotesHeld() + if err != nil { + return &PrepareResult{Success: false, Rejected: true, + Error: fmt.Sprintf("auto-increment claim for %s refused: %v", claim.Table, err)} + } + if held { + // Not a verdict on the claim: this node's bases may be below a range + // it or a majority granted, until it has merged bases from a + // majority (AutoIncHoldTable). A missing ACK leaves the claim to the + // rest of the cluster, and to this node once the merge completes. + return &PrepareResult{Success: false, + Error: fmt.Sprintf("auto-increment claim for %s declined: this node's votes are held until it merges claim bases from a majority", claim.Table)} + } + storedBase, err := claimStore.ReadBase(req.Database, claim.Table) + if errors.Is(err, ErrAutoIncBaseAbsent) { + storedBase, err = re.backfillAutoIncBase(replicatedDB, claimStore, req.Database, claim.Table) + } if err != nil { - // Absent, unreadable - either way this node cannot vote yes. + // Absent with nothing to backfill from, or unreadable - either way + // this node cannot vote yes. return &PrepareResult{Success: false, Rejected: true, Error: fmt.Sprintf("auto-increment claim for %s refused: %v", claim.Table, err)} } + membership, err := re.dbMgr.ClusterMembership() + if err != nil || claim.Membership == 0 || int(claim.Membership) != membership { + // Majorities of two different memberships need not intersect, so a + // claim is only voted on by a node that counts the cluster the way + // the claimant did. A claimant that predates the field sends 0, + // which no membership equals. The base rides along like any + // rejection's; the claimant retries once gossip has converged. + return &PrepareResult{Success: false, Rejected: true, + Error: fmt.Sprintf("auto-increment claim for %s counted a membership of %d, this node counts %d (%v)", + claim.Table, claim.Membership, membership, err), + AutoIDStoredBase: storedBase} + } + widthMax, err := replicatedDB.AutoIncWidthMax(claim.Table) if err != nil { return &PrepareResult{Success: false, Rejected: true, @@ -420,12 +465,47 @@ func (re *ReplicationEngine) prepareAutoIncClaim( // the sum is uint64 and a large size wraps, so the additive form // lets a claim past the ceiling read as if it were inside it. Error: fmt.Sprintf("auto-increment claim for %s exhausts the column: %d+%d > %d", claim.Table, claim.NewBase, claim.Size, widthMax), - AutoIDStoredBase: storedBase} + AutoIDStoredBase: storedBase, + // ER_DUP_ENTRY is what MySQL reports for a full AUTO_INCREMENT + // column, and it is how the claimant tells this terminal verdict + // apart from every retryable one (coordinator.ClaimRange). + ErrorCode: mysqlcode.ErrCodeDupEntry} } return nil } +// backfillAutoIncBase gives a table with no claim row the base the DDL-time +// seed would have written (autoIncSeedFloor: the declared floor, raised by +// the largest id the column's width can hold) and returns it. It runs under +// the claim key's lock, only on a node whose votes are not held. +// +// That is what makes it sound where "absent means 0" is not. On such a node +// an absent row means this node never ACKed a claim on the table: an ACK +// writes the row durably, no DDL removes a row, a restore never lowers or +// drops one, and a node that lost its rows is held (AutoIncHoldTable). So the node is exactly one that missed every claim on +// the table, whose low base majority intersection already tolerates. Rows +// go missing this way for a table marked before claim rows existed, and on a +// node whose user database was restored from a peer while it was down for +// the table's CREATE. +// +// A table with no explicitly declared AUTO_INCREMENT marker has nothing to +// backfill and +// stays refused. +func (re *ReplicationEngine) backfillAutoIncBase(replicatedDB *ReplicatedDatabase, claimStore *AutoIncClaimStore, database, table string) (uint64, error) { + floor, ok, err := autoIncSeedFloor(replicatedDB.GetReadDB(), table) + if err != nil { + return 0, fmt.Errorf("backfill auto-increment base: %w", err) + } + if !ok { + return 0, ErrAutoIncBaseAbsent + } + if err := claimStore.Seed(database, table, floor, re.nodeID); err != nil { + return 0, fmt.Errorf("backfill auto-increment base: %w", err) + } + return claimStore.ReadBase(database, table) +} + // processStatement processes a single statement within a transaction func (re *ReplicationEngine) processStatement(txn *Transaction, txnMgr *TransactionManager, metaStore MetaStore, stmt protocol.Statement, req *PrepareRequest, stmtSeq uint64, adoptCapturedRows bool) *PrepareResult { if err := txnMgr.AddStatement(txn, stmt); err != nil { @@ -730,13 +810,20 @@ func (re *ReplicationEngine) Commit(ctx context.Context, req *CommitRequest) *Co return &CommitResult{Success: false, Error: fmt.Sprintf("auto-increment claim apply failed: %v", csErr)} } if err := claimStore.ApplyClaims(req.Database, req.TxnID, replicatedDB.GetMetaStore()); err != nil { - log.Error(). - Err(err). + // A claim this node cannot apply is the protocol refusing an ACK + // it must not give; anything else is a failed write. + notApplicable := errors.Is(err, ErrAutoIncClaimNotApplicable) + ev := log.Error() + if notApplicable { + ev = log.Warn() + } + ev.Err(err). Uint64("txn_id", req.TxnID). Uint64("node_id", re.nodeID). Str("database", req.Database). Msg("COMMIT REFUSED: auto-increment claim could not be applied") - return &CommitResult{Success: false, Error: fmt.Sprintf("auto-increment claim apply failed: %v", err)} + return &CommitResult{Success: false, ClaimNotApplicable: notApplicable, + Error: fmt.Sprintf("auto-increment claim apply failed: %v", err)} } } @@ -816,8 +903,9 @@ func (pr *PrepareResult) ToCoordinatorResponse() *coordinator.ReplicationRespons // ToCoordinatorResponse converts CommitResult to coordinator.ReplicationResponse func (cr *CommitResult) ToCoordinatorResponse() *coordinator.ReplicationResponse { return &coordinator.ReplicationResponse{ - Success: cr.Success, - Error: cr.Error, + Success: cr.Success, + Error: cr.Error, + ClaimNotApplicable: cr.ClaimNotApplicable, } } diff --git a/db/replication_engine_test.go b/db/replication_engine_test.go index 0781245e..1b4c1aaf 100644 --- a/db/replication_engine_test.go +++ b/db/replication_engine_test.go @@ -67,6 +67,10 @@ func setupTestReplicationEngine(t *testing.T) (*ReplicationEngine, *DatabaseMana dm, err := NewDatabaseManager(tmpDir, 1, clock) require.NoError(t, err, "Failed to create DatabaseManager") + // A new system database holds the node's claim votes; these tests model + // a node of a cluster that has merged. A test about the hold places it + // itself. + require.NoError(t, dm.MergeAutoIncBasesAndReleaseVotes(nil)) engine := NewReplicationEngine(1, dm, clock) diff --git a/db/schema_change.go b/db/schema_change.go new file mode 100644 index 00000000..d37ca73a --- /dev/null +++ b/db/schema_change.go @@ -0,0 +1,115 @@ +package db + +import ( + "context" + "database/sql" + "fmt" + + "github.com/maxpert/marmot/protocol" +) + +// sqlExecQuerier is what running one DDL statement between two reads of the +// schema needs. *sql.Conn and *sql.Tx both satisfy it. +type sqlExecQuerier interface { + ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) + QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) +} + +// SchemaChange is what a sequence of DDL statements did to one database's +// tables, as AUTO_INCREMENT claims need to know it: which table incarnations +// ended, which tables took the place of removed ones, and which tables to +// seed a claim base for. +// +// Every path that changes a database's schema on a node records its change +// here and finishes it the same way, in this order: +// 1. TransactionManager.endIncarnations, before the new schema is visible to +// queries, so no insert into a new incarnation takes an id from a range +// claimed for an old one; +// 2. TransactionManager.seedSchemaChange, which raises inherited bases and +// seeds every touched table from its own MAX(id). +// +// The paths are the 2PC COMMIT (applyNonDMLIntents), anti-entropy replay +// (ReplicatedDatabase.ApplyReplayedDDL) and a restored database's reattach +// (DatabaseManager.AttachDatabase). +type SchemaChange struct { + ended []string + inherited []autoIncInheritance + tables []ddlTableOwner +} + +// Exec runs one DDL statement on q, with the database's table definitions +// read on q just before and just after it, and records what it changed. +// table is the table the statement names, owner the node that authored it. +func (c *SchemaChange) Exec(ctx context.Context, q sqlExecQuerier, stmt, table string, owner uint64) error { + before, err := tableDefinitions(ctx, q) + if err != nil { + return fmt.Errorf("failed to read tables before DDL: %w", err) + } + if _, err := q.ExecContext(ctx, stmt); err != nil { + return fmt.Errorf("failed to execute DDL statement: %w", err) + } + after, err := tableDefinitions(ctx, q) + if err != nil { + return fmt.Errorf("failed to read tables after DDL: %w", err) + } + c.record(before, after, owner) + c.tables = append(c.tables, ddlTableOwner{table: table, owner: owner}) + return nil +} + +// record adds one before/after pair of table definitions to the change. +func (c *SchemaChange) record(before, after map[string]string, owner uint64) { + c.ended = append(c.ended, endedIncarnations(before, after)...) + for _, in := range inheritedTables(before, after, owner) { + c.inherited = append(c.inherited, in) + c.tables = append(c.tables, ddlTableOwner{table: in.table, owner: owner}) + } +} + +// Empty reports whether the change recorded nothing. +func (c *SchemaChange) Empty() bool { + return len(c.ended) == 0 && len(c.inherited) == 0 && len(c.tables) == 0 +} + +// endIncarnations reports every table incarnation the change ended to the +// claim store's listener. +func (tm *TransactionManager) endIncarnations(change *SchemaChange) { + for _, name := range change.ended { + tm.autoIncIncarnationEnded(name) + } +} + +// seedSchemaChange raises the claim bases the change requires, reading the +// seed floors through q: the connection or transaction that sees the +// change's schema and rows. +func (tm *TransactionManager) seedSchemaChange(q rowQuerier, change *SchemaChange) error { + if len(change.tables) == 0 { + return nil + } + return tm.seedAutoIncBasesForDDL(q, change.inherited, change.tables) +} + +// ApplyReplayedDDL runs one replayed DDL statement, rewritten for idempotent +// replay, in the replay's transaction and records it in change. The caller +// finishes change with FinishReplayedSchemaChange in the same transaction +// before committing it. +func (mdb *ReplicatedDatabase) ApplyReplayedDDL(ctx context.Context, tx *sql.Tx, ddlSQL, table string, owner uint64, change *SchemaChange) error { + if tx == nil { + return fmt.Errorf("transaction is nil") + } + return change.Exec(ctx, tx, protocol.RewriteDDLForIdempotency(ddlSQL), table, owner) +} + +// FinishReplayedSchemaChange ends the incarnations change ended and raises +// the claim bases it requires, reading floors inside tx: tx holds this +// database's only write connection, so reading through anything else would +// wait on it. It runs before tx commits, so the new schema is not yet +// visible when the incarnations end, and a failure rolls the replay back to +// be retried whole. +func (mdb *ReplicatedDatabase) FinishReplayedSchemaChange(tx *sql.Tx, change *SchemaChange) error { + if change.Empty() { + return nil + } + mdb.txnMgr.endIncarnations(change) + return mdb.txnMgr.seedSchemaChange(tx, change) +} diff --git a/db/schema_change_test.go b/db/schema_change_test.go new file mode 100644 index 00000000..ab111a44 --- /dev/null +++ b/db/schema_change_test.go @@ -0,0 +1,132 @@ +package db + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// withinDeadline fails the test when f has not returned in time: every path +// here runs on a database whose write handle has one connection, and a read +// that waits on it never returns. +func withinDeadline(t *testing.T, what string, f func() error) { + t.Helper() + done := make(chan error, 1) + go func() { done <- f() }() + select { + case err := <-done: + require.NoError(t, err, what) + case <-time.After(10 * time.Second): + t.Fatalf("%s did not finish: it is waiting on the write connection its own transaction holds", what) + } +} + +// replayDDL applies statements the way anti-entropy replay does: in one +// transaction on the database's write handle, finished before the commit. +func replayDDL(t *testing.T, mdb *ReplicatedDatabase, table string, statements ...string) { + t.Helper() + withinDeadline(t, "replay", func() error { + ctx := context.Background() + tx, err := mdb.GetDB().BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + var change SchemaChange + for _, stmt := range statements { + if err := mdb.ApplyReplayedDDL(ctx, tx, stmt, table, 2, &change); err != nil { + return err + } + } + if err := mdb.FinishReplayedSchemaChange(tx, &change); err != nil { + return err + } + return tx.Commit() + }) +} + +// TestReplayedDDLChangesIncarnationsLikeACommit pins R3c-10 on the +// anti-entropy replay path (grpc handleReplay -> ApplyReplayedDDL): a node +// that catches up on a DDL it missed ends the same incarnations, inherits +// the same bases and seeds the same tables as one that applied its COMMIT. +// Reviewer B's H1: a lagging n1 keeps a cursor for t while it misses +// "RENAME t TO t2"; without the forget it would go on issuing from it. +// +// Mutations: FinishReplayedSchemaChange returns nil at once - "a replayed +// rename did not end both names' incarnations" fires; it seeds through the +// database's write handle instead of the replay's transaction - the replay +// misses its deadline. +func TestReplayedDDLChangesIncarnationsLikeACommit(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE t (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + _, err := mdb.GetWriteDB().Exec("INSERT INTO t (id, v) VALUES (90, 'x')") + require.NoError(t, err) + seedClaimBase(t, dm, "testdb", "t", 500, 1) + + rec := &incarnationRecorder{} + dm.SetAutoIncIncarnationListener(rec) + + replayDDL(t, mdb, "t", "ALTER TABLE t RENAME TO t2") + require.Equal(t, []string{"testdb.t", "testdb.t2"}, rec.takeTables(), "a replayed rename did not end both names' incarnations") + require.Equal(t, uint64(500), readClaimBase(t, dm, "testdb", "t2"), "a replayed rename did not inherit t's base") + + replayDDL(t, mdb, "u", "CREATE TABLE u (id INTEGER /*M:16a*/ PRIMARY KEY, v TEXT)", "INSERT INTO u (id, v) VALUES (700, 'y')") + require.Equal(t, []string{"testdb.u"}, rec.takeTables(), "a replayed CREATE did not end u's incarnation") + // The seed reads inside the replay's transaction, so it sees the row the + // same replay wrote. + require.Equal(t, uint64(700), readClaimBase(t, dm, "testdb", "u"), "a replayed CREATE was not seeded from its rows") + + // Replayed DDL is idempotent: replaying the CREATE again changes nothing. + replayDDL(t, mdb, "u", "CREATE TABLE u (id INTEGER /*M:16a*/ PRIMARY KEY, v TEXT)") + require.Empty(t, rec.takeTables(), "an idempotent replay ended an incarnation") +} + +// TestAttachAfterRestoreEndsIncarnationsAndSeeds pins R3c-10 on the restore +// path (grpc CatchUpFromPeer -> DetachDatabase, file replaced, +// AttachDatabase): every table in the restored file may be a new +// incarnation, so the node forgets the whole database's ranges before it is +// back in service, a table that took the place of one the old file had +// inherits its base, and every table is seeded from its own MAX(id). +// +// Mutation: skip restoredSchemaChange in AttachDatabase. "a restore did not +// end the database's incarnations" fires. +func TestAttachAfterRestoreEndsIncarnationsAndSeeds(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE t (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "t", 500, 1) + + rec := &incarnationRecorder{} + dm.SetAutoIncIncarnationListener(rec) + + var relPath string + require.NoError(t, dm.GetSystemDatabase().GetDB().QueryRow( + "SELECT path FROM __marmot_databases WHERE name = ?", "testdb").Scan(&relPath)) + + require.NoError(t, dm.DetachDatabase(context.Background(), "testdb")) + // The restored file: t was renamed to t2 on the peer, which also holds + // rows this node never saw, and a table this node never had. + restored, err := sql.Open("sqlite3", filepath.Join(dm.dataDir, relPath)) + require.NoError(t, err) + for _, stmt := range []string{ + "ALTER TABLE t RENAME TO t2", + "INSERT INTO t2 (id, v) VALUES (300, 'peer')", + "CREATE TABLE w (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)", + "INSERT INTO w (id, v) VALUES (800, 'peer')", + } { + _, err := restored.Exec(stmt) + require.NoError(t, err, stmt) + } + require.NoError(t, restored.Close()) + + require.NoError(t, dm.AttachDatabase("testdb")) + require.Equal(t, []string{"testdb"}, rec.databases, "a restore did not end the database's incarnations") + require.Equal(t, uint64(500), readClaimBase(t, dm, "testdb", "t2"), "the restored t2 did not inherit t's base") + require.Equal(t, uint64(800), readClaimBase(t, dm, "testdb", "w"), "the restored w was not seeded from its rows") + require.Equal(t, uint64(500), readClaimBase(t, dm, "testdb", "t"), "the restore changed the replaced name's row") +} diff --git a/db/transaction.go b/db/transaction.go index 71d06f19..608128c3 100644 --- a/db/transaction.go +++ b/db/transaction.go @@ -191,6 +191,15 @@ func (tm *TransactionManager) autoIncClaimStoreAndDatabaseName() (*AutoIncClaimS return tm.autoIncClaimStore, tm.databaseName } +// autoIncIncarnationEnded tells the claim store's listener that a DDL +// statement ended the incarnation of table in this manager's database +// (AutoIncIncarnationListener). +func (tm *TransactionManager) autoIncIncarnationEnded(table string) { + if claimStore, dbName := tm.autoIncClaimStoreAndDatabaseName(); claimStore != nil { + claimStore.tableIncarnationEnded(dbName, table) + } +} + // BeginTransaction starts a new distributed transaction with auto-generated ID func (tm *TransactionManager) BeginTransaction(nodeID uint64) (*Transaction, error) { startTS := tm.clock.Now() @@ -529,8 +538,7 @@ func (tm *TransactionManager) applyNonDMLIntents(txnID uint64, intents []*WriteI return nonDMLIntents[i].CreatedAt < nonDMLIntents[j].CreatedAt }) - hasDDL := false - var ddlTables []ddlTableOwner + var change SchemaChange for _, intent := range nonDMLIntents { var vectorChange common.VectorIndexChange if err := DeserializeData(intent.DataSnapshot, &vectorChange); err == nil && vectorChange.Action != 0 { @@ -553,44 +561,58 @@ func (tm *TransactionManager) applyNonDMLIntents(txnID uint64, intents []*WriteI log.Debug().Uint64("txn_id", txnID).Msg("LOAD DATA statement executed") continue } - if _, err := tm.db.Exec(intent.SQLStatement); err != nil { - return fmt.Errorf("failed to execute DDL statement: %w", err) + if err := tm.execDDL(&change, intent); err != nil { + return err } - hasDDL = true - ddlTables = append(ddlTables, ddlTableOwner{table: intent.TableName, owner: intent.NodeID}) - log.Debug().Uint64("txn_id", txnID).Str("sql", intent.SQLStatement).Msg("DDL statement executed") } + if change.Empty() { + return nil + } + + // Reported before the schema cache reload below makes the new definitions + // visible to queries, so no insert into a new incarnation can take an id + // from a range claimed for an old one. + tm.endIncarnations(&change) // Reload schema cache after DDL operations. A failed reload leaves the // cache stale, so subsequent preupdate hooks would silently drop CDC data // for any column the DDL added/changed - fail the DDL apply instead of // swallowing the error. - if hasDDL && tm.schemaCache != nil { + if tm.schemaCache != nil { if err := tm.reloadSchemaCache(); err != nil { return fmt.Errorf("failed to reload schema cache after DDL (txn %d): %w", txnID, err) } } // Seed __marmot__autoinc for every table this DDL tagged AUTO_INCREMENT, - // after the DDL has executed and the schema cache reflects it. The claim table now lives in the system database - a - // separate SQLite file from tm.db (db/autoinc_claim.go) - so the seed can - // no longer share the DDL's own transaction the way it once did: it is - // written in its own system-database transaction, still before this - // COMMIT is ACKed. A seeding failure therefore still fails the DDL apply - // and the COMMIT with it, the same durability boundary + // after the DDL has executed and the schema cache reflects it. The claim + // table lives in the system database - a separate SQLite file from tm.db + // (db/autoinc_claim.go) - so the seed cannot share the DDL's own + // transaction: it is written in its own system-database transaction, + // still before this COMMIT is ACKed. A seeding failure therefore still + // fails the DDL apply and the COMMIT with it, the same durability boundary // AutoIncClaimStore.ApplyClaims relies on for the claim itself: a table // left untagged here would let a subsequent range claim be evaluated // against a missing base row. - if len(ddlTables) > 0 { - if err := tm.seedAutoIncBasesForDDL(ddlTables); err != nil { - return fmt.Errorf("failed to seed auto-increment base after DDL (txn %d): %w", txnID, err) - } + if err := tm.seedSchemaChange(tm.db, &change); err != nil { + return fmt.Errorf("failed to seed auto-increment base after DDL (txn %d): %w", txnID, err) } - return nil } +// execDDL executes one DDL intent on one connection, so the schema reads +// SchemaChange.Exec takes around it bracket exactly this statement. +func (tm *TransactionManager) execDDL(change *SchemaChange, intent *WriteIntentRecord) error { + ctx := context.Background() + conn, err := tm.db.Conn(ctx) + if err != nil { + return fmt.Errorf("failed to execute DDL statement: %w", err) + } + defer conn.Close() + return change.Exec(ctx, conn, intent.SQLStatement, intent.TableName, intent.NodeID) +} + // writeNonDMLToCDC writes DDL/LOAD DATA statements to CDC storage for streaming replication. func (tm *TransactionManager) writeNonDMLToCDC(txnID uint64, intents []*WriteIntentRecord) error { var seq uint64 diff --git a/examples/run-single-node.sh b/examples/run-single-node.sh index 7a5d61aa..7eb6b343 100755 --- a/examples/run-single-node.sh +++ b/examples/run-single-node.sh @@ -39,6 +39,7 @@ lock_wait_timeout_seconds = 50 grpc_bind_address = "0.0.0.0" grpc_port = 8080 seed_nodes = [] # No peers - single node +standalone = true # A deliberate single-node deployment: AUTO_INCREMENT claims need no peers gossip_interval_ms = 1000 gossip_fanout = 2 suspect_timeout_ms = 5000 diff --git a/examples/wordpress/marmot-config.toml b/examples/wordpress/marmot-config.toml index 75a0cb7f..58b9fdf0 100644 --- a/examples/wordpress/marmot-config.toml +++ b/examples/wordpress/marmot-config.toml @@ -20,6 +20,8 @@ grpc_bind_address = "0.0.0.0" grpc_advertise_address = "marmot:8090" grpc_port = 8090 seed_nodes = [] +# A deliberate single-node deployment: AUTO_INCREMENT claims need no peers. +standalone = true gossip_interval_ms = 1000 gossip_fanout = 2 suspect_timeout_ms = 5000 diff --git a/grpc/autoinc_claim_handler_test.go b/grpc/autoinc_claim_handler_test.go index 5c1a39b7..89d25865 100644 --- a/grpc/autoinc_claim_handler_test.go +++ b/grpc/autoinc_claim_handler_test.go @@ -19,6 +19,9 @@ func newClaimTestHandler(t *testing.T, ddl string) (*ReplicationHandler, *db.Dat dm, err := db.NewDatabaseManager(t.TempDir(), 1, clock) require.NoError(t, err) t.Cleanup(func() { dm.Close() }) + // A participant of a cluster that has merged: a new system database + // holds the node's claim votes until then. + require.NoError(t, dm.MergeAutoIncBasesAndReleaseVotes(nil)) require.NoError(t, dm.CreateDatabase("testdb")) mdb, err := dm.GetDatabase("testdb") require.NoError(t, err) @@ -85,7 +88,8 @@ func TestHandlePrepareReturnsTheStoredBaseOfARejectedClaim(t *testing.T) { } // TestHandlePrepareReturnsTheParticipantsErrorCode: a DDL this participant -// refuses with a MySQL-coded error (here the width ceiling, 1264) returns that +// refuses with a MySQL-coded error (here the width ceiling, 1264: it declares a +// TINYINT AUTO_INCREMENT column over the value 200) returns that // code on the wire, so a client whose coordinator is another node sees 1264 // rather than 1105. // @@ -93,17 +97,17 @@ func TestHandlePrepareReturnsTheStoredBaseOfARejectedClaim(t *testing.T) { // ReplicationHandler.handlePrepare. "the participant's MySQL code did not // reach the wire" fires. func TestHandlePrepareReturnsTheParticipantsErrorCode(t *testing.T) { - rh, dm := newClaimTestHandler(t, "CREATE TABLE t (id INTEGER /*M:8a*/ PRIMARY KEY, v TEXT)") + rh, dm := newClaimTestHandler(t, "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)") mdb, err := dm.GetDatabase("testdb") require.NoError(t, err) - _, err = mdb.GetWriteDB().Exec("INSERT INTO t (id, v) VALUES (200, 'x')") + _, err = mdb.GetWriteDB().Exec("INSERT INTO t (id, v) VALUES (1, 'x')") require.NoError(t, err) resp := prepareOverWire(t, rh, 101, []protocol.Statement{{ Type: protocol.StatementDDL, Database: "testdb", TableName: "t", - SQL: "ALTER TABLE t ADD COLUMN extra INTEGER", + SQL: "ALTER TABLE t ADD COLUMN seq INTEGER /*M:8a*/ DEFAULT 200", }}, unchanged) require.False(t, resp.Success) require.True(t, resp.Rejected) diff --git a/grpc/autoinc_claim_schema_gate_test.go b/grpc/autoinc_claim_schema_gate_test.go new file mode 100644 index 00000000..97d83698 --- /dev/null +++ b/grpc/autoinc_claim_schema_gate_test.go @@ -0,0 +1,190 @@ +package grpc + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/maxpert/marmot/coordinator" + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// gateNode is one participant of an in-process cluster whose remote +// participants are reached through their real ReplicationHandler, schema +// version check included. +type gateNode struct { + id uint64 + clock *hlc.Clock + dm *db.DatabaseManager + versions *db.SchemaVersionManager + handler *ReplicationHandler + wc *coordinator.WriteCoordinator +} + +// handlerFanout delivers every remote request to the target node's +// ReplicationHandler, converted exactly as GRPCReplicator converts it. +type handlerFanout struct { + mu sync.Mutex + nodes map[uint64]*gateNode + unreachable map[uint64]bool +} + +func (f *handlerFanout) setUnreachable(id uint64, unreachable bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.unreachable[id] = unreachable +} + +func (f *handlerFanout) ReplicateTransaction(ctx context.Context, nodeID uint64, req *coordinator.ReplicationRequest) (*coordinator.ReplicationResponse, error) { + f.mu.Lock() + node, unreachable := f.nodes[nodeID], f.unreachable[nodeID] + f.mu.Unlock() + if unreachable { + return nil, fmt.Errorf("node %d unreachable", nodeID) + } + grpcReq, err := transactionRequestToProto(req) + if err != nil { + return nil, err + } + resp, err := node.handler.HandleReplicateTransaction(ctx, grpcReq) + if err != nil { + return nil, err + } + return convertTransactionResponse(resp), nil +} + +// gateNodeProvider is a fixed membership of every node. +type gateNodeProvider struct{ nodes []uint64 } + +func (p gateNodeProvider) GetAliveNodes() ([]uint64, error) { return p.nodes, nil } +func (p gateNodeProvider) GetClusterSize() int { return len(p.nodes) } +func (p gateNodeProvider) GetTotalMembershipSize() int { return len(p.nodes) } +func (p gateNodeProvider) HasSeedNodes() bool { return false } + +func newGateCluster(t *testing.T, ids []uint64) (*handlerFanout, map[uint64]*gateNode) { + t.Helper() + fanout := &handlerFanout{nodes: map[uint64]*gateNode{}, unreachable: map[uint64]bool{}} + provider := gateNodeProvider{nodes: ids} + for _, id := range ids { + clock := hlc.NewClock(id) + dm, err := db.NewDatabaseManager(t.TempDir(), id, clock) + require.NoError(t, err) + t.Cleanup(func() { dm.Close() }) + // A new cluster: every node is held, and all of them release. + require.NoError(t, dm.MergeAutoIncBasesAndReleaseVotes(nil)) + dm.SetClusterMembership(provider.GetTotalMembershipSize) + require.NoError(t, dm.CreateDatabase("gate")) + systemDB, err := dm.GetDatabase(db.SystemDatabaseName) + require.NoError(t, err) + versions := db.NewSchemaVersionManager(systemDB.GetMetaStore()) + wc := coordinator.NewWriteCoordinator(id, provider, fanout, db.NewLocalReplicator(id, dm, clock), 30*time.Second, clock) + wc.SetSchemaVersionSource(versions.GetSchemaVersion) + fanout.nodes[id] = &gateNode{id: id, clock: clock, dm: dm, versions: versions, + handler: NewReplicationHandler(id, dm, clock, versions), wc: wc} + } + return fanout, fanout.nodes +} + +// gateDDL replicates ddl from node through 2PC and bumps that node's own +// schema version, as CoordinatorHandler does after a DDL commit; remote +// participants bump theirs when they apply the COMMIT. +func gateDDL(t *testing.T, node *gateNode, table, ddl string) { + t.Helper() + startTS := node.clock.Now() + txn := &coordinator.Transaction{ + ID: startTS.ToTxnID(), NodeID: node.id, StartTS: startTS, Database: "gate", + WriteConsistency: protocol.ConsistencyQuorum, + Statements: []protocol.Statement{{Type: protocol.StatementDDL, SQL: ddl, TableName: table, Database: "gate"}}, + } + require.NoError(t, node.wc.WriteTransaction(context.Background(), txn), ddl) + _, err := node.versions.IncrementSchemaVersion("gate", ddl, txn.ID) + require.NoError(t, err) +} + +// waitGate polls until check passes, failing after a deadline: a commit +// returns once a quorum applied it, so the last participant may lag. +func waitGate(t *testing.T, what string, check func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for !check() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(20 * time.Millisecond) + } +} + +func (n *gateNode) version(t *testing.T) uint64 { + v, err := n.versions.GetSchemaVersion("gate") + require.NoError(t, err) + return v +} + +func (n *gateNode) base(table string) (uint64, error) { + return db.NewAutoIncClaimStore(n.dm.GetSystemDatabase()).ReadBase("gate", table) +} + +// TestClaimFromALaggingVoterIsDeclined is R3c-12's counterexample, driven +// through the real DDL apply path and every participant's ReplicationHandler. +// +// Nodes g=1, h=2, k=3. A claim on t2, (0,64], commits on {k,g} while h is +// away, and row 50 lands in t2 on k and g. While g is away, k replicates +// DROP TABLE t and RENAME t2 TO t to {k,h}: k's t base inherits 64, h's +// stays 0 (h missed the claim and holds no row). g still has the old t, base +// 0. When h claims t from base 0, {h,g} is a majority that would grant (0,64] +// again, over row 50 that the renamed table holds on k and g. The claim +// carries h's schema version, so g, which has not applied the two DDLs, +// declines; k refuses with its base 64; h's claim lands above it. +// +// Mutation: claims carry schema version 0 (claimSchemaVersion returns 0). +// g votes with the old incarnation's base and "a claim from a lagging voter's +// base overlapped the renamed rows" fires. +func TestClaimFromALaggingVoterIsDeclined(t *testing.T) { + fanout, nodes := newGateCluster(t, []uint64{1, 2, 3}) + g, h, k := nodes[1], nodes[2], nodes[3] + ctx := context.Background() + size := func(uint64) (uint64, error) { return 64, nil } + + gateDDL(t, k, "t", "CREATE TABLE t (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + gateDDL(t, k, "t2", "CREATE TABLE t2 (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + for _, n := range []*gateNode{g, h, k} { + waitGate(t, fmt.Sprintf("node %d at schema version 2", n.id), func() bool { return n.version(t) == 2 }) + } + + fanout.setUnreachable(h.id, true) + cBase, cSize, err := k.wc.ClaimRange(ctx, "gate", "t2", 0, size) + require.NoError(t, err) + require.Equal(t, uint64(0), cBase) + for _, n := range []*gateNode{g, k} { + waitGate(t, fmt.Sprintf("node %d t2 base 64", n.id), func() bool { b, err := n.base("t2"); return err == nil && b == cBase+cSize }) + mdb, err := n.dm.GetDatabase("gate") + require.NoError(t, err) + _, err = mdb.GetWriteDB().Exec("INSERT INTO t2 (id, v) VALUES (50, 'moved')") + require.NoError(t, err) + } + fanout.setUnreachable(h.id, false) + + fanout.setUnreachable(g.id, true) + gateDDL(t, k, "t", "DROP TABLE t") + gateDDL(t, k, "t2", "ALTER TABLE t2 RENAME TO t") + waitGate(t, "node 2 at schema version 4", func() bool { return h.version(t) == 4 }) + fanout.setUnreachable(g.id, false) + + hBase, err := h.base("t") + require.NoError(t, err) + require.Equal(t, uint64(0), hBase, "h must hold the low base for the sequence to mean anything") + gBase, err := g.base("t") + require.NoError(t, err) + require.Equal(t, uint64(0), gBase, "g must hold the old incarnation's base") + require.Equal(t, uint64(2), g.version(t), "g must lag the two DDLs") + + dBase, dSize, err := h.wc.ClaimRange(ctx, "gate", "t", 0, size) + require.NoError(t, err) + require.True(t, dBase >= 50 || dBase+dSize < 50, + "a claim from a lagging voter's base overlapped the renamed rows: granted (%d,%d] over id 50", dBase, dBase+dSize) +} diff --git a/grpc/autoinc_merge.go b/grpc/autoinc_merge.go new file mode 100644 index 00000000..f5d2f0a4 --- /dev/null +++ b/grpc/autoinc_merge.go @@ -0,0 +1,436 @@ +package grpc + +import ( + "context" + "fmt" + "slices" + "time" + + "github.com/maxpert/marmot/coordinator" + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/protocol" + "github.com/rs/zerolog/log" +) + +// autoIncMergeInterval is how often a node whose claim votes are held retries +// the merge that releases them, and autoIncMergePeerTimeout bounds each +// peer's answer within one round. +// +// autoIncBaseSyncInterval is how often an unheld node raises its claim bases +// to the maximum every alive peer reports (the membership backstop). Claim +// safety is proved for a stable membership; after a membership change a +// claim committed under the old membership may be known only to members a +// new quorum can avoid. Every node pulling every peer's bases bounds that +// window to one interval. +const ( + autoIncMergeInterval = 2 * time.Second + autoIncMergePeerTimeout = 5 * time.Second + autoIncBaseSyncInterval = 10 * time.Second +) + +// GetAutoIncBases serves this node's AUTO_INCREMENT claim bases +// (db.AutoIncClaimStore.Bases) to a peer's vote-hold merge or base sync. +func (s *Server) GetAutoIncBases(ctx context.Context, req *AutoIncBasesRequest) (*AutoIncBasesResponse, error) { + s.mu.RLock() + dbManager := s.dbManager + s.mu.RUnlock() + if dbManager == nil { + return nil, fmt.Errorf("database manager not initialized") + } + + bases, held, err := dbManager.AutoIncBases() + if err != nil { + return nil, err + } + resp := &AutoIncBasesResponse{VotesHeld: held, Bases: make([]*AutoIncBase, len(bases))} + for i, b := range bases { + resp.Bases[i] = &AutoIncBase{Database: b.Database, Table: b.Table, Base: b.Base} + } + return resp, nil +} + +// SyncAutoIncBases runs one claim base sync on this node now +// (syncAutoIncBasesOnce) and reports which members it reached. +func (s *Server) SyncAutoIncBases(ctx context.Context, req *AutoIncSyncRequest) (*AutoIncSyncResponse, error) { + s.mu.RLock() + dbManager, registry, gossip := s.dbManager, s.registry, s.gossip + s.mu.RUnlock() + if dbManager == nil || registry == nil || gossip == nil || gossip.GetClient() == nil { + return nil, fmt.Errorf("cluster not initialized") + } + report, err := syncAutoIncBasesOnce(ctx, s.nodeID, dbManager, registry, peerAutoIncBases(s.nodeID, gossip.GetClient())) + if err != nil { + return nil, err + } + return report.toProto(), nil +} + +// autoIncBaseStore is the part of db.DatabaseManager the merge needs. +type autoIncBaseStore interface { + AutoIncVotesHeld() (bool, error) + MergeAutoIncBasesAndReleaseVotes(bases []db.AutoIncBase) error + RaiseAutoIncBases(bases []db.AutoIncBase) error +} + +// fetchAutoIncBases asks one peer for its bases. +type fetchAutoIncBases func(ctx context.Context, nodeID uint64) (*AutoIncBasesResponse, error) + +// autoIncMergeSafe reports whether a held node may merge and release its +// votes, in a membership of membership nodes, after unheld peers answered +// with their votes not held and held nodes - this node included - answered +// held. Every other member did not answer. +// +// Failure model: at most f' = max(N-Q, 1) members have lost claim state they +// voted with and not yet merged, where N is the membership and Q its quorum +// (the 1 because this node itself is held and so may be one of them). +// +// Proof. Take any committed claim C and its ACK set A, |A| >= Q. A node is +// held only after losing its claim state, and a held node never votes, so an +// ACKer now held lost state after ACKing: |A n held| <= min(held, f'). Every +// ACKer that did not answer is among the unanswered. So when +// unanswered + min(held, f') < Q, some unheld peer that answered ACKed C. +// That peer's base is at or above C's end - an unheld node either kept the +// base it ACKed with, or became unheld through a merge that met this same +// condition - so the maximum this node merges covers C. +// +// With only this node held the condition is the classic one: answers from +// ceil(N/2) unheld peers of the full membership. Held peers never count as +// unheld. When every member answers, it always holds: with no unanswered +// member the sum is at most f' < Q, which is also why a cluster whose every +// node is held - a new cluster - releases on its own: under the failure model +// fewer than Q of its nodes ever voted, so no claim ever committed. A node +// whose membership is 1 never releases this way (0 + 1 < 1 fails): its view +// may be one only because it has not learned its cluster yet. Only a node +// configured as standalone releases alone. +func autoIncMergeSafe(membership, unheld, held int, standalone bool) bool { + if membership <= 1 { + return standalone + } + quorum := coordinator.QuorumSize(protocol.ConsistencyQuorum, membership) + lost := max(membership-quorum, 1) + unanswered := max(membership-unheld-held, 0) + return unanswered+min(held, lost) < quorum +} + +// autoIncAnswers is what one round of asking every peer for its bases +// learned. +type autoIncAnswers struct { + bases []db.AutoIncBase + unheldPeers int + heldPeers int + answered []uint64 +} + +// gatherAutoIncBases asks every peer for its bases and returns the maximum +// reported for each table - held peers included, since a higher base only +// ever raises the merge - with how many unheld and held peers answered. +func gatherAutoIncBases(ctx context.Context, peers []uint64, fetch fetchAutoIncBases) autoIncAnswers { + var answers autoIncAnswers + merged := make(map[[2]string]uint64) + for _, peer := range peers { + peerCtx, cancel := context.WithTimeout(ctx, autoIncMergePeerTimeout) + resp, err := fetch(peerCtx, peer) + cancel() + if err != nil { + continue + } + answers.answered = append(answers.answered, peer) + if resp.GetVotesHeld() { + answers.heldPeers++ + } else { + answers.unheldPeers++ + } + for _, b := range resp.GetBases() { + key := [2]string{b.GetDatabase(), b.GetTable()} + merged[key] = max(merged[key], b.GetBase()) + } + } + answers.bases = make([]db.AutoIncBase, 0, len(merged)) + for key, base := range merged { + answers.bases = append(answers.bases, db.AutoIncBase{Database: key[0], Table: key[1], Base: base}) + } + return answers +} + +// mergeAutoIncBasesOnce runs one merge round: it asks every peer and, once +// autoIncMergeSafe holds for the answers, raises this node's bases to the +// maximum any answer reported and releases its votes. It reports whether the +// votes are now released. +func mergeAutoIncBasesOnce(ctx context.Context, store autoIncBaseStore, peers []uint64, membership int, standalone bool, fetch fetchAutoIncBases) (bool, error) { + held, err := store.AutoIncVotesHeld() + if err != nil || !held { + return !held, err + } + answers := gatherAutoIncBases(ctx, peers, fetch) + // This node is held, so it counts among the held. + if !autoIncMergeSafe(membership, answers.unheldPeers, answers.heldPeers+1, standalone) { + return false, nil + } + if err := store.MergeAutoIncBasesAndReleaseVotes(answers.bases); err != nil { + return false, err + } + return true, nil +} + +// alivePeers lists every alive member other than nodeID. +func alivePeers(registry autoIncMembership, nodeID uint64) []uint64 { + var peers []uint64 + for _, node := range registry.GetAlive() { + if node.NodeId != nodeID { + peers = append(peers, node.NodeId) + } + } + return peers +} + +// peerAutoIncBases asks peers for their bases through client. +func peerAutoIncBases(nodeID uint64, client *Client) fetchAutoIncBases { + return func(ctx context.Context, peer uint64) (*AutoIncBasesResponse, error) { + c, err := client.GetClient(peer) + if err != nil { + return nil, err + } + return c.GetAutoIncBases(ctx, &AutoIncBasesRequest{RequestingNodeId: nodeID}) + } +} + +// RunAutoIncBaseMerge releases this node's held claim votes once it has merged +// claim bases from enough of its membership (autoIncMergeSafe), retrying +// every autoIncMergeInterval until then. From then on - at once on a node +// whose votes were never held - it runs the membership backstop: one sync +// immediately, then one every autoIncBaseSyncInterval, until ctx ends. +// +// standalone is the node's cluster.standalone setting: only such a node +// releases while it is its own whole membership, and only while it has never +// seen another member (standaloneRelease). +func RunAutoIncBaseMerge(ctx context.Context, nodeID uint64, store autoIncBaseStore, registry *NodeRegistry, client *Client, standalone bool) { + runAutoIncBaseMerge(ctx, nodeID, store, registry, standalone, peerAutoIncBases(nodeID, client)) +} + +// runAutoIncBaseMerge is RunAutoIncBaseMerge over an explicit membership and +// peer fetch. +func runAutoIncBaseMerge(ctx context.Context, nodeID uint64, store autoIncBaseStore, registry autoIncMembership, standalone bool, fetch fetchAutoIncBases) { + if !holdUntilMerged(ctx, nodeID, store, registry, standalone, fetch) { + return + } + ticker := time.NewTicker(autoIncBaseSyncInterval) + defer ticker.Stop() + for { + if _, err := syncAutoIncBasesOnce(ctx, nodeID, store, registry, fetch); err != nil { + log.Warn().Err(err).Msg("AUTO_INCREMENT claim base sync failed; retrying") + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +// holdUntilMerged runs merge rounds every autoIncMergeInterval until this +// node's votes are released. It reports false when ctx ended first. +func holdUntilMerged(ctx context.Context, nodeID uint64, store autoIncBaseStore, registry autoIncMembership, standalone bool, fetch fetchAutoIncBases) bool { + if held, err := store.AutoIncVotesHeld(); err == nil && !held { + return true + } + ticker := time.NewTicker(autoIncMergeInterval) + defer ticker.Stop() + release := standaloneRelease{configured: standalone} + warned := false + for { + membership := registry.Count() + released, err := mergeAutoIncBasesOnce(ctx, store, alivePeers(registry, nodeID), membership, release.allowed(membership), fetch) + if err != nil { + log.Warn().Err(err).Msg("AUTO_INCREMENT claim base merge failed; retrying") + } + if released { + log.Info().Msg("AUTO_INCREMENT claim votes released after merging claim bases") + return true + } + if !warned && err == nil { + warned = true + log.Warn().Int("membership", membership).Bool("standalone", standalone). + Msg("AUTO_INCREMENT claim votes held until claim bases are merged from enough cluster members; " + + "narrow AUTO_INCREMENT inserts may return 1205 until then. A single-node deployment sets " + + "cluster.standalone = true; an operator can force a release with " + + "POST /admin/cluster/autoinc/release-votes?accept_risk=" + ForceReleaseRiskToken) + } + select { + case <-ctx.Done(): + return false + case <-ticker.C: + } + } +} + +// standaloneRelease decides whether a node configured as standalone may +// release its votes alone. A standalone node joins no cluster, so a node that +// has ever seen another member is not one, whatever its configuration says: +// releasing alone could lose a claim that member holds. +type standaloneRelease struct { + configured bool + sawPeers bool + logged bool +} + +// allowed reports whether the node may release while its membership is +// membership, remembering whether it has ever seen another member. +func (r *standaloneRelease) allowed(membership int) bool { + if membership > 1 { + r.sawPeers = true + } + if r.configured && r.sawPeers && !r.logged { + r.logged = true + log.Warn().Int("membership", membership). + Msg("cluster.standalone is set but this node has seen other cluster members; it will not release " + + "its AUTO_INCREMENT claim votes alone and merges claim bases from its cluster instead") + } + return r.configured && !r.sawPeers +} + +// AutoIncSyncReport is what one claim base sync on a node learned. +type AutoIncSyncReport struct { + NodeID uint64 + VotesHeld bool + // Members is the node's membership, itself included. + Members []uint64 + // Reached is the members whose bases the sync merged, itself included. + Reached []uint64 +} + +// ReachedEveryMember reports whether the sync merged the bases of every +// member of the node's membership. +func (r AutoIncSyncReport) ReachedEveryMember() bool { + reached := make(map[uint64]bool, len(r.Reached)) + for _, id := range r.Reached { + reached[id] = true + } + for _, id := range r.Members { + if !reached[id] { + return false + } + } + return true +} + +func (r AutoIncSyncReport) toProto() *AutoIncSyncResponse { + return &AutoIncSyncResponse{NodeId: r.NodeID, VotesHeld: r.VotesHeld, Members: r.Members, Reached: r.Reached} +} + +// autoIncMembership is the part of NodeRegistry the merge and the sync need. +type autoIncMembership interface { + Count() int + MemberIDs() []uint64 + GetAlive() []*NodeState +} + +// syncAutoIncBasesOnce asks every alive peer - not a quorum, every one, held +// peers included - for its bases and raises this node's bases to the maximum +// reported, in one transaction. Raising is always safe (db.RaiseBases), so no +// count of answers gates it. +func syncAutoIncBasesOnce(ctx context.Context, nodeID uint64, store autoIncBaseStore, registry autoIncMembership, fetch fetchAutoIncBases) (AutoIncSyncReport, error) { + members := registry.MemberIDs() + answers := gatherAutoIncBases(ctx, alivePeers(registry, nodeID), fetch) + if err := store.RaiseAutoIncBases(answers.bases); err != nil { + return AutoIncSyncReport{}, err + } + held, err := store.AutoIncVotesHeld() + if err != nil { + return AutoIncSyncReport{}, err + } + reached := append([]uint64{nodeID}, answers.answered...) + slices.Sort(reached) + return AutoIncSyncReport{NodeID: nodeID, VotesHeld: held, Members: members, Reached: reached}, nil +} + +// syncAutoIncBasesEverywhere runs a claim base sync on this node and on every +// other member of its membership - alive or not - and returns every node's +// report, a member that could not be asked included with its error. +func syncAutoIncBasesEverywhere(ctx context.Context, nodeID uint64, store autoIncBaseStore, registry autoIncMembership, fetch fetchAutoIncBases, ask askAutoIncSync) []AutoIncSyncOutcome { + local, err := syncAutoIncBasesOnce(ctx, nodeID, store, registry, fetch) + outcomes := []AutoIncSyncOutcome{{NodeID: nodeID, Report: local, Err: err}} + for _, member := range registry.MemberIDs() { + if member == nodeID { + continue + } + memberCtx, cancel := context.WithTimeout(ctx, autoIncSyncMemberTimeout) + resp, err := ask(memberCtx, member) + cancel() + outcome := AutoIncSyncOutcome{NodeID: member, Err: err} + if err == nil { + outcome.Report = AutoIncSyncReport{NodeID: resp.GetNodeId(), VotesHeld: resp.GetVotesHeld(), Members: resp.GetMembers(), Reached: resp.GetReached()} + } + outcomes = append(outcomes, outcome) + } + return outcomes +} + +// autoIncSyncMemberTimeout bounds one member's sync when an operator runs +// it everywhere: the member asks each of its own peers within +// autoIncMergePeerTimeout. +const autoIncSyncMemberTimeout = 3 * autoIncMergePeerTimeout + +// askAutoIncSync asks one member to run a sync now. +type askAutoIncSync func(ctx context.Context, nodeID uint64) (*AutoIncSyncResponse, error) + +// AutoIncSyncOutcome is one member's answer to a sync run everywhere. +type AutoIncSyncOutcome struct { + NodeID uint64 + Report AutoIncSyncReport + Err error +} + +// Complete reports whether the member answered, is unheld and reached every +// member of its membership: the state the membership-change procedure waits +// for before the next change. +func (o AutoIncSyncOutcome) Complete() bool { + return o.Err == nil && !o.Report.VotesHeld && o.Report.ReachedEveryMember() +} + +// SyncAutoIncBasesEverywhere runs the membership backstop's sync now on this +// node and on every member, for the operator's membership-change procedure: +// change one member, wait until it is ALIVE and unheld, sync - re-run until +// the report is complete, since a sync gathers only from peers a node sees +// ALIVE - confirm every outcome is Complete, then make the next change. +func SyncAutoIncBasesEverywhere(ctx context.Context, nodeID uint64, store autoIncBaseStore, registry *NodeRegistry, client *Client) []AutoIncSyncOutcome { + ask := func(ctx context.Context, member uint64) (*AutoIncSyncResponse, error) { + c, err := client.GetClient(member) + if err != nil { + return nil, err + } + return c.SyncAutoIncBases(ctx, &AutoIncSyncRequest{RequestingNodeId: nodeID}) + } + return syncAutoIncBasesEverywhere(ctx, nodeID, store, registry, peerAutoIncBases(nodeID, client), ask) +} + +// ForceReleaseRiskToken is the value an operator must pass to force a vote +// release, naming the risk they accept. +const ForceReleaseRiskToken = "duplicate-ids" + +// ForceReleaseAutoIncVotes merges the bases of every peer that answers and +// releases this node's votes whether or not autoIncMergeSafe holds. It is the +// operator's way out of a cluster held beyond the failure model - for +// example, more than N-Q members rebuilt at once, or a membership that cannot +// be reached. It is unsafe exactly when a member that holds a newer claim +// than any answer reports did not answer: this node may then grant a range +// that overlaps it, and the same ids can be issued twice. It returns how many +// peers answered and whether this node's votes were held at all. +func ForceReleaseAutoIncVotes(ctx context.Context, nodeID uint64, store autoIncBaseStore, registry *NodeRegistry, client *Client) (answered int, wasHeld bool, err error) { + return forceReleaseAutoIncVotes(ctx, store, alivePeers(registry, nodeID), peerAutoIncBases(nodeID, client)) +} + +// forceReleaseAutoIncVotes is ForceReleaseAutoIncVotes over explicit peers. +func forceReleaseAutoIncVotes(ctx context.Context, store autoIncBaseStore, peers []uint64, fetch fetchAutoIncBases) (answered int, wasHeld bool, err error) { + held, err := store.AutoIncVotesHeld() + if err != nil || !held { + return 0, false, err + } + answers := gatherAutoIncBases(ctx, peers, fetch) + if err := store.MergeAutoIncBasesAndReleaseVotes(answers.bases); err != nil { + return 0, true, err + } + log.Warn().Int("peers_answered", answers.unheldPeers+answers.heldPeers). + Msg("AUTO_INCREMENT claim votes released by operator without a safe merge; ids may be issued twice " + + "if a member holding a newer claim did not answer") + return answers.unheldPeers + answers.heldPeers, true, nil +} diff --git a/grpc/autoinc_merge_test.go b/grpc/autoinc_merge_test.go new file mode 100644 index 00000000..79487a2e --- /dev/null +++ b/grpc/autoinc_merge_test.go @@ -0,0 +1,370 @@ +package grpc + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" +) + +// fakeBaseStore records the merge a held node performs and the raises the +// membership backstop performs. +type fakeBaseStore struct { + mu sync.Mutex + held bool + merged []db.AutoIncBase + merges int + raised []db.AutoIncBase + raises int +} + +func (s *fakeBaseStore) RaiseAutoIncBases(bases []db.AutoIncBase) error { + s.mu.Lock() + defer s.mu.Unlock() + s.raised = bases + s.raises++ + return nil +} + +func (s *fakeBaseStore) raiseCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.raises +} + +func (s *fakeBaseStore) AutoIncVotesHeld() (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.held, nil +} + +func (s *fakeBaseStore) MergeAutoIncBasesAndReleaseVotes(bases []db.AutoIncBase) error { + s.mu.Lock() + defer s.mu.Unlock() + s.merged = bases + s.merges++ + s.held = false + return nil +} + +func peerAnswers(answers map[uint64]*AutoIncBasesResponse) fetchAutoIncBases { + return func(_ context.Context, peer uint64) (*AutoIncBasesResponse, error) { + resp, ok := answers[peer] + if !ok { + return nil, errors.New("unreachable") + } + return resp, nil + } +} + +func basesByTable(bases []db.AutoIncBase) map[string]uint64 { + out := make(map[string]uint64, len(bases)) + for _, b := range bases { + out[b.Database+"."+b.Table] = b.Base + } + return out +} + +// TestAutoIncMergeSafe pins the release condition: unanswered + min(held, +// max(N-Q, 1)) < Q, with this node counted among the held. +// +// Mutation: drop the held nodes from the bound (min(held, f') -> 0). Among +// others, the N=5 row with two held and one unanswered releases, and "B's +// H1" is reported. +func TestAutoIncMergeSafe(t *testing.T) { + cases := []struct { + name string + membership, unheld, held int + standalone, want bool + }{ + {"membership of one, not standalone (B d)", 1, 0, 1, false, false}, + {"membership of one, standalone", 1, 0, 1, true, true}, + {"two nodes, the peer answers unheld", 2, 1, 1, false, true}, + {"two nodes, the peer does not answer", 2, 0, 1, false, false}, + {"three nodes, both peers unheld", 3, 2, 1, false, true}, + {"three nodes, one peer silent: ceil(3/2) not met", 3, 1, 1, false, false}, + {"three nodes, all held (new cluster)", 3, 0, 3, false, true}, + {"three nodes, one released, one held", 3, 1, 2, false, true}, + {"four nodes, two of three peers unheld", 4, 2, 1, false, true}, + {"five nodes, three unheld: ceil(5/2)", 5, 3, 1, false, true}, + {"five nodes, two unheld: not ceil(5/2)", 5, 2, 1, false, false}, + {"five nodes, B's H1: two held, one silent ACKer", 5, 2, 2, false, false}, + {"five nodes, two held, all answer", 5, 3, 2, false, true}, + {"five nodes, all held", 5, 0, 5, false, true}, + } + for _, tc := range cases { + if got := autoIncMergeSafe(tc.membership, tc.unheld, tc.held, tc.standalone); got != tc.want { + t.Errorf("%s: autoIncMergeSafe = %v, want %v", tc.name, got, tc.want) + } + } +} + +// TestMergeAutoIncBasesOnceIntersectsEveryCommittedClaim is reviewer B's N=5 +// counterexample: claim C committed on {n1,n2,n3}; n2 and n3 were rebuilt and +// are held; n1 is slow. n2 (this node) hears held n3 and unheld n4 and n5, +// which both missed C. It must stay held. Once n1 answers it releases with +// C's base. +// +// Mutation: count a peer that answered held as unheld (gatherAutoIncBases). +// The first round releases on n3, n4 and n5 and "released without hearing +// from any ACKer of C" fires. +func TestMergeAutoIncBasesOnceIntersectsEveryCommittedClaim(t *testing.T) { + ctx := context.Background() + answers := map[uint64]*AutoIncBasesResponse{ + 3: {VotesHeld: true}, + 4: {Bases: []*AutoIncBase{{Database: "d", Table: "t", Base: 10}}}, + 5: {Bases: []*AutoIncBase{{Database: "d", Table: "t", Base: 10}, {Database: "d", Table: "u", Base: 5}}}, + } + store := &fakeBaseStore{held: true} + released, err := mergeAutoIncBasesOnce(ctx, store, []uint64{1, 3, 4, 5}, 5, false, peerAnswers(answers)) + require.NoError(t, err) + require.False(t, released, "released without hearing from any ACKer of C") + require.Equal(t, 0, store.merges) + + answers[1] = &AutoIncBasesResponse{Bases: []*AutoIncBase{{Database: "d", Table: "t", Base: 64}}} + released, err = mergeAutoIncBasesOnce(ctx, store, []uint64{1, 3, 4, 5}, 5, false, peerAnswers(answers)) + require.NoError(t, err) + require.True(t, released, "every member answered") + require.Equal(t, map[string]uint64{"d.t": 64, "d.u": 5}, basesByTable(store.merged), + "the merge must take every table's maximum across the answers") +} + +// TestMergeAutoIncBasesOnceReleasesANewCluster is the all-held case: every +// member of a new cluster is held, and once all of them answer each one +// releases, including one that asks after another already released. +// +// Mutation: do not count this node among the held (answers.heldPeers instead +// of answers.heldPeers+1). It then counts as a member that did not answer, +// and "a cluster whose every member is held never releases" fires. +func TestMergeAutoIncBasesOnceReleasesANewCluster(t *testing.T) { + ctx := context.Background() + store := &fakeBaseStore{held: true} + allHeld := map[uint64]*AutoIncBasesResponse{2: {VotesHeld: true}, 3: {VotesHeld: true}} + released, err := mergeAutoIncBasesOnce(ctx, store, []uint64{2, 3}, 3, false, peerAnswers(allHeld)) + require.NoError(t, err) + require.True(t, released, "a cluster whose every member is held never releases") + + store = &fakeBaseStore{held: true} + oneReleased := map[uint64]*AutoIncBasesResponse{2: {}, 3: {VotesHeld: true}} + released, err = mergeAutoIncBasesOnce(ctx, store, []uint64{2, 3}, 3, false, peerAnswers(oneReleased)) + require.NoError(t, err) + require.True(t, released, "a node asking after a peer released stayed held") + + store = &fakeBaseStore{held: true} + released, err = mergeAutoIncBasesOnce(ctx, store, nil, 1, false, peerAnswers(nil)) + require.NoError(t, err) + require.False(t, released, "a node that knows only itself released without being standalone") +} + +// TestMergeAutoIncBasesOnceLeavesAnUnheldNodeAlone pins that a node whose +// votes are not held never merges. +func TestMergeAutoIncBasesOnceLeavesAnUnheldNodeAlone(t *testing.T) { + store := &fakeBaseStore{} + released, err := mergeAutoIncBasesOnce(context.Background(), store, []uint64{1}, 3, false, + func(context.Context, uint64) (*AutoIncBasesResponse, error) { + t.Fatal("an unheld node asked a peer for its bases") + return nil, nil + }) + require.NoError(t, err) + require.True(t, released) + require.Equal(t, 0, store.merges) +} + +// TestGetAutoIncBasesServesBasesAndHold pins the RPC a held peer merges from: +// it returns every base this node holds and whether its own votes are held. +// +// Mutation: drop VotesHeld from the response. A held node's bases would then +// count toward a peer's majority, and "the responder's hold did not reach +// the wire" fires. +func TestGetAutoIncBasesServesBasesAndHold(t *testing.T) { + dm, err := db.NewDatabaseManager(t.TempDir(), 1, hlc.NewClock(1)) + require.NoError(t, err) + defer dm.Close() + require.NoError(t, db.NewAutoIncClaimStore(dm.GetSystemDatabase()).Seed("d", "t", 77, 1)) + + s := &Server{} + s.SetDatabaseManager(dm) + resp, err := s.GetAutoIncBases(context.Background(), &AutoIncBasesRequest{RequestingNodeId: 2}) + require.NoError(t, err) + require.True(t, resp.GetVotesHeld(), "the responder's hold did not reach the wire") + require.Len(t, resp.GetBases(), 1) + require.Equal(t, "d", resp.GetBases()[0].GetDatabase()) + require.Equal(t, "t", resp.GetBases()[0].GetTable()) + require.Equal(t, uint64(77), resp.GetBases()[0].GetBase()) +} + +// TestForceReleaseAutoIncVotesReleasesWithoutASafeMerge pins the operator's +// way out: it merges whatever the answering peers report and releases even +// where autoIncMergeSafe does not hold, and it leaves an unheld node alone. +// +// Mutation: gate forceReleaseAutoIncVotes on autoIncMergeSafe. With one of +// four peers answering the node stays held and "the operator release did not +// release" fires. +func TestForceReleaseAutoIncVotesReleasesWithoutASafeMerge(t *testing.T) { + store := &fakeBaseStore{held: true} + answers := map[uint64]*AutoIncBasesResponse{2: {Bases: []*AutoIncBase{{Database: "d", Table: "t", Base: 30}}}} + answered, wasHeld, err := forceReleaseAutoIncVotes(context.Background(), store, []uint64{2, 3, 4, 5}, peerAnswers(answers)) + require.NoError(t, err) + require.False(t, store.held, "the operator release did not release") + require.True(t, wasHeld) + require.Equal(t, 1, answered) + require.Equal(t, map[string]uint64{"d.t": 30}, basesByTable(store.merged)) + + answered, wasHeld, err = forceReleaseAutoIncVotes(context.Background(), &fakeBaseStore{}, []uint64{2}, peerAnswers(answers)) + require.NoError(t, err) + require.False(t, wasHeld) + require.Zero(t, answered) +} + +// fakeMembership is a membership whose every member is alive unless listed +// in down. +type fakeMembership struct { + members []uint64 + down map[uint64]bool +} + +func (m fakeMembership) Count() int { return len(m.members) } +func (m fakeMembership) MemberIDs() []uint64 { return m.members } +func (m fakeMembership) GetAlive() []*NodeState { + var alive []*NodeState + for _, id := range m.members { + if !m.down[id] { + alive = append(alive, &NodeState{NodeId: id, Status: NodeStatus_ALIVE}) + } + } + return alive +} + +// TestSyncAutoIncBasesRaisesFromEveryAlivePeer pins the membership backstop +// (R3c-8a) on fix1-B C1's shape: after n4 joined, claim C = (0,64] is known +// only to n1, and a quorum of the grown membership (n2, n3, n4) can avoid it. +// n4's sync asks every alive peer, not a quorum, so it raises to C's end as +// soon as n1 answers, and reports that it reached every member. +// +// Mutation: gatherAutoIncBases stops after a quorum of answers. n1 is asked +// last and "the backstop missed a claim only one member knows" fires. +func TestSyncAutoIncBasesRaisesFromEveryAlivePeer(t *testing.T) { + ctx := context.Background() + answers := map[uint64]*AutoIncBasesResponse{ + 2: {Bases: []*AutoIncBase{{Database: "d", Table: "t", Base: 10}}}, + 3: {Bases: []*AutoIncBase{{Database: "d", Table: "t", Base: 10}}, VotesHeld: true}, + 1: {Bases: []*AutoIncBase{{Database: "d", Table: "t", Base: 64}, {Database: "d", Table: "u", Base: 7}}}, + } + members := fakeMembership{members: []uint64{2, 3, 4, 1}} + store := &fakeBaseStore{} + + report, err := syncAutoIncBasesOnce(ctx, 4, store, members, peerAnswers(answers)) + require.NoError(t, err) + require.Equal(t, map[string]uint64{"d.t": 64, "d.u": 7}, basesByTable(store.raised), + "the backstop missed a claim only one member knows") + require.Equal(t, 0, store.merges, "the backstop must never touch the vote hold") + require.Equal(t, []uint64{1, 2, 3, 4}, report.Reached) + require.True(t, report.ReachedEveryMember()) + require.False(t, report.VotesHeld) + + // n1 down: the sync still raises from the rest, and says it did not + // reach every member. + members.down = map[uint64]bool{1: true} + report, err = syncAutoIncBasesOnce(ctx, 4, store, members, peerAnswers(answers)) + require.NoError(t, err) + require.False(t, report.ReachedEveryMember(), "a sync that missed a member reported reaching every one") + require.Equal(t, []uint64{2, 3, 4}, report.Reached) + + // An alive member that does not answer is not reached either. + members.down = nil + delete(answers, 1) + report, err = syncAutoIncBasesOnce(ctx, 4, store, members, peerAnswers(answers)) + require.NoError(t, err) + require.False(t, report.ReachedEveryMember()) +} + +// TestReleasedNodeSyncsAtOnce pins that a node whose votes are released runs +// the backstop immediately, not an interval later: a node that just merged +// is exactly one whose membership just changed. +// +// Mutation: wait for the first ticker tick before the first sync in +// runAutoIncBaseMerge. No raise happens within the deadline, and "a released +// node did not sync at once" fires. +func TestReleasedNodeSyncsAtOnce(t *testing.T) { + answers := map[uint64]*AutoIncBasesResponse{ + 2: {Bases: []*AutoIncBase{{Database: "d", Table: "t", Base: 30}}}, + 3: {Bases: []*AutoIncBase{{Database: "d", Table: "t", Base: 50}}}, + } + store := &fakeBaseStore{held: true} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + runAutoIncBaseMerge(ctx, 1, store, fakeMembership{members: []uint64{1, 2, 3}}, false, peerAnswers(answers)) + close(done) + }() + defer func() { + cancel() + <-done + }() + + deadline := time.Now().Add(autoIncBaseSyncInterval / 2) + for store.raiseCount() == 0 { + if time.Now().After(deadline) { + t.Fatal("a released node did not sync at once") + } + time.Sleep(10 * time.Millisecond) + } + held, err := store.AutoIncVotesHeld() + require.NoError(t, err) + require.False(t, held) +} + +// TestStandaloneNeverReleasesAloneAfterSeeingPeers pins R3c-11: a node +// configured standalone releases alone only while it has never seen another +// member. One that did is a member of a cluster, whatever its configuration +// says, and releasing alone could lose a claim that member holds. +// +// Mutation: allowed returns r.configured. "a standalone node that saw peers +// released alone" fires. +func TestStandaloneNeverReleasesAloneAfterSeeingPeers(t *testing.T) { + release := standaloneRelease{configured: true} + require.True(t, release.allowed(1), "a standalone node that never saw a peer must release alone") + require.False(t, release.allowed(3), "a standalone node that sees peers released alone") + require.False(t, release.allowed(1), "a standalone node that saw peers released alone") + + notStandalone := standaloneRelease{} + require.False(t, notStandalone.allowed(1)) +} + +// TestSyncAutoIncBasesEverywhereReportsEveryMember pins the operator's sync +// command (R3c-8b): it syncs locally and asks every member, alive or not, +// and an outcome is Complete only for a member that answered, is unheld and +// reached every member. +// +// Mutation: skip members that are not alive in syncAutoIncBasesEverywhere. +// The outcome for the down member 3 is missing and "every member must be +// reported" fires. +func TestSyncAutoIncBasesEverywhereReportsEveryMember(t *testing.T) { + members := fakeMembership{members: []uint64{1, 2, 3}, down: map[uint64]bool{3: true}} + fetch := peerAnswers(map[uint64]*AutoIncBasesResponse{2: {}}) + ask := func(_ context.Context, member uint64) (*AutoIncSyncResponse, error) { + if member == 2 { + return &AutoIncSyncResponse{NodeId: 2, Members: []uint64{1, 2, 3}, Reached: []uint64{1, 2, 3}}, nil + } + return nil, errors.New("unreachable") + } + + outcomes := syncAutoIncBasesEverywhere(context.Background(), 1, &fakeBaseStore{}, members, fetch, ask) + require.Len(t, outcomes, 3, "every member must be reported") + byNode := map[uint64]AutoIncSyncOutcome{} + for _, o := range outcomes { + byNode[o.NodeID] = o + } + require.False(t, byNode[1].Complete(), "node 1 did not reach the down member 3") + require.True(t, byNode[2].Complete()) + require.Error(t, byNode[3].Err) + require.False(t, byNode[3].Complete()) + + held := AutoIncSyncOutcome{NodeID: 2, Report: AutoIncSyncReport{VotesHeld: true, Members: []uint64{2}, Reached: []uint64{2}}} + require.False(t, held.Complete(), "a held member is not ready for the next membership change") +} diff --git a/grpc/delta_sync.go b/grpc/delta_sync.go index b962625b..95962708 100644 --- a/grpc/delta_sync.go +++ b/grpc/delta_sync.go @@ -343,7 +343,7 @@ func (ds *DeltaSyncClient) applyChangeEvent(ctx context.Context, event *ChangeEv return fmt.Errorf("failed to begin transaction: %w", err) } defer tx.Rollback() - hasDDL := false + var schemaChange db.SchemaChange for _, stmt := range event.Statements { // Check for CDC data (RowChange payload) @@ -386,10 +386,9 @@ func (ds *DeltaSyncClient) applyChangeEvent(ctx context.Context, event *ChangeEv // Fail the sync so anti-entropy knows to try again or use snapshot return fmt.Errorf("statement has no SQL and no CDC data (table=%s, type=%d) - CDC data may have been lost during serialization", stmt.TableName, stmt.Type) } - if err := db.ApplyDDLSQLInTx(ctx, tx, sql); err != nil { + if err := mdb.ApplyReplayedDDL(ctx, tx, sql, stmt.TableName, event.Timestamp.GetNodeId(), &schemaChange); err != nil { return fmt.Errorf("failed to execute statement: %w", err) } - hasDDL = true log.Debug(). Str("sql_prefix", func() string { if len(sql) > 50 { @@ -400,6 +399,10 @@ func (ds *DeltaSyncClient) applyChangeEvent(ctx context.Context, event *ChangeEv Msg("DELTA-SYNC: Executed SQL") } + if err := mdb.FinishReplayedSchemaChange(tx, &schemaChange); err != nil { + return fmt.Errorf("failed to finish replayed DDL: %w", err) + } + // Commit the transaction if err := db.MarkSQLiteTxnApplied(tx, event.TxnId, HLCToTimestamp(event.Timestamp)); err != nil { return fmt.Errorf("failed to mark applied txn: %w", err) @@ -407,7 +410,7 @@ func (ds *DeltaSyncClient) applyChangeEvent(ctx context.Context, event *ChangeEv if err := tx.Commit(); err != nil { return fmt.Errorf("failed to commit: %w", err) } - if hasDDL { + if !schemaChange.Empty() { if err := mdb.ReloadSchema(); err != nil { log.Warn().Err(err).Str("database", database).Msg("DELTA-SYNC: failed to reload schema after DDL") } diff --git a/grpc/marmot.pb.go b/grpc/marmot.pb.go index 031cb7ba..d7ac713b 100644 --- a/grpc/marmot.pb.go +++ b/grpc/marmot.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.10 -// protoc v6.33.6 +// protoc v7.36.0 // source: grpc/marmot.proto package grpc @@ -2706,6 +2706,280 @@ func (x *LatestTxnIDsResponse) GetDatabaseInfo() []*DatabaseInfo { return nil } +// ===== AUTO_INCREMENT CLAIMS ===== +type AutoIncBasesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestingNodeId uint64 `protobuf:"varint,1,opt,name=requesting_node_id,json=requestingNodeId,proto3" json:"requesting_node_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AutoIncBasesRequest) Reset() { + *x = AutoIncBasesRequest{} + mi := &file_grpc_marmot_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AutoIncBasesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AutoIncBasesRequest) ProtoMessage() {} + +func (x *AutoIncBasesRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_marmot_proto_msgTypes[33] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AutoIncBasesRequest.ProtoReflect.Descriptor instead. +func (*AutoIncBasesRequest) Descriptor() ([]byte, []int) { + return file_grpc_marmot_proto_rawDescGZIP(), []int{33} +} + +func (x *AutoIncBasesRequest) GetRequestingNodeId() uint64 { + if x != nil { + return x.RequestingNodeId + } + return 0 +} + +type AutoIncBase struct { + state protoimpl.MessageState `protogen:"open.v1"` + Database string `protobuf:"bytes,1,opt,name=database,proto3" json:"database,omitempty"` + Table string `protobuf:"bytes,2,opt,name=table,proto3" json:"table,omitempty"` + Base uint64 `protobuf:"varint,3,opt,name=base,proto3" json:"base,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AutoIncBase) Reset() { + *x = AutoIncBase{} + mi := &file_grpc_marmot_proto_msgTypes[34] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AutoIncBase) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AutoIncBase) ProtoMessage() {} + +func (x *AutoIncBase) ProtoReflect() protoreflect.Message { + mi := &file_grpc_marmot_proto_msgTypes[34] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AutoIncBase.ProtoReflect.Descriptor instead. +func (*AutoIncBase) Descriptor() ([]byte, []int) { + return file_grpc_marmot_proto_rawDescGZIP(), []int{34} +} + +func (x *AutoIncBase) GetDatabase() string { + if x != nil { + return x.Database + } + return "" +} + +func (x *AutoIncBase) GetTable() string { + if x != nil { + return x.Table + } + return "" +} + +func (x *AutoIncBase) GetBase() uint64 { + if x != nil { + return x.Base + } + return 0 +} + +type AutoIncBasesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Bases []*AutoIncBase `protobuf:"bytes,1,rep,name=bases,proto3" json:"bases,omitempty"` + // The responder's own votes are held: its bases may be missing claims, so + // a merge must not count it toward its majority. + VotesHeld bool `protobuf:"varint,2,opt,name=votes_held,json=votesHeld,proto3" json:"votes_held,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AutoIncBasesResponse) Reset() { + *x = AutoIncBasesResponse{} + mi := &file_grpc_marmot_proto_msgTypes[35] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AutoIncBasesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AutoIncBasesResponse) ProtoMessage() {} + +func (x *AutoIncBasesResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_marmot_proto_msgTypes[35] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AutoIncBasesResponse.ProtoReflect.Descriptor instead. +func (*AutoIncBasesResponse) Descriptor() ([]byte, []int) { + return file_grpc_marmot_proto_rawDescGZIP(), []int{35} +} + +func (x *AutoIncBasesResponse) GetBases() []*AutoIncBase { + if x != nil { + return x.Bases + } + return nil +} + +func (x *AutoIncBasesResponse) GetVotesHeld() bool { + if x != nil { + return x.VotesHeld + } + return false +} + +type AutoIncSyncRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestingNodeId uint64 `protobuf:"varint,1,opt,name=requesting_node_id,json=requestingNodeId,proto3" json:"requesting_node_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AutoIncSyncRequest) Reset() { + *x = AutoIncSyncRequest{} + mi := &file_grpc_marmot_proto_msgTypes[36] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AutoIncSyncRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AutoIncSyncRequest) ProtoMessage() {} + +func (x *AutoIncSyncRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_marmot_proto_msgTypes[36] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AutoIncSyncRequest.ProtoReflect.Descriptor instead. +func (*AutoIncSyncRequest) Descriptor() ([]byte, []int) { + return file_grpc_marmot_proto_rawDescGZIP(), []int{36} +} + +func (x *AutoIncSyncRequest) GetRequestingNodeId() uint64 { + if x != nil { + return x.RequestingNodeId + } + return 0 +} + +type AutoIncSyncResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + NodeId uint64 `protobuf:"varint,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` + // The node's own votes are held: it has not merged claim bases yet. + VotesHeld bool `protobuf:"varint,2,opt,name=votes_held,json=votesHeld,proto3" json:"votes_held,omitempty"` + // Every member of the node's membership, itself included. + Members []uint64 `protobuf:"varint,3,rep,packed,name=members,proto3" json:"members,omitempty"` + // The members whose bases this sync merged, itself included. + Reached []uint64 `protobuf:"varint,4,rep,packed,name=reached,proto3" json:"reached,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AutoIncSyncResponse) Reset() { + *x = AutoIncSyncResponse{} + mi := &file_grpc_marmot_proto_msgTypes[37] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AutoIncSyncResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AutoIncSyncResponse) ProtoMessage() {} + +func (x *AutoIncSyncResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_marmot_proto_msgTypes[37] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AutoIncSyncResponse.ProtoReflect.Descriptor instead. +func (*AutoIncSyncResponse) Descriptor() ([]byte, []int) { + return file_grpc_marmot_proto_rawDescGZIP(), []int{37} +} + +func (x *AutoIncSyncResponse) GetNodeId() uint64 { + if x != nil { + return x.NodeId + } + return 0 +} + +func (x *AutoIncSyncResponse) GetVotesHeld() bool { + if x != nil { + return x.VotesHeld + } + return false +} + +func (x *AutoIncSyncResponse) GetMembers() []uint64 { + if x != nil { + return x.Members + } + return nil +} + +func (x *AutoIncSyncResponse) GetReached() []uint64 { + if x != nil { + return x.Reached + } + return nil +} + // ===== CLUSTER NODES ===== type GetClusterNodesRequest struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -2715,7 +2989,7 @@ type GetClusterNodesRequest struct { func (x *GetClusterNodesRequest) Reset() { *x = GetClusterNodesRequest{} - mi := &file_grpc_marmot_proto_msgTypes[33] + mi := &file_grpc_marmot_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2727,7 +3001,7 @@ func (x *GetClusterNodesRequest) String() string { func (*GetClusterNodesRequest) ProtoMessage() {} func (x *GetClusterNodesRequest) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[33] + mi := &file_grpc_marmot_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2740,7 +3014,7 @@ func (x *GetClusterNodesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetClusterNodesRequest.ProtoReflect.Descriptor instead. func (*GetClusterNodesRequest) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{33} + return file_grpc_marmot_proto_rawDescGZIP(), []int{38} } type GetClusterNodesResponse struct { @@ -2752,7 +3026,7 @@ type GetClusterNodesResponse struct { func (x *GetClusterNodesResponse) Reset() { *x = GetClusterNodesResponse{} - mi := &file_grpc_marmot_proto_msgTypes[34] + mi := &file_grpc_marmot_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2764,7 +3038,7 @@ func (x *GetClusterNodesResponse) String() string { func (*GetClusterNodesResponse) ProtoMessage() {} func (x *GetClusterNodesResponse) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[34] + mi := &file_grpc_marmot_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2777,7 +3051,7 @@ func (x *GetClusterNodesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetClusterNodesResponse.ProtoReflect.Descriptor instead. func (*GetClusterNodesResponse) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{34} + return file_grpc_marmot_proto_rawDescGZIP(), []int{39} } func (x *GetClusterNodesResponse) GetNodes() []*NodeState { @@ -2801,7 +3075,7 @@ type TransactionChunk struct { func (x *TransactionChunk) Reset() { *x = TransactionChunk{} - mi := &file_grpc_marmot_proto_msgTypes[35] + mi := &file_grpc_marmot_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2813,7 +3087,7 @@ func (x *TransactionChunk) String() string { func (*TransactionChunk) ProtoMessage() {} func (x *TransactionChunk) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[35] + mi := &file_grpc_marmot_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2826,7 +3100,7 @@ func (x *TransactionChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use TransactionChunk.ProtoReflect.Descriptor instead. func (*TransactionChunk) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{35} + return file_grpc_marmot_proto_rawDescGZIP(), []int{40} } func (x *TransactionChunk) GetTxnId() uint64 { @@ -2870,7 +3144,7 @@ type TransactionCommit struct { func (x *TransactionCommit) Reset() { *x = TransactionCommit{} - mi := &file_grpc_marmot_proto_msgTypes[36] + mi := &file_grpc_marmot_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2882,7 +3156,7 @@ func (x *TransactionCommit) String() string { func (*TransactionCommit) ProtoMessage() {} func (x *TransactionCommit) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[36] + mi := &file_grpc_marmot_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2895,7 +3169,7 @@ func (x *TransactionCommit) ProtoReflect() protoreflect.Message { // Deprecated: Use TransactionCommit.ProtoReflect.Descriptor instead. func (*TransactionCommit) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{36} + return file_grpc_marmot_proto_rawDescGZIP(), []int{41} } func (x *TransactionCommit) GetTxnId() uint64 { @@ -2940,7 +3214,7 @@ type TransactionStreamMessage struct { func (x *TransactionStreamMessage) Reset() { *x = TransactionStreamMessage{} - mi := &file_grpc_marmot_proto_msgTypes[37] + mi := &file_grpc_marmot_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2952,7 +3226,7 @@ func (x *TransactionStreamMessage) String() string { func (*TransactionStreamMessage) ProtoMessage() {} func (x *TransactionStreamMessage) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[37] + mi := &file_grpc_marmot_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2965,7 +3239,7 @@ func (x *TransactionStreamMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use TransactionStreamMessage.ProtoReflect.Descriptor instead. func (*TransactionStreamMessage) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{37} + return file_grpc_marmot_proto_rawDescGZIP(), []int{42} } func (x *TransactionStreamMessage) GetPayload() isTransactionStreamMessage_Payload { @@ -3026,7 +3300,7 @@ type ForwardQueryRequest struct { func (x *ForwardQueryRequest) Reset() { *x = ForwardQueryRequest{} - mi := &file_grpc_marmot_proto_msgTypes[38] + mi := &file_grpc_marmot_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3038,7 +3312,7 @@ func (x *ForwardQueryRequest) String() string { func (*ForwardQueryRequest) ProtoMessage() {} func (x *ForwardQueryRequest) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[38] + mi := &file_grpc_marmot_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3051,7 +3325,7 @@ func (x *ForwardQueryRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ForwardQueryRequest.ProtoReflect.Descriptor instead. func (*ForwardQueryRequest) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{38} + return file_grpc_marmot_proto_rawDescGZIP(), []int{43} } func (x *ForwardQueryRequest) GetReplicaNodeId() uint64 { @@ -3142,7 +3416,7 @@ type ForwardQueryResponse struct { func (x *ForwardQueryResponse) Reset() { *x = ForwardQueryResponse{} - mi := &file_grpc_marmot_proto_msgTypes[39] + mi := &file_grpc_marmot_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3154,7 +3428,7 @@ func (x *ForwardQueryResponse) String() string { func (*ForwardQueryResponse) ProtoMessage() {} func (x *ForwardQueryResponse) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[39] + mi := &file_grpc_marmot_proto_msgTypes[44] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3167,7 +3441,7 @@ func (x *ForwardQueryResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ForwardQueryResponse.ProtoReflect.Descriptor instead. func (*ForwardQueryResponse) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{39} + return file_grpc_marmot_proto_rawDescGZIP(), []int{44} } func (x *ForwardQueryResponse) GetSuccess() bool { @@ -3242,7 +3516,7 @@ type ForwardLoadDataRequest struct { func (x *ForwardLoadDataRequest) Reset() { *x = ForwardLoadDataRequest{} - mi := &file_grpc_marmot_proto_msgTypes[40] + mi := &file_grpc_marmot_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3254,7 +3528,7 @@ func (x *ForwardLoadDataRequest) String() string { func (*ForwardLoadDataRequest) ProtoMessage() {} func (x *ForwardLoadDataRequest) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[40] + mi := &file_grpc_marmot_proto_msgTypes[45] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3267,7 +3541,7 @@ func (x *ForwardLoadDataRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ForwardLoadDataRequest.ProtoReflect.Descriptor instead. func (*ForwardLoadDataRequest) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{40} + return file_grpc_marmot_proto_rawDescGZIP(), []int{45} } func (x *ForwardLoadDataRequest) GetReplicaNodeId() uint64 { @@ -3338,7 +3612,7 @@ type LoadDataChunkRequest struct { func (x *LoadDataChunkRequest) Reset() { *x = LoadDataChunkRequest{} - mi := &file_grpc_marmot_proto_msgTypes[41] + mi := &file_grpc_marmot_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3350,7 +3624,7 @@ func (x *LoadDataChunkRequest) String() string { func (*LoadDataChunkRequest) ProtoMessage() {} func (x *LoadDataChunkRequest) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[41] + mi := &file_grpc_marmot_proto_msgTypes[46] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3363,7 +3637,7 @@ func (x *LoadDataChunkRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LoadDataChunkRequest.ProtoReflect.Descriptor instead. func (*LoadDataChunkRequest) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{41} + return file_grpc_marmot_proto_rawDescGZIP(), []int{46} } func (x *LoadDataChunkRequest) GetRequestingNodeId() uint64 { @@ -3404,7 +3678,7 @@ type LoadDataChunkResponse struct { func (x *LoadDataChunkResponse) Reset() { *x = LoadDataChunkResponse{} - mi := &file_grpc_marmot_proto_msgTypes[42] + mi := &file_grpc_marmot_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3416,7 +3690,7 @@ func (x *LoadDataChunkResponse) String() string { func (*LoadDataChunkResponse) ProtoMessage() {} func (x *LoadDataChunkResponse) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[42] + mi := &file_grpc_marmot_proto_msgTypes[47] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3429,7 +3703,7 @@ func (x *LoadDataChunkResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use LoadDataChunkResponse.ProtoReflect.Descriptor instead. func (*LoadDataChunkResponse) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{42} + return file_grpc_marmot_proto_rawDescGZIP(), []int{47} } func (x *LoadDataChunkResponse) GetData() []byte { @@ -3660,7 +3934,25 @@ const file_grpc_marmot_proto_rawDesc = "" + "\rdatabase_info\x18\x02 \x03(\v2\x17.marmot.v2.DatabaseInfoR\fdatabaseInfo\x1aA\n" + "\x13DatabaseTxnIdsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\x04R\x05value:\x028\x01\"\x18\n" + + "\x05value\x18\x02 \x01(\x04R\x05value:\x028\x01\"C\n" + + "\x13AutoIncBasesRequest\x12,\n" + + "\x12requesting_node_id\x18\x01 \x01(\x04R\x10requestingNodeId\"S\n" + + "\vAutoIncBase\x12\x1a\n" + + "\bdatabase\x18\x01 \x01(\tR\bdatabase\x12\x14\n" + + "\x05table\x18\x02 \x01(\tR\x05table\x12\x12\n" + + "\x04base\x18\x03 \x01(\x04R\x04base\"c\n" + + "\x14AutoIncBasesResponse\x12,\n" + + "\x05bases\x18\x01 \x03(\v2\x16.marmot.v2.AutoIncBaseR\x05bases\x12\x1d\n" + + "\n" + + "votes_held\x18\x02 \x01(\bR\tvotesHeld\"B\n" + + "\x12AutoIncSyncRequest\x12,\n" + + "\x12requesting_node_id\x18\x01 \x01(\x04R\x10requestingNodeId\"\x81\x01\n" + + "\x13AutoIncSyncResponse\x12\x17\n" + + "\anode_id\x18\x01 \x01(\x04R\x06nodeId\x12\x1d\n" + + "\n" + + "votes_held\x18\x02 \x01(\bR\tvotesHeld\x12\x18\n" + + "\amembers\x18\x03 \x03(\x04R\amembers\x12\x18\n" + + "\areached\x18\x04 \x03(\x04R\areached\"\x18\n" + "\x16GetClusterNodesRequest\"E\n" + "\x17GetClusterNodesResponse\x12*\n" + "\x05nodes\x18\x01 \x03(\v2\x14.marmot.v2.NodeStateR\x05nodes\"\x9c\x01\n" + @@ -3755,7 +4047,8 @@ const file_grpc_marmot_proto_rawDesc = "" + "\fFWD_TXN_NONE\x10\x00\x12\x11\n" + "\rFWD_TXN_BEGIN\x10\x01\x12\x12\n" + "\x0eFWD_TXN_COMMIT\x10\x02\x12\x14\n" + - "\x10FWD_TXN_ROLLBACK\x10\x032\x9c\t\n" + + "\x10FWD_TXN_ROLLBACK\x10\x032\xc3\n" + + "\n" + "\rMarmotService\x12=\n" + "\x06Gossip\x12\x18.marmot.v2.GossipRequest\x1a\x19.marmot.v2.GossipResponse\x127\n" + "\x04Join\x12\x16.marmot.v2.JoinRequest\x1a\x17.marmot.v2.JoinResponse\x127\n" + @@ -3770,7 +4063,9 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x0fGetClusterNodes\x12!.marmot.v2.GetClusterNodesRequest\x1a\".marmot.v2.GetClusterNodesResponse\x12O\n" + "\fForwardQuery\x12\x1e.marmot.v2.ForwardQueryRequest\x1a\x1f.marmot.v2.ForwardQueryResponse\x12U\n" + "\x0fForwardLoadData\x12!.marmot.v2.ForwardLoadDataRequest\x1a\x1f.marmot.v2.ForwardQueryResponse\x12U\n" + - "\x10GetLoadDataChunk\x12\x1f.marmot.v2.LoadDataChunkRequest\x1a .marmot.v2.LoadDataChunkResponse\x12Z\n" + + "\x10GetLoadDataChunk\x12\x1f.marmot.v2.LoadDataChunkRequest\x1a .marmot.v2.LoadDataChunkResponse\x12R\n" + + "\x0fGetAutoIncBases\x12\x1e.marmot.v2.AutoIncBasesRequest\x1a\x1f.marmot.v2.AutoIncBasesResponse\x12Q\n" + + "\x10SyncAutoIncBases\x12\x1d.marmot.v2.AutoIncSyncRequest\x1a\x1e.marmot.v2.AutoIncSyncResponse\x12Z\n" + "\x11TransactionStream\x12#.marmot.v2.TransactionStreamMessage\x1a\x1e.marmot.v2.TransactionResponse(\x01B Z\x1egithub.com/maxpert/marmot/grpcb\x06proto3" var ( @@ -3786,7 +4081,7 @@ func file_grpc_marmot_proto_rawDescGZIP() []byte { } var file_grpc_marmot_proto_enumTypes = make([]protoimpl.EnumInfo, 5) -var file_grpc_marmot_proto_msgTypes = make([]protoimpl.MessageInfo, 46) +var file_grpc_marmot_proto_msgTypes = make([]protoimpl.MessageInfo, 51) var file_grpc_marmot_proto_goTypes = []any{ (NodeStatus)(0), // 0: marmot.v2.NodeStatus (TransactionPhase)(0), // 1: marmot.v2.TransactionPhase @@ -3826,33 +4121,38 @@ var file_grpc_marmot_proto_goTypes = []any{ (*LatestTxnIDsRequest)(nil), // 35: marmot.v2.LatestTxnIDsRequest (*DatabaseInfo)(nil), // 36: marmot.v2.DatabaseInfo (*LatestTxnIDsResponse)(nil), // 37: marmot.v2.LatestTxnIDsResponse - (*GetClusterNodesRequest)(nil), // 38: marmot.v2.GetClusterNodesRequest - (*GetClusterNodesResponse)(nil), // 39: marmot.v2.GetClusterNodesResponse - (*TransactionChunk)(nil), // 40: marmot.v2.TransactionChunk - (*TransactionCommit)(nil), // 41: marmot.v2.TransactionCommit - (*TransactionStreamMessage)(nil), // 42: marmot.v2.TransactionStreamMessage - (*ForwardQueryRequest)(nil), // 43: marmot.v2.ForwardQueryRequest - (*ForwardQueryResponse)(nil), // 44: marmot.v2.ForwardQueryResponse - (*ForwardLoadDataRequest)(nil), // 45: marmot.v2.ForwardLoadDataRequest - (*LoadDataChunkRequest)(nil), // 46: marmot.v2.LoadDataChunkRequest - (*LoadDataChunkResponse)(nil), // 47: marmot.v2.LoadDataChunkResponse - nil, // 48: marmot.v2.NodeState.DatabaseSchemaVersionsEntry - nil, // 49: marmot.v2.Row.ColumnsEntry - nil, // 50: marmot.v2.LatestTxnIDsResponse.DatabaseTxnIdsEntry - (common.StatementType)(0), // 51: marmot.common.StatementType + (*AutoIncBasesRequest)(nil), // 38: marmot.v2.AutoIncBasesRequest + (*AutoIncBase)(nil), // 39: marmot.v2.AutoIncBase + (*AutoIncBasesResponse)(nil), // 40: marmot.v2.AutoIncBasesResponse + (*AutoIncSyncRequest)(nil), // 41: marmot.v2.AutoIncSyncRequest + (*AutoIncSyncResponse)(nil), // 42: marmot.v2.AutoIncSyncResponse + (*GetClusterNodesRequest)(nil), // 43: marmot.v2.GetClusterNodesRequest + (*GetClusterNodesResponse)(nil), // 44: marmot.v2.GetClusterNodesResponse + (*TransactionChunk)(nil), // 45: marmot.v2.TransactionChunk + (*TransactionCommit)(nil), // 46: marmot.v2.TransactionCommit + (*TransactionStreamMessage)(nil), // 47: marmot.v2.TransactionStreamMessage + (*ForwardQueryRequest)(nil), // 48: marmot.v2.ForwardQueryRequest + (*ForwardQueryResponse)(nil), // 49: marmot.v2.ForwardQueryResponse + (*ForwardLoadDataRequest)(nil), // 50: marmot.v2.ForwardLoadDataRequest + (*LoadDataChunkRequest)(nil), // 51: marmot.v2.LoadDataChunkRequest + (*LoadDataChunkResponse)(nil), // 52: marmot.v2.LoadDataChunkResponse + nil, // 53: marmot.v2.NodeState.DatabaseSchemaVersionsEntry + nil, // 54: marmot.v2.Row.ColumnsEntry + nil, // 55: marmot.v2.LatestTxnIDsResponse.DatabaseTxnIdsEntry + (common.StatementType)(0), // 56: marmot.common.StatementType } var file_grpc_marmot_proto_depIdxs = []int32{ 7, // 0: marmot.v2.GossipRequest.nodes:type_name -> marmot.v2.NodeState 7, // 1: marmot.v2.GossipResponse.nodes:type_name -> marmot.v2.NodeState 0, // 2: marmot.v2.NodeState.status:type_name -> marmot.v2.NodeStatus - 48, // 3: marmot.v2.NodeState.database_schema_versions:type_name -> marmot.v2.NodeState.DatabaseSchemaVersionsEntry + 53, // 3: marmot.v2.NodeState.database_schema_versions:type_name -> marmot.v2.NodeState.DatabaseSchemaVersionsEntry 7, // 4: marmot.v2.JoinResponse.cluster_nodes:type_name -> marmot.v2.NodeState 0, // 5: marmot.v2.PingResponse.status:type_name -> marmot.v2.NodeStatus 13, // 6: marmot.v2.TransactionRequest.statements:type_name -> marmot.v2.Statement 19, // 7: marmot.v2.TransactionRequest.timestamp:type_name -> marmot.v2.HLC 1, // 8: marmot.v2.TransactionRequest.phase:type_name -> marmot.v2.TransactionPhase 2, // 9: marmot.v2.TransactionRequest.consistency:type_name -> marmot.v2.ConsistencyLevel - 51, // 10: marmot.v2.Statement.type:type_name -> marmot.common.StatementType + 56, // 10: marmot.v2.Statement.type:type_name -> marmot.common.StatementType 15, // 11: marmot.v2.Statement.row_change:type_name -> marmot.v2.RowChange 17, // 12: marmot.v2.Statement.ddl_change:type_name -> marmot.v2.DDLChange 18, // 13: marmot.v2.Statement.load_data_change:type_name -> marmot.v2.LoadDataChange @@ -3864,7 +4164,7 @@ var file_grpc_marmot_proto_depIdxs = []int32{ 2, // 19: marmot.v2.ReadRequest.consistency:type_name -> marmot.v2.ConsistencyLevel 23, // 20: marmot.v2.ReadResponse.rows:type_name -> marmot.v2.Row 19, // 21: marmot.v2.ReadResponse.timestamp:type_name -> marmot.v2.HLC - 49, // 22: marmot.v2.Row.columns:type_name -> marmot.v2.Row.ColumnsEntry + 54, // 22: marmot.v2.Row.columns:type_name -> marmot.v2.Row.ColumnsEntry 13, // 23: marmot.v2.ChangeEvent.statements:type_name -> marmot.v2.Statement 19, // 24: marmot.v2.ChangeEvent.timestamp:type_name -> marmot.v2.HLC 28, // 25: marmot.v2.ReplicationStateResponse.states:type_name -> marmot.v2.DatabaseReplicationState @@ -3872,49 +4172,54 @@ var file_grpc_marmot_proto_depIdxs = []int32{ 19, // 27: marmot.v2.SnapshotInfoResponse.timestamp:type_name -> marmot.v2.HLC 31, // 28: marmot.v2.SnapshotInfoResponse.databases:type_name -> marmot.v2.DatabaseFileInfo 32, // 29: marmot.v2.SnapshotInfoResponse.database_metadata:type_name -> marmot.v2.DatabaseSnapshotMetadata - 50, // 30: marmot.v2.LatestTxnIDsResponse.database_txn_ids:type_name -> marmot.v2.LatestTxnIDsResponse.DatabaseTxnIdsEntry + 55, // 30: marmot.v2.LatestTxnIDsResponse.database_txn_ids:type_name -> marmot.v2.LatestTxnIDsResponse.DatabaseTxnIdsEntry 36, // 31: marmot.v2.LatestTxnIDsResponse.database_info:type_name -> marmot.v2.DatabaseInfo - 7, // 32: marmot.v2.GetClusterNodesResponse.nodes:type_name -> marmot.v2.NodeState - 13, // 33: marmot.v2.TransactionChunk.statements:type_name -> marmot.v2.Statement - 19, // 34: marmot.v2.TransactionCommit.timestamp:type_name -> marmot.v2.HLC - 40, // 35: marmot.v2.TransactionStreamMessage.chunk:type_name -> marmot.v2.TransactionChunk - 41, // 36: marmot.v2.TransactionStreamMessage.commit:type_name -> marmot.v2.TransactionCommit - 4, // 37: marmot.v2.ForwardQueryRequest.txn_control:type_name -> marmot.v2.ForwardTxnControl - 5, // 38: marmot.v2.MarmotService.Gossip:input_type -> marmot.v2.GossipRequest - 8, // 39: marmot.v2.MarmotService.Join:input_type -> marmot.v2.JoinRequest - 10, // 40: marmot.v2.MarmotService.Ping:input_type -> marmot.v2.PingRequest - 12, // 41: marmot.v2.MarmotService.ReplicateTransaction:input_type -> marmot.v2.TransactionRequest - 21, // 42: marmot.v2.MarmotService.Read:input_type -> marmot.v2.ReadRequest - 24, // 43: marmot.v2.MarmotService.StreamChanges:input_type -> marmot.v2.StreamRequest - 26, // 44: marmot.v2.MarmotService.GetReplicationState:input_type -> marmot.v2.ReplicationStateRequest - 29, // 45: marmot.v2.MarmotService.GetSnapshotInfo:input_type -> marmot.v2.SnapshotInfoRequest - 33, // 46: marmot.v2.MarmotService.StreamSnapshot:input_type -> marmot.v2.SnapshotRequest - 35, // 47: marmot.v2.MarmotService.GetLatestTxnIDs:input_type -> marmot.v2.LatestTxnIDsRequest - 38, // 48: marmot.v2.MarmotService.GetClusterNodes:input_type -> marmot.v2.GetClusterNodesRequest - 43, // 49: marmot.v2.MarmotService.ForwardQuery:input_type -> marmot.v2.ForwardQueryRequest - 45, // 50: marmot.v2.MarmotService.ForwardLoadData:input_type -> marmot.v2.ForwardLoadDataRequest - 46, // 51: marmot.v2.MarmotService.GetLoadDataChunk:input_type -> marmot.v2.LoadDataChunkRequest - 42, // 52: marmot.v2.MarmotService.TransactionStream:input_type -> marmot.v2.TransactionStreamMessage - 6, // 53: marmot.v2.MarmotService.Gossip:output_type -> marmot.v2.GossipResponse - 9, // 54: marmot.v2.MarmotService.Join:output_type -> marmot.v2.JoinResponse - 11, // 55: marmot.v2.MarmotService.Ping:output_type -> marmot.v2.PingResponse - 20, // 56: marmot.v2.MarmotService.ReplicateTransaction:output_type -> marmot.v2.TransactionResponse - 22, // 57: marmot.v2.MarmotService.Read:output_type -> marmot.v2.ReadResponse - 25, // 58: marmot.v2.MarmotService.StreamChanges:output_type -> marmot.v2.ChangeEvent - 27, // 59: marmot.v2.MarmotService.GetReplicationState:output_type -> marmot.v2.ReplicationStateResponse - 30, // 60: marmot.v2.MarmotService.GetSnapshotInfo:output_type -> marmot.v2.SnapshotInfoResponse - 34, // 61: marmot.v2.MarmotService.StreamSnapshot:output_type -> marmot.v2.SnapshotChunk - 37, // 62: marmot.v2.MarmotService.GetLatestTxnIDs:output_type -> marmot.v2.LatestTxnIDsResponse - 39, // 63: marmot.v2.MarmotService.GetClusterNodes:output_type -> marmot.v2.GetClusterNodesResponse - 44, // 64: marmot.v2.MarmotService.ForwardQuery:output_type -> marmot.v2.ForwardQueryResponse - 44, // 65: marmot.v2.MarmotService.ForwardLoadData:output_type -> marmot.v2.ForwardQueryResponse - 47, // 66: marmot.v2.MarmotService.GetLoadDataChunk:output_type -> marmot.v2.LoadDataChunkResponse - 20, // 67: marmot.v2.MarmotService.TransactionStream:output_type -> marmot.v2.TransactionResponse - 53, // [53:68] is the sub-list for method output_type - 38, // [38:53] is the sub-list for method input_type - 38, // [38:38] is the sub-list for extension type_name - 38, // [38:38] is the sub-list for extension extendee - 0, // [0:38] is the sub-list for field type_name + 39, // 32: marmot.v2.AutoIncBasesResponse.bases:type_name -> marmot.v2.AutoIncBase + 7, // 33: marmot.v2.GetClusterNodesResponse.nodes:type_name -> marmot.v2.NodeState + 13, // 34: marmot.v2.TransactionChunk.statements:type_name -> marmot.v2.Statement + 19, // 35: marmot.v2.TransactionCommit.timestamp:type_name -> marmot.v2.HLC + 45, // 36: marmot.v2.TransactionStreamMessage.chunk:type_name -> marmot.v2.TransactionChunk + 46, // 37: marmot.v2.TransactionStreamMessage.commit:type_name -> marmot.v2.TransactionCommit + 4, // 38: marmot.v2.ForwardQueryRequest.txn_control:type_name -> marmot.v2.ForwardTxnControl + 5, // 39: marmot.v2.MarmotService.Gossip:input_type -> marmot.v2.GossipRequest + 8, // 40: marmot.v2.MarmotService.Join:input_type -> marmot.v2.JoinRequest + 10, // 41: marmot.v2.MarmotService.Ping:input_type -> marmot.v2.PingRequest + 12, // 42: marmot.v2.MarmotService.ReplicateTransaction:input_type -> marmot.v2.TransactionRequest + 21, // 43: marmot.v2.MarmotService.Read:input_type -> marmot.v2.ReadRequest + 24, // 44: marmot.v2.MarmotService.StreamChanges:input_type -> marmot.v2.StreamRequest + 26, // 45: marmot.v2.MarmotService.GetReplicationState:input_type -> marmot.v2.ReplicationStateRequest + 29, // 46: marmot.v2.MarmotService.GetSnapshotInfo:input_type -> marmot.v2.SnapshotInfoRequest + 33, // 47: marmot.v2.MarmotService.StreamSnapshot:input_type -> marmot.v2.SnapshotRequest + 35, // 48: marmot.v2.MarmotService.GetLatestTxnIDs:input_type -> marmot.v2.LatestTxnIDsRequest + 43, // 49: marmot.v2.MarmotService.GetClusterNodes:input_type -> marmot.v2.GetClusterNodesRequest + 48, // 50: marmot.v2.MarmotService.ForwardQuery:input_type -> marmot.v2.ForwardQueryRequest + 50, // 51: marmot.v2.MarmotService.ForwardLoadData:input_type -> marmot.v2.ForwardLoadDataRequest + 51, // 52: marmot.v2.MarmotService.GetLoadDataChunk:input_type -> marmot.v2.LoadDataChunkRequest + 38, // 53: marmot.v2.MarmotService.GetAutoIncBases:input_type -> marmot.v2.AutoIncBasesRequest + 41, // 54: marmot.v2.MarmotService.SyncAutoIncBases:input_type -> marmot.v2.AutoIncSyncRequest + 47, // 55: marmot.v2.MarmotService.TransactionStream:input_type -> marmot.v2.TransactionStreamMessage + 6, // 56: marmot.v2.MarmotService.Gossip:output_type -> marmot.v2.GossipResponse + 9, // 57: marmot.v2.MarmotService.Join:output_type -> marmot.v2.JoinResponse + 11, // 58: marmot.v2.MarmotService.Ping:output_type -> marmot.v2.PingResponse + 20, // 59: marmot.v2.MarmotService.ReplicateTransaction:output_type -> marmot.v2.TransactionResponse + 22, // 60: marmot.v2.MarmotService.Read:output_type -> marmot.v2.ReadResponse + 25, // 61: marmot.v2.MarmotService.StreamChanges:output_type -> marmot.v2.ChangeEvent + 27, // 62: marmot.v2.MarmotService.GetReplicationState:output_type -> marmot.v2.ReplicationStateResponse + 30, // 63: marmot.v2.MarmotService.GetSnapshotInfo:output_type -> marmot.v2.SnapshotInfoResponse + 34, // 64: marmot.v2.MarmotService.StreamSnapshot:output_type -> marmot.v2.SnapshotChunk + 37, // 65: marmot.v2.MarmotService.GetLatestTxnIDs:output_type -> marmot.v2.LatestTxnIDsResponse + 44, // 66: marmot.v2.MarmotService.GetClusterNodes:output_type -> marmot.v2.GetClusterNodesResponse + 49, // 67: marmot.v2.MarmotService.ForwardQuery:output_type -> marmot.v2.ForwardQueryResponse + 49, // 68: marmot.v2.MarmotService.ForwardLoadData:output_type -> marmot.v2.ForwardQueryResponse + 52, // 69: marmot.v2.MarmotService.GetLoadDataChunk:output_type -> marmot.v2.LoadDataChunkResponse + 40, // 70: marmot.v2.MarmotService.GetAutoIncBases:output_type -> marmot.v2.AutoIncBasesResponse + 42, // 71: marmot.v2.MarmotService.SyncAutoIncBases:output_type -> marmot.v2.AutoIncSyncResponse + 20, // 72: marmot.v2.MarmotService.TransactionStream:output_type -> marmot.v2.TransactionResponse + 56, // [56:73] is the sub-list for method output_type + 39, // [39:56] is the sub-list for method input_type + 39, // [39:39] is the sub-list for extension type_name + 39, // [39:39] is the sub-list for extension extendee + 0, // [0:39] is the sub-list for field type_name } func init() { file_grpc_marmot_proto_init() } @@ -3929,7 +4234,7 @@ func file_grpc_marmot_proto_init() { (*Statement_VectorIndexChange)(nil), (*Statement_DmlIntent)(nil), } - file_grpc_marmot_proto_msgTypes[37].OneofWrappers = []any{ + file_grpc_marmot_proto_msgTypes[42].OneofWrappers = []any{ (*TransactionStreamMessage_Chunk)(nil), (*TransactionStreamMessage_Commit)(nil), } @@ -3939,7 +4244,7 @@ func file_grpc_marmot_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_marmot_proto_rawDesc), len(file_grpc_marmot_proto_rawDesc)), NumEnums: 5, - NumMessages: 46, + NumMessages: 51, NumExtensions: 0, NumServices: 1, }, diff --git a/grpc/marmot.proto b/grpc/marmot.proto index 45af0137..7443bde8 100644 --- a/grpc/marmot.proto +++ b/grpc/marmot.proto @@ -59,6 +59,15 @@ service MarmotService { // Fetch staged LOAD DATA payload chunk by chunk. rpc GetLoadDataChunk(LoadDataChunkRequest) returns (LoadDataChunkResponse); + // ===== AUTO_INCREMENT CLAIMS ===== + // This node's committed AUTO_INCREMENT claim bases, which a node whose + // claim votes are held merges from a majority before voting again. + rpc GetAutoIncBases(AutoIncBasesRequest) returns (AutoIncBasesResponse); + + // Runs one claim base sync on this node now: it raises its bases to the + // maximum every alive peer reports, and reports which members it reached. + rpc SyncAutoIncBases(AutoIncSyncRequest) returns (AutoIncSyncResponse); + // ===== CDC STREAMING ===== // Client-streaming RPC for large CDC payloads (≥128KB) // Sends TransactionChunks followed by TransactionCommit @@ -409,6 +418,38 @@ message LatestTxnIDsResponse { repeated DatabaseInfo database_info = 2; // Detailed database info } +// ===== AUTO_INCREMENT CLAIMS ===== +message AutoIncBasesRequest { + uint64 requesting_node_id = 1; +} + +message AutoIncBase { + string database = 1; + string table = 2; + uint64 base = 3; +} + +message AutoIncBasesResponse { + repeated AutoIncBase bases = 1; + // The responder's own votes are held: its bases may be missing claims, so + // a merge must not count it toward its majority. + bool votes_held = 2; +} + +message AutoIncSyncRequest { + uint64 requesting_node_id = 1; +} + +message AutoIncSyncResponse { + uint64 node_id = 1; + // The node's own votes are held: it has not merged claim bases yet. + bool votes_held = 2; + // Every member of the node's membership, itself included. + repeated uint64 members = 3; + // The members whose bases this sync merged, itself included. + repeated uint64 reached = 4; +} + // ===== CLUSTER NODES ===== message GetClusterNodesRequest {} diff --git a/grpc/marmot_grpc.pb.go b/grpc/marmot_grpc.pb.go index 4c01c85b..7df594f1 100644 --- a/grpc/marmot_grpc.pb.go +++ b/grpc/marmot_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.5.1 -// - protoc v6.33.6 +// - protoc v7.36.0 // source: grpc/marmot.proto package grpc @@ -33,6 +33,8 @@ const ( MarmotService_ForwardQuery_FullMethodName = "/marmot.v2.MarmotService/ForwardQuery" MarmotService_ForwardLoadData_FullMethodName = "/marmot.v2.MarmotService/ForwardLoadData" MarmotService_GetLoadDataChunk_FullMethodName = "/marmot.v2.MarmotService/GetLoadDataChunk" + MarmotService_GetAutoIncBases_FullMethodName = "/marmot.v2.MarmotService/GetAutoIncBases" + MarmotService_SyncAutoIncBases_FullMethodName = "/marmot.v2.MarmotService/SyncAutoIncBases" MarmotService_TransactionStream_FullMethodName = "/marmot.v2.MarmotService/TransactionStream" ) @@ -74,6 +76,13 @@ type MarmotServiceClient interface { ForwardLoadData(ctx context.Context, in *ForwardLoadDataRequest, opts ...grpc.CallOption) (*ForwardQueryResponse, error) // Fetch staged LOAD DATA payload chunk by chunk. GetLoadDataChunk(ctx context.Context, in *LoadDataChunkRequest, opts ...grpc.CallOption) (*LoadDataChunkResponse, error) + // ===== AUTO_INCREMENT CLAIMS ===== + // This node's committed AUTO_INCREMENT claim bases, which a node whose + // claim votes are held merges from a majority before voting again. + GetAutoIncBases(ctx context.Context, in *AutoIncBasesRequest, opts ...grpc.CallOption) (*AutoIncBasesResponse, error) + // Runs one claim base sync on this node now: it raises its bases to the + // maximum every alive peer reports, and reports which members it reached. + SyncAutoIncBases(ctx context.Context, in *AutoIncSyncRequest, opts ...grpc.CallOption) (*AutoIncSyncResponse, error) // ===== CDC STREAMING ===== // Client-streaming RPC for large CDC payloads (≥128KB) // Sends TransactionChunks followed by TransactionCommit @@ -246,6 +255,26 @@ func (c *marmotServiceClient) GetLoadDataChunk(ctx context.Context, in *LoadData return out, nil } +func (c *marmotServiceClient) GetAutoIncBases(ctx context.Context, in *AutoIncBasesRequest, opts ...grpc.CallOption) (*AutoIncBasesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(AutoIncBasesResponse) + err := c.cc.Invoke(ctx, MarmotService_GetAutoIncBases_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *marmotServiceClient) SyncAutoIncBases(ctx context.Context, in *AutoIncSyncRequest, opts ...grpc.CallOption) (*AutoIncSyncResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(AutoIncSyncResponse) + err := c.cc.Invoke(ctx, MarmotService_SyncAutoIncBases_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *marmotServiceClient) TransactionStream(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[TransactionStreamMessage, TransactionResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &MarmotService_ServiceDesc.Streams[2], MarmotService_TransactionStream_FullMethodName, cOpts...) @@ -297,6 +326,13 @@ type MarmotServiceServer interface { ForwardLoadData(context.Context, *ForwardLoadDataRequest) (*ForwardQueryResponse, error) // Fetch staged LOAD DATA payload chunk by chunk. GetLoadDataChunk(context.Context, *LoadDataChunkRequest) (*LoadDataChunkResponse, error) + // ===== AUTO_INCREMENT CLAIMS ===== + // This node's committed AUTO_INCREMENT claim bases, which a node whose + // claim votes are held merges from a majority before voting again. + GetAutoIncBases(context.Context, *AutoIncBasesRequest) (*AutoIncBasesResponse, error) + // Runs one claim base sync on this node now: it raises its bases to the + // maximum every alive peer reports, and reports which members it reached. + SyncAutoIncBases(context.Context, *AutoIncSyncRequest) (*AutoIncSyncResponse, error) // ===== CDC STREAMING ===== // Client-streaming RPC for large CDC payloads (≥128KB) // Sends TransactionChunks followed by TransactionCommit @@ -353,6 +389,12 @@ func (UnimplementedMarmotServiceServer) ForwardLoadData(context.Context, *Forwar func (UnimplementedMarmotServiceServer) GetLoadDataChunk(context.Context, *LoadDataChunkRequest) (*LoadDataChunkResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetLoadDataChunk not implemented") } +func (UnimplementedMarmotServiceServer) GetAutoIncBases(context.Context, *AutoIncBasesRequest) (*AutoIncBasesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetAutoIncBases not implemented") +} +func (UnimplementedMarmotServiceServer) SyncAutoIncBases(context.Context, *AutoIncSyncRequest) (*AutoIncSyncResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SyncAutoIncBases not implemented") +} func (UnimplementedMarmotServiceServer) TransactionStream(grpc.ClientStreamingServer[TransactionStreamMessage, TransactionResponse]) error { return status.Errorf(codes.Unimplemented, "method TransactionStream not implemented") } @@ -615,6 +657,42 @@ func _MarmotService_GetLoadDataChunk_Handler(srv interface{}, ctx context.Contex return interceptor(ctx, in, info, handler) } +func _MarmotService_GetAutoIncBases_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AutoIncBasesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MarmotServiceServer).GetAutoIncBases(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: MarmotService_GetAutoIncBases_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MarmotServiceServer).GetAutoIncBases(ctx, req.(*AutoIncBasesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _MarmotService_SyncAutoIncBases_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AutoIncSyncRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MarmotServiceServer).SyncAutoIncBases(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: MarmotService_SyncAutoIncBases_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MarmotServiceServer).SyncAutoIncBases(ctx, req.(*AutoIncSyncRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _MarmotService_TransactionStream_Handler(srv interface{}, stream grpc.ServerStream) error { return srv.(MarmotServiceServer).TransactionStream(&grpc.GenericServerStream[TransactionStreamMessage, TransactionResponse]{ServerStream: stream}) } @@ -677,6 +755,14 @@ var MarmotService_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetLoadDataChunk", Handler: _MarmotService_GetLoadDataChunk_Handler, }, + { + MethodName: "GetAutoIncBases", + Handler: _MarmotService_GetAutoIncBases_Handler, + }, + { + MethodName: "SyncAutoIncBases", + Handler: _MarmotService_SyncAutoIncBases_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/grpc/node_registry.go b/grpc/node_registry.go index eb271eb2..6ce7fad5 100644 --- a/grpc/node_registry.go +++ b/grpc/node_registry.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "fmt" "hash/fnv" + "slices" "sort" "sync" "time" @@ -42,7 +43,7 @@ type NodeRegistry struct { nodes map[uint64]*NodeState lastSeen map[uint64]time.Time mu sync.RWMutex - onNodeAliveFunc func(*NodeState) // Callback when node transitions to ALIVE + onNodeAliveFunc func(*NodeState) // Callback when node transitions to ALIVE, or is discovered ALIVE onNodeDeadFunc func(*NodeState) // Callback when node transitions to DEAD onNodeLeavingFunc func() // Callback when local node marked LEAVING via remote decommission callbackMu sync.RWMutex @@ -140,7 +141,10 @@ func (nr *NodeRegistry) Update(node *NodeState) { return } - // Rule 2: Discover new nodes + // Rule 2: Discover new nodes. A node discovered ALIVE gets the same + // callback as one that becomes ALIVE: it may never change status again + // (a restarted peer keeps its incarnation), and the callback is what + // opens this node's connection to it. existing, exists := nr.nodes[node.NodeId] if !exists { log.Debug(). @@ -153,6 +157,9 @@ func (nr *NodeRegistry) Update(node *NodeState) { nr.lastSeen[node.NodeId] = time.Now() nr.updateClusterMetricsLocked() nr.mu.Unlock() + if node.Status == NodeStatus_ALIVE { + nr.fireOnNodeAlive(node) + } return } @@ -247,13 +254,19 @@ func (nr *NodeRegistry) Update(node *NodeState) { // Call callback outside lock to avoid deadlock if becameAlive { - nr.callbackMu.RLock() - callback := nr.onNodeAliveFunc - nr.callbackMu.RUnlock() + nr.fireOnNodeAlive(node) + } +} - if callback != nil { - callback(node) - } +// fireOnNodeAlive runs the ALIVE callback for node. The caller must not hold +// nr.mu. +func (nr *NodeRegistry) fireOnNodeAlive(node *NodeState) { + nr.callbackMu.RLock() + callback := nr.onNodeAliveFunc + nr.callbackMu.RUnlock() + + if callback != nil { + callback(node) } } @@ -503,6 +516,21 @@ func (nr *NodeRegistry) CheckTimeouts(suspectTimeout, deadTimeout time.Duration) } } +// MemberIDs returns the IDs of the nodes Count counts, sorted. +func (nr *NodeRegistry) MemberIDs() []uint64 { + nr.mu.RLock() + defer nr.mu.RUnlock() + + ids := make([]uint64, 0, len(nr.nodes)) + for id, node := range nr.nodes { + if node.Status != NodeStatus_REMOVED && node.Status != NodeStatus_LEAVING { + ids = append(ids, id) + } + } + slices.Sort(ids) + return ids +} + // Count returns the number of nodes in membership (excludes REMOVED and LEAVING nodes) // Used for quorum calculation to prevent split-brain. LEAVING nodes are excluded // because they are intentionally departing and should not count toward quorum. diff --git a/grpc/node_registry_test.go b/grpc/node_registry_test.go index d32ae295..b471efdd 100644 --- a/grpc/node_registry_test.go +++ b/grpc/node_registry_test.go @@ -1211,3 +1211,41 @@ func TestLeavingThenCrashesSuspect(t *testing.T) { t.Errorf("LEAVING -> SUSPECT should be valid via escalation, got %v", node.Status) } } + +// TestNodeRegistry_DiscoveringAnAliveNodeConnectsToIt pins R3c-14 (reviewer +// A's F-L1): the ALIVE callback, which opens this node's connection to a +// peer, fires when gossip first tells us of a peer that is already ALIVE, not +// only when a known peer turns ALIVE. A wiped node that learned of its +// restarted seed as ALIVE, from another member, never saw that seed change +// status again, so it never connected to it and stayed held. +// +// Mutation: drop the callback from Update's discovery branch. "a peer +// discovered ALIVE was never connected" fires, and so does the re-gossip +// row: the same state again is not a transition either. +func TestNodeRegistry_DiscoveringAnAliveNodeConnectsToIt(t *testing.T) { + nr := NewNodeRegistry(3, "localhost:8083") + var connected []uint64 + nr.SetOnNodeAlive(func(node *NodeState) { connected = append(connected, node.NodeId) }) + + nr.Update(&NodeState{NodeId: 1, Address: "localhost:8081", Status: NodeStatus_ALIVE, Incarnation: 1}) + if len(connected) != 1 || connected[0] != 1 { + t.Fatalf("a peer discovered ALIVE was never connected: callbacks for %v", connected) + } + + // The restarted seed announces the same state; no second callback is + // needed, the connection the first opened reconnects on its own. + nr.Update(&NodeState{NodeId: 1, Address: "localhost:8081", Status: NodeStatus_ALIVE, Incarnation: 1}) + if len(connected) != 1 { + t.Fatalf("an unchanged ALIVE peer fired the callback again: %v", connected) + } + + // A peer discovered in any other state is connected when it turns ALIVE. + nr.Update(&NodeState{NodeId: 2, Address: "localhost:8082", Status: NodeStatus_SUSPECT, Incarnation: 1}) + if len(connected) != 1 { + t.Fatalf("a peer discovered SUSPECT fired the ALIVE callback: %v", connected) + } + nr.Update(&NodeState{NodeId: 2, Address: "localhost:8082", Status: NodeStatus_ALIVE, Incarnation: 2}) + if len(connected) != 2 || connected[1] != 2 { + t.Fatalf("a peer that turned ALIVE was never connected: %v", connected) + } +} diff --git a/grpc/replication_handler.go b/grpc/replication_handler.go index 40b44256..e7abbb37 100644 --- a/grpc/replication_handler.go +++ b/grpc/replication_handler.go @@ -372,7 +372,7 @@ func (rh *ReplicationHandler) handleReplay(ctx context.Context, req *Transaction // Create schema adapter for CDC operations schemaAdapter := &replicationSchemaAdapter{dbMgr: rh.dbMgr, dbName: dbName} - hasDDL := false + var schemaChange db.SchemaChange for _, stmt := range req.Statements { // Check for CDC data (RowChange payload) @@ -403,14 +403,13 @@ func (rh *ReplicationHandler) handleReplay(ctx context.Context, req *Transaction // DDL path: execute SQL directly if ddl := stmt.GetDdlChange(); ddl != nil && ddl.Sql != "" { - if err := db.ApplyDDLSQLInTx(ctx, tx, ddl.Sql); err != nil { + if err := dbInstance.ApplyReplayedDDL(ctx, tx, ddl.Sql, stmt.TableName, req.SourceNodeId, &schemaChange); err != nil { telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() return &TransactionResponse{ Success: false, ErrorMessage: fmt.Sprintf("failed to execute DDL: %v", err), }, nil } - hasDDL = true continue } @@ -433,6 +432,15 @@ func (rh *ReplicationHandler) handleReplay(ctx context.Context, req *Transaction Msg("handleReplay: statement has no CDC data or DDL") } + // Replayed DDL changes table incarnations exactly as a committed one does. + if err := dbInstance.FinishReplayedSchemaChange(tx, &schemaChange); err != nil { + telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() + return &TransactionResponse{ + Success: false, + ErrorMessage: fmt.Sprintf("failed to finish replayed DDL: %v", err), + }, nil + } + // Commit the transaction if err := db.MarkSQLiteTxnApplied(tx, req.TxnId, HLCToTimestamp(req.Timestamp)); err != nil { telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() @@ -448,7 +456,7 @@ func (rh *ReplicationHandler) handleReplay(ctx context.Context, req *Transaction ErrorMessage: fmt.Sprintf("failed to commit: %v", err), }, nil } - if hasDDL { + if !schemaChange.Empty() { if err := dbInstance.ReloadSchema(); err != nil { log.Warn().Err(err).Str("database", dbName).Msg("handleReplay: failed to reload schema after DDL") } diff --git a/grpc/replication_handler_test.go b/grpc/replication_handler_test.go index badf14f2..b271d79d 100644 --- a/grpc/replication_handler_test.go +++ b/grpc/replication_handler_test.go @@ -3,12 +3,14 @@ package grpc import ( "context" "os" + "sync" "testing" "github.com/maxpert/marmot/db" "github.com/maxpert/marmot/encoding" pb "github.com/maxpert/marmot/grpc/common" "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" ) // TestSchemaVersionRejection verifies that transactions with higher required schema version are rejected @@ -479,3 +481,80 @@ func TestReplicationHandler_AcceptsPrepareWhenNotLeaving(t *testing.T) { } } } + +// incarnationRecorder records what a DDL apply reports to the node's +// AutoIncIncarnationListener. +type incarnationRecorder struct { + mu sync.Mutex + tables []string +} + +func (r *incarnationRecorder) TableIncarnationEnded(database, table string) { + r.mu.Lock() + defer r.mu.Unlock() + r.tables = append(r.tables, database+"."+table) +} + +func (r *incarnationRecorder) DatabaseIncarnationEnded(string) {} + +func (r *incarnationRecorder) take() []string { + r.mu.Lock() + defer r.mu.Unlock() + tables := r.tables + r.tables = nil + return tables +} + +// TestReplicationHandler_ReplayedDDLEndsIncarnations pins R3c-10 end to end +// on the anti-entropy replay RPC: a DDL a node catches up on through REPLAY +// ends table incarnations, inherits and seeds claim bases exactly as the +// same DDL's 2PC COMMIT would. +// +// Mutation: handleReplay executes DDL with db.ApplyDDLSQLInTx again. "a +// replayed CREATE did not end nt's incarnation" fires. +func TestReplicationHandler_ReplayedDDLEndsIncarnations(t *testing.T) { + clock := hlc.NewClock(1) + dbMgr, err := db.NewDatabaseManager(t.TempDir(), 1, clock) + require.NoError(t, err) + defer dbMgr.Close() + const testDB = "replay_incarnations" + require.NoError(t, dbMgr.CreateDatabase(testDB)) + systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) + require.NoError(t, err) + handler := NewReplicationHandler(1, dbMgr, clock, db.NewSchemaVersionManager(systemDB.GetMetaStore())) + rec := &incarnationRecorder{} + dbMgr.SetAutoIncIncarnationListener(rec) + + replay := func(txnID uint64, table, ddl string) { + t.Helper() + now := clock.Now() + resp, err := handler.HandleReplicateTransaction(context.Background(), &TransactionRequest{ + TxnId: txnID, + SourceNodeId: 2, + Database: testDB, + Phase: TransactionPhase_REPLAY, + Timestamp: &HLC{WallTime: now.WallTime, Logical: now.Logical, NodeId: 2}, + Statements: []*Statement{{ + Type: pb.StatementType_DDL, + TableName: table, + Database: testDB, + Payload: &Statement_DdlChange{DdlChange: &DDLChange{Sql: ddl}}, + }}, + }) + require.NoError(t, err) + require.True(t, resp.Success, "replay of %q: %s", ddl, resp.ErrorMessage) + } + + claims := db.NewAutoIncClaimStore(dbMgr.GetSystemDatabase()) + replay(300, "nt", "CREATE TABLE nt (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + require.Equal(t, []string{testDB + ".nt"}, rec.take(), "a replayed CREATE did not end nt's incarnation") + _, err = claims.ReadBase(testDB, "nt") + require.NoError(t, err, "a replayed CREATE was not seeded") + + require.NoError(t, claims.Seed(testDB, "nt", 400, 1)) + replay(301, "nt", "ALTER TABLE nt RENAME TO nt2") + require.Equal(t, []string{testDB + ".nt", testDB + ".nt2"}, rec.take(), "a replayed rename did not end both names' incarnations") + base, err := claims.ReadBase(testDB, "nt2") + require.NoError(t, err) + require.Equal(t, uint64(400), base, "a replayed rename did not inherit nt's base") +} diff --git a/grpc/replicator_adapter.go b/grpc/replicator_adapter.go index dd13186b..1901911f 100644 --- a/grpc/replicator_adapter.go +++ b/grpc/replicator_adapter.go @@ -33,12 +33,28 @@ func (gr *GRPCReplicator) CleanupStagedPayload(txnID uint64) { // ReplicateTransaction implements coordinator.Replicator func (gr *GRPCReplicator) ReplicateTransaction(ctx context.Context, nodeID uint64, req *coordinator.ReplicationRequest) (*coordinator.ReplicationResponse, error) { + grpcReq, err := transactionRequestToProto(req) + if err != nil { + return nil, err + } + + // Call gRPC client + grpcResp, err := gr.client.ReplicateTransaction(ctx, nodeID, grpcReq) + if err != nil { + return nil, fmt.Errorf("gRPC call failed: %w", err) + } + + return convertTransactionResponse(grpcResp), nil +} + +// transactionRequestToProto converts a coordinator.ReplicationRequest into +// the gRPC TransactionRequest a participant's ReplicationHandler receives. +func transactionRequestToProto(req *coordinator.ReplicationRequest) (*TransactionRequest, error) { statements, err := convertStatementsToProto(req.Statements, req.Database, req.TxnID) if err != nil { return nil, err } - // Convert coordinator.ReplicationRequest to gRPC TransactionRequest - grpcReq := &TransactionRequest{ + return &TransactionRequest{ TxnId: req.TxnID, SourceNodeId: req.NodeID, Statements: statements, @@ -50,15 +66,7 @@ func (gr *GRPCReplicator) ReplicateTransaction(ctx context.Context, nodeID uint6 Phase: convertPhaseToProto(req.Phase), Database: req.Database, RequiredSchemaVersion: req.RequiredSchemaVersion, - } - - // Call gRPC client - grpcResp, err := gr.client.ReplicateTransaction(ctx, nodeID, grpcReq) - if err != nil { - return nil, fmt.Errorf("gRPC call failed: %w", err) - } - - return convertTransactionResponse(grpcResp), nil + }, nil } // convertTransactionResponse converts a gRPC TransactionResponse into diff --git a/id/generator.go b/id/generator.go index 7430954e..c592072c 100644 --- a/id/generator.go +++ b/id/generator.go @@ -9,8 +9,13 @@ import ( // Generator provides unique IDs for distributed autoincrement columns. // IDs are guaranteed unique across nodes and roughly time-ordered. +// +// It mints every id one statement needs in one call, and returns an error so +// that a generator which must coordinate to mint ids can refuse; every caller +// has to hand that error to the client. type Generator interface { - NextID() uint64 + // NextIDs fills ids with distinct ids in ascending order. + NextIDs(ids []uint64) error } // HLCGenerator generates unique IDs using the Hybrid Logical Clock. @@ -24,11 +29,15 @@ func NewHLCGenerator(clock *hlc.Clock) *HLCGenerator { return &HLCGenerator{clock: clock} } -// NextID generates a unique 64-bit ID. +// NextIDs fills ids with successive unique 64-bit ids. The clock is strictly +// monotonic, so they ascend. // Format: (physical_ms << 22) | (node_id << 16) | logical // See hlc.Timestamp.ToTxnID for bit allocation details. -func (g *HLCGenerator) NextID() uint64 { - return g.clock.Now().ToTxnID() +func (g *HLCGenerator) NextIDs(ids []uint64) error { + for i := range ids { + ids[i] = g.clock.Now().ToTxnID() + } + return nil } // Compact ID format constants (53-bit) @@ -61,12 +70,20 @@ func NewCompactGenerator(nodeID uint64) *CompactGenerator { } } -// NextID generates a unique 53-bit ID safe for JavaScript. -// Thread-safe. IDs are monotonically increasing and unique. -func (g *CompactGenerator) NextID() uint64 { +// NextIDs fills ids with unique 53-bit ids safe for JavaScript. Thread-safe. +// IDs are monotonically increasing and unique; they are minted under one lock +// acquisition, so no other caller's id lands between them. +func (g *CompactGenerator) NextIDs(ids []uint64) error { g.mu.Lock() defer g.mu.Unlock() + for i := range ids { + ids[i] = g.next() + } + return nil +} +// next mints one id. The caller holds g.mu. +func (g *CompactGenerator) next() uint64 { nowMS := time.Now().UnixMilli() relativeMS := nowMS - CompactEpoch diff --git a/id/generator_test.go b/id/generator_test.go index 687bd9c2..1266428a 100644 --- a/id/generator_test.go +++ b/id/generator_test.go @@ -7,7 +7,7 @@ import ( "github.com/maxpert/marmot/hlc" ) -func TestHLCGenerator_NextID_Uniqueness(t *testing.T) { +func TestHLCGenerator_NextIDs_Uniqueness(t *testing.T) { clock := hlc.NewClock(1) gen := NewHLCGenerator(clock) @@ -15,7 +15,7 @@ func TestHLCGenerator_NextID_Uniqueness(t *testing.T) { const iterations = 10000 for i := 0; i < iterations; i++ { - id := gen.NextID() + id := oneID(gen) if seen[id] { t.Fatalf("duplicate ID generated at iteration %d: %d", i, id) } @@ -23,7 +23,7 @@ func TestHLCGenerator_NextID_Uniqueness(t *testing.T) { } } -func TestHLCGenerator_NextID_Monotonic(t *testing.T) { +func TestHLCGenerator_NextIDs_Monotonic(t *testing.T) { clock := hlc.NewClock(1) gen := NewHLCGenerator(clock) @@ -31,7 +31,7 @@ func TestHLCGenerator_NextID_Monotonic(t *testing.T) { const iterations = 1000 for i := 0; i < iterations; i++ { - id := gen.NextID() + id := oneID(gen) if id <= prev { t.Fatalf("non-monotonic ID at iteration %d: prev=%d, curr=%d", i, prev, id) } @@ -39,7 +39,7 @@ func TestHLCGenerator_NextID_Monotonic(t *testing.T) { } } -func TestHLCGenerator_NextID_Concurrent(t *testing.T) { +func TestHLCGenerator_NextIDs_Concurrent(t *testing.T) { clock := hlc.NewClock(1) gen := NewHLCGenerator(clock) @@ -54,7 +54,7 @@ func TestHLCGenerator_NextID_Concurrent(t *testing.T) { go func() { defer wg.Done() for i := 0; i < idsPerGoroutine; i++ { - idsChan <- gen.NextID() + idsChan <- oneID(gen) } }() } @@ -81,8 +81,8 @@ func TestHLCGenerator_DifferentNodes(t *testing.T) { gen1 := NewHLCGenerator(clock1) gen2 := NewHLCGenerator(clock2) - id1 := gen1.NextID() - id2 := gen2.NextID() + id1 := oneID(gen1) + id2 := oneID(gen2) if id1 == id2 { t.Fatalf("IDs from different nodes should differ: %d == %d", id1, id2) @@ -100,36 +100,36 @@ func TestHLCGenerator_DifferentNodes(t *testing.T) { } } -func BenchmarkHLCGenerator_NextID(b *testing.B) { +func BenchmarkHLCGenerator_NextIDs(b *testing.B) { clock := hlc.NewClock(1) gen := NewHLCGenerator(clock) b.ResetTimer() for i := 0; i < b.N; i++ { - gen.NextID() + oneID(gen) } } -func BenchmarkHLCGenerator_NextID_Parallel(b *testing.B) { +func BenchmarkHLCGenerator_NextIDs_Parallel(b *testing.B) { clock := hlc.NewClock(1) gen := NewHLCGenerator(clock) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { - gen.NextID() + oneID(gen) } }) } -func TestCompactGenerator_NextID_Uniqueness(t *testing.T) { +func TestCompactGenerator_NextIDs_Uniqueness(t *testing.T) { gen := NewCompactGenerator(1) seen := make(map[uint64]bool) const iterations = 10000 for i := 0; i < iterations; i++ { - id := gen.NextID() + id := oneID(gen) if seen[id] { t.Fatalf("duplicate ID generated at iteration %d: %d", i, id) } @@ -137,14 +137,14 @@ func TestCompactGenerator_NextID_Uniqueness(t *testing.T) { } } -func TestCompactGenerator_NextID_Monotonic(t *testing.T) { +func TestCompactGenerator_NextIDs_Monotonic(t *testing.T) { gen := NewCompactGenerator(1) var prev uint64 const iterations = 1000 for i := 0; i < iterations; i++ { - id := gen.NextID() + id := oneID(gen) if id <= prev { t.Fatalf("non-monotonic ID at iteration %d: prev=%d, curr=%d", i, prev, id) } @@ -152,7 +152,7 @@ func TestCompactGenerator_NextID_Monotonic(t *testing.T) { } } -func TestCompactGenerator_NextID_Concurrent(t *testing.T) { +func TestCompactGenerator_NextIDs_Concurrent(t *testing.T) { gen := NewCompactGenerator(1) const goroutines = 10 @@ -166,7 +166,7 @@ func TestCompactGenerator_NextID_Concurrent(t *testing.T) { go func() { defer wg.Done() for i := 0; i < idsPerGoroutine; i++ { - idsChan <- gen.NextID() + idsChan <- oneID(gen) } }() } @@ -191,8 +191,8 @@ func TestCompactGenerator_DifferentNodes(t *testing.T) { gen1 := NewCompactGenerator(1) gen2 := NewCompactGenerator(2) - id1 := gen1.NextID() - id2 := gen2.NextID() + id1 := oneID(gen1) + id2 := oneID(gen2) if id1 == id2 { t.Fatalf("IDs from different nodes should differ: %d == %d", id1, id2) @@ -216,7 +216,7 @@ func TestCompactGenerator_MaxValue(t *testing.T) { gen := NewCompactGenerator(63) // Max node ID for i := 0; i < 10000; i++ { - id := gen.NextID() + id := oneID(gen) if id > maxSafeInt { t.Fatalf("ID %d exceeds MAX_SAFE_INTEGER %d at iteration %d", id, maxSafeInt, i) } @@ -232,7 +232,7 @@ func TestCompactGenerator_SequenceOverflow(t *testing.T) { seen := make(map[uint64]bool) for i := 0; i < iterations; i++ { - id := gen.NextID() + id := oneID(gen) if seen[id] { t.Fatalf("duplicate ID at iteration %d: %d", i, id) } @@ -248,7 +248,7 @@ func TestCompactGenerator_NodeIDMasking(t *testing.T) { // Test that node IDs > 63 are properly masked gen := NewCompactGenerator(127) // Should be masked to 63 - id := gen.NextID() + id := oneID(gen) nodeID := (id >> CompactNodeShift) & CompactNodeMask if nodeID != 63 { @@ -256,22 +256,79 @@ func TestCompactGenerator_NodeIDMasking(t *testing.T) { } } -func BenchmarkCompactGenerator_NextID(b *testing.B) { +func BenchmarkCompactGenerator_NextIDs(b *testing.B) { gen := NewCompactGenerator(1) b.ResetTimer() for i := 0; i < b.N; i++ { - gen.NextID() + oneID(gen) } } -func BenchmarkCompactGenerator_NextID_Parallel(b *testing.B) { +func BenchmarkCompactGenerator_NextIDs_Parallel(b *testing.B) { gen := NewCompactGenerator(1) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { - gen.NextID() + oneID(gen) } }) } + +// oneID mints a single id. Neither generator here can fail, so an error is a +// broken invariant; it panics rather than calling t.Fatal so that it fails +// the test from any goroutine. +func oneID(g Generator) uint64 { + var ids [1]uint64 + if err := g.NextIDs(ids[:]); err != nil { + panic(err) + } + return ids[0] +} + +// TestGenerators_NextIDsBatchIsAscendingAndDisjoint pins the multi-row form: +// one call returns ascending ids, and batches minted concurrently never share +// an id. +func TestGenerators_NextIDsBatchIsAscendingAndDisjoint(t *testing.T) { + gens := map[string]Generator{ + "hlc": NewHLCGenerator(hlc.NewClock(1)), + "compact": NewCompactGenerator(1), + } + for name, gen := range gens { + t.Run(name, func(t *testing.T) { + const goroutines, batches, batchSize = 8, 50, 7 + var mu sync.Mutex + seen := make(map[uint64]bool, goroutines*batches*batchSize) + var wg sync.WaitGroup + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for b := 0; b < batches; b++ { + ids := make([]uint64, batchSize) + if err := gen.NextIDs(ids); err != nil { + t.Errorf("NextIDs: %v", err) + return + } + mu.Lock() + for i, id := range ids { + if i > 0 && id <= ids[i-1] { + t.Errorf("batch not ascending: %v", ids) + } + if seen[id] { + t.Errorf("id %d minted twice", id) + } + seen[id] = true + } + mu.Unlock() + } + }() + } + wg.Wait() + if len(seen) != goroutines*batches*batchSize { + t.Fatalf("minted %d distinct ids, want %d", len(seen), goroutines*batches*batchSize) + } + }) + } +} diff --git a/id/range_allocator.go b/id/range_allocator.go new file mode 100644 index 00000000..5708fd50 --- /dev/null +++ b/id/range_allocator.go @@ -0,0 +1,269 @@ +package id + +import ( + "context" + "errors" + "fmt" + "sort" + "sync" + "time" + + "github.com/puzpuzpuz/xsync/v3" +) + +// ErrRangeExhausted reports that a narrow AUTO_INCREMENT column has no ids +// left below its declared width. It is terminal: retrying cannot succeed, so +// it must never reach a client as the retryable lock-wait code. +var ErrRangeExhausted = errors.New("auto-increment column exhausted") + +// ErrIDOutOfRange reports a client-supplied id its column cannot hold. +var ErrIDOutOfRange = errors.New("id above the column maximum") + +// BelowBaseError refuses a client-supplied id at or below the cluster's +// allocation base that lies in no range this node claimed. Such an id may lie +// in another node's unissued range; when that node later issues it, the +// replicated row would replace the client's on every node, because replicated +// inserts apply as INSERT OR REPLACE. +type BelowBaseError struct { + ID uint64 + Base uint64 +} + +func (e *BelowBaseError) Error() string { + return fmt.Sprintf("id %d is at or below the allocation base %d", e.ID, e.Base) +} + +// RangeSizer returns how many ids to claim directly above base, the committed +// base a claim is about to propose. It is re-evaluated on every claim +// attempt, because a rejected attempt teaches the claimant a higher base, and +// both the endgame taper and the width ceiling depend on it. It returns +// ErrRangeExhausted when no claim above base can serve the request. +type RangeSizer func(base uint64) (uint64, error) + +// Claimer obtains a cluster-wide range of ids for one table: on success +// (newBase, newBase+granted] belongs to the caller alone. It is +// coordinator.WriteCoordinator.ClaimRange. +type Claimer interface { + ClaimRange(ctx context.Context, database, table string, prevBase uint64, size RangeSizer) (newBase, granted uint64, err error) +} + +// Range sizing (FINAL.md §1.1). A claim is +// +// R = min(Rcap, ramp) +// Rcap = max(1, min(rangeCap, widthMax/rangeWidthShare, (widthMax-base)/rangeTaperShare)) +// +// rangeCap bounds cross-node ordering skew; widthMax/rangeWidthShare bounds +// what one abandoned range burns to under a thousandth of the column; the +// taper term shrinks the last claims so they fit instead of being rejected; +// the ramp bounds what a short-lived process abandons. +const ( + rangeCap = 65536 + rangeWidthShare = 1024 + rangeTaperShare = 64 + rampInitial = 64 + rampFactor = 8 +) + +// RangeAllocator mints ids for narrow AUTO_INCREMENT columns from ranges +// claimed through the cluster-wide claim protocol. +// +// Its cursors live in memory only. A restarted process holds none, so its +// first insert into a table claims a fresh range; it never resumes a range +// across a process boundary (FINAL.md §1.6), and whatever the previous +// process had not issued is a permanent gap. Claims are lazy: a table no +// insert touches costs nothing. +type RangeAllocator struct { + claimer Claimer + timeout time.Duration + cursors *xsync.MapOf[tableKey, *cursor] +} + +type tableKey struct { + database string + table string +} + +// cursor is one table's local range: ids next..end are this node's alone. +// base is the highest committed base this node knows of - the end of its own +// last claim - and is what its next claim proposes. ramp is the size cap the +// next claim may ask for. owned lists every range this process claimed for +// the table, ascending and merged where adjacent: no other node can ever issue +// an id inside one. +type cursor struct { + mu sync.Mutex + next uint64 + end uint64 + base uint64 + ramp uint64 + owned []idSpan +} + +// idSpan is the closed interval [first, last]. +type idSpan struct { + first uint64 + last uint64 +} + +// owns reports whether id lies in a range this process claimed. +func (c *cursor) owns(id uint64) bool { + i := sort.Search(len(c.owned), func(i int) bool { return c.owned[i].last >= id }) + return i < len(c.owned) && c.owned[i].first <= id +} + +// addOwned records a newly claimed range. Claims only ever move upward, so it +// either extends the last span or follows it. +func (c *cursor) addOwned(first, last uint64) { + if n := len(c.owned); n > 0 && c.owned[n-1].last+1 == first { + c.owned[n-1].last = last + return + } + c.owned = append(c.owned, idSpan{first: first, last: last}) +} + +// NewRangeAllocator returns an allocator that claims through claimer, giving +// each claim at most timeout. +func NewRangeAllocator(claimer Claimer, timeout time.Duration) *RangeAllocator { + return &RangeAllocator{ + claimer: claimer, + timeout: timeout, + cursors: xsync.NewMapOf[tableKey, *cursor](), + } +} + +func (a *RangeAllocator) cursorFor(database, table string) *cursor { + c, _ := a.cursors.LoadOrCompute(tableKey{database: database, table: table}, func() *cursor { + return &cursor{ramp: rampInitial} + }) + return c +} + +// Allocate returns first, where first..first+n-1 are n contiguous ids no +// other node will ever issue for the table. A statement never straddles two +// ranges: when the current range cannot hold all n, its remainder is +// abandoned and one claim of at least n is made. +func (a *RangeAllocator) Allocate(database, table string, widthMax uint64, n int) (uint64, error) { + if n < 1 { + return 0, fmt.Errorf("allocate %d ids for %s.%s: count must be positive", n, database, table) + } + want := uint64(n) + c := a.cursorFor(database, table) + c.mu.Lock() + defer c.mu.Unlock() + + if c.next == 0 || c.next > c.end || c.end-c.next+1 < want { + if err := a.claim(c, database, table, widthMax, want, 0); err != nil { + return 0, err + } + } + first := c.next + c.next += want + return first, nil +} + +// Forget discards the table's cursor, abandoning whatever of its range is +// unissued. A cursor is valid only for the table incarnation it claimed for, +// and Forget is called when a DDL statement ends that incarnation: the ranges +// claimed for it are disjoint from every later grant under the name, but not +// from the grants under another name a renamed table's rows came from. +func (a *RangeAllocator) Forget(database, table string) { + a.cursors.Delete(tableKey{database: database, table: table}) +} + +// ForgetDatabase discards the cursor of every table in database (Forget). +func (a *RangeAllocator) ForgetDatabase(database string) { + a.cursors.Range(func(key tableKey, _ *cursor) bool { + if key.database == database { + a.cursors.Delete(key) + } + return true + }) +} + +// Admit decides whether a table may hold explicit, an id that did not come +// from Allocate: a client-supplied one, or one SQLite or a query produced. +// +// - Above widthMax it is refused with ErrIDOutOfRange. +// - Inside a range this node claimed it is admitted: no other node can +// issue it. If this node has not issued it yet, its cursor moves past it. +// - At or below the base this node knows, it is refused with a +// *BelowBaseError: it may lie in another node's unissued range. +// - Above that base, one claim covering it is made. The claim learns the +// cluster's real base through the protocol, so an id that turns out to be +// at or below it is refused as well; otherwise it now lies in this node's +// own range and is admitted. +// +// Admitting the same id twice is harmless, so every place that learns of an +// id may call it. +func (a *RangeAllocator) Admit(database, table string, widthMax, explicit uint64) error { + if explicit > widthMax { + return fmt.Errorf("%w: id %d, maximum %d", ErrIDOutOfRange, explicit, widthMax) + } + if explicit == 0 { + return nil + } + c := a.cursorFor(database, table) + c.mu.Lock() + defer c.mu.Unlock() + + if !c.owns(explicit) { + if explicit <= c.base { + return &BelowBaseError{ID: explicit, Base: c.base} + } + if err := a.claim(c, database, table, widthMax, 1, explicit); err != nil { + return err + } + if !c.owns(explicit) { + return &BelowBaseError{ID: explicit, Base: c.base} + } + } + if c.next != 0 && explicit >= c.next && explicit <= c.end { + c.next = explicit + 1 + } + return nil +} + +// claim replaces c's range with a freshly claimed one of at least want ids, +// extended to cover cover when cover lies above the base. The caller holds +// c.mu. +func (a *RangeAllocator) claim(c *cursor, database, table string, widthMax, want, cover uint64) error { + ramp := c.ramp + sizer := func(base uint64) (uint64, error) { + return claimSize(base, widthMax, want, cover, ramp) + } + + ctx, cancel := context.WithTimeout(context.Background(), a.timeout) + defer cancel() + newBase, granted, err := a.claimer.ClaimRange(ctx, database, table, c.base, sizer) + if err != nil { + return err + } + + c.next = newBase + 1 + c.end = newBase + granted + c.base = c.end + c.addOwned(c.next, c.end) + if c.ramp < rangeCap { + c.ramp *= rampFactor + } + return nil +} + +// claimSize sizes a claim above base per FINAL.md §1.1, for a request of want +// contiguous ids that must, when cover is above base, also reach cover and +// leave the same room above it. cover never exceeds widthMax. +func claimSize(base, widthMax, want, cover, ramp uint64) (uint64, error) { + if base >= widthMax || widthMax-base < want { + return 0, fmt.Errorf("%w: base %d leaves fewer than %d ids below %d", ErrRangeExhausted, base, want, widthMax) + } + size := max(min(rangeCapFor(base, widthMax), ramp), want) + if cover > base { + size += cover - base + } + return min(size, widthMax-base), nil +} + +// rangeCapFor is Rcap: max(1, min(rangeCap, widthMax/rangeWidthShare, +// (widthMax-base)/rangeTaperShare)). base is below widthMax. +func rangeCapFor(base, widthMax uint64) uint64 { + return max(1, min(rangeCap, widthMax/rangeWidthShare, (widthMax-base)/rangeTaperShare)) +} diff --git a/id/range_allocator_test.go b/id/range_allocator_test.go new file mode 100644 index 00000000..d3bb9964 --- /dev/null +++ b/id/range_allocator_test.go @@ -0,0 +1,439 @@ +package id + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +// Width ceilings of the narrow MySQL integer types, signed. +const ( + tinyMax = 127 + smallMax = 32767 + mediumMax = 8388607 + intMax = 2147483647 +) + +// fakeCluster models the claim protocol's observable contract for one table: +// a committed base that only advances, a proposal below it is rejected with +// the base (the claimant retries there, re-sizing), and a granted claim of +// size s moves the base to newBase+s. +type fakeCluster struct { + mu sync.Mutex + base map[string]uint64 + claims []fakeClaim + fail error +} + +type fakeClaim struct { + table string + prevBase, size uint64 +} + +func newFakeCluster() *fakeCluster { + return &fakeCluster{base: make(map[string]uint64)} +} + +func (f *fakeCluster) ClaimRange(_ context.Context, database, table string, prevBase uint64, size RangeSizer) (uint64, uint64, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.fail != nil { + return 0, 0, f.fail + } + key := database + "." + table + if stored := f.base[key]; stored > prevBase { + prevBase = stored // a rejection teaches the claimant the stored base + } + s, err := size(prevBase) + if err != nil { + return 0, 0, err + } + f.base[key] = prevBase + s + f.claims = append(f.claims, fakeClaim{table: table, prevBase: prevBase, size: s}) + return prevBase, s, nil +} + +func (f *fakeCluster) claimCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.claims) +} + +func (f *fakeCluster) claimSizes() []uint64 { + f.mu.Lock() + defer f.mu.Unlock() + sizes := make([]uint64, len(f.claims)) + for i, c := range f.claims { + sizes[i] = c.size + } + return sizes +} + +func mustAllocate(t *testing.T, a *RangeAllocator, table string, widthMax uint64, n int) uint64 { + t.Helper() + first, err := a.Allocate("db", table, widthMax, n) + if err != nil { + t.Fatalf("Allocate(%s, %d): %v", table, n, err) + } + return first +} + +func TestRangeAllocator_ClaimsLazilyOnFirstUse(t *testing.T) { + cluster := newFakeCluster() + a := NewRangeAllocator(cluster, time.Second) + if got := cluster.claimCount(); got != 0 { + t.Fatalf("constructing an allocator made %d claims; claims must wait for the first insert", got) + } + if first := mustAllocate(t, a, "t", intMax, 1); first != 1 { + t.Fatalf("first id = %d, want 1", first) + } + mustAllocate(t, a, "t", intMax, 1) + if got := cluster.claimCount(); got != 1 { + t.Fatalf("two inserts inside one range made %d claims, want 1", got) + } +} + +func TestRangeAllocator_RampClimbsToTheCap(t *testing.T) { + cluster := newFakeCluster() + a := NewRangeAllocator(cluster, time.Second) + // Draining each range one id at a time forces one claim per range. + for range 6 { + first := mustAllocate(t, a, "t", intMax, 1) + c := a.cursorFor("db", "t") + for id := first + 1; id <= c.end; id++ { + mustAllocate(t, a, "t", intMax, 1) + } + } + want := []uint64{64, 512, 4096, 32768, 65536, 65536} + got := cluster.claimSizes() + if len(got) != len(want) { + t.Fatalf("claim sizes %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("claim sizes %v, want %v: the ramp must climb x8 from 64 and stop at Rcap", got, want) + } + } +} + +func TestRangeCapFor(t *testing.T) { + cases := []struct { + name string + base, widthMax uint64 + want uint64 + }{ + {"tinyint at zero", 0, tinyMax, 1}, + {"smallint at zero", 0, smallMax, 31}, + {"mediumint at zero", 0, mediumMax, 8191}, + {"int at zero", 0, intMax, 65536}, + {"unsigned int at zero", 0, 4294967295, 65536}, + {"int mid-range", intMax / 2, intMax, 65536}, + {"int endgame taper", intMax - 64*100, intMax, 100}, + {"one short of the ceiling", intMax - 1, intMax, 1}, + } + for _, tc := range cases { + if got := rangeCapFor(tc.base, tc.widthMax); got != tc.want { + t.Errorf("%s: Rcap = %d, want %d", tc.name, got, tc.want) + } + } +} + +func TestRangeAllocator_SingleNodeIsDenseAcrossRangeBoundaries(t *testing.T) { + cluster := newFakeCluster() + a := NewRangeAllocator(cluster, time.Second) + for want := uint64(1); want <= 700; want++ { + if got := mustAllocate(t, a, "t", intMax, 1); got != want { + t.Fatalf("id %d issued where %d was expected: a single writer must mint 1, 2, 3 with no jump", got, want) + } + } + if cluster.claimCount() < 2 { + t.Fatalf("700 ids crossed no range boundary; the test proves nothing") + } +} + +func TestRangeAllocator_NAtOnceIsContiguousAndNeverStraddles(t *testing.T) { + cluster := newFakeCluster() + a := NewRangeAllocator(cluster, time.Second) + mustAllocate(t, a, "t", intMax, 60) // ids 1..60 of the first range (64) + + // Four left in the range; a five-row statement must not take them. + first := mustAllocate(t, a, "t", intMax, 5) + if first != 65 { + t.Fatalf("five-row statement started at %d, want 65: it must start a fresh range, not straddle", first) + } + sizes := cluster.claimSizes() + if len(sizes) != 2 { + t.Fatalf("claims %v, want two", sizes) + } + + // A statement larger than the ramp claims at least its own size. + first = mustAllocate(t, a, "t", intMax, 5000) + c := a.cursorFor("db", "t") + if c.end-first+1 < 5000 { + t.Fatalf("5000-row statement got a range ending at %d from %d", c.end, first) + } + next := mustAllocate(t, a, "t", intMax, 1) + if next <= first+4999 { + t.Fatalf("id %d reissued inside the 5000-row statement's ids", next) + } +} + +func TestRangeAllocator_ExhaustionIsTerminal(t *testing.T) { + cluster := newFakeCluster() + a := NewRangeAllocator(cluster, time.Second) + for want := uint64(1); want <= tinyMax; want++ { + if got := mustAllocate(t, a, "tiny", tinyMax, 1); got != want { + t.Fatalf("TINYINT id %d, want %d", got, want) + } + } + _, err := a.Allocate("db", "tiny", tinyMax, 1) + if !errors.Is(err, ErrRangeExhausted) { + t.Fatalf("insert past 127 into TINYINT: err = %v, want ErrRangeExhausted", err) + } + + // A multi-row statement that cannot fit in what is left is refused whole. + b := NewRangeAllocator(newFakeCluster(), time.Second) + mustAllocate(t, b, "s", smallMax, smallMax-3) + if _, err := b.Allocate("db", "s", smallMax, 5); !errors.Is(err, ErrRangeExhausted) { + t.Fatalf("five rows into a SMALLINT with three ids left: err = %v, want ErrRangeExhausted", err) + } +} + +func TestRangeAllocator_RestartClaimsFreshAndNeverReissues(t *testing.T) { + cluster := newFakeCluster() + first := NewRangeAllocator(cluster, time.Second) + var highest uint64 + for range 10 { + highest = mustAllocate(t, first, "t", intMax, 1) + } + + // A new process over the same cluster: the old range's unissued tail + // (11..64) is a gap, never a source of ids. + restarted := NewRangeAllocator(cluster, time.Second) + got := mustAllocate(t, restarted, "t", intMax, 1) + if got <= highest || got <= 64 { + t.Fatalf("restarted allocator issued %d; ids up to %d were issued and ids up to 64 were claimed before the restart", got, highest) + } +} + +func TestRangeAllocator_TwoNodesNeverShareAnID(t *testing.T) { + cluster := newFakeCluster() + nodes := []*RangeAllocator{ + NewRangeAllocator(cluster, time.Second), + NewRangeAllocator(cluster, time.Second), + NewRangeAllocator(cluster, time.Second), + } + var mu sync.Mutex + seen := make(map[uint64]bool) + var wg sync.WaitGroup + for _, node := range nodes { + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + for i := range 300 { + n := 1 + i%3 + first, err := node.Allocate("db", "t", smallMax, n) + if err != nil { + t.Errorf("Allocate: %v", err) + return + } + mu.Lock() + for id := first; id < first+uint64(n); id++ { + if id == 0 || id > smallMax { + t.Errorf("id %d outside [1, %d]", id, smallMax) + } + if seen[id] { + t.Errorf("id %d issued twice", id) + } + seen[id] = true + } + mu.Unlock() + } + }() + } + } + wg.Wait() +} + +// TestRangeAllocator_AdmitExplicitIDs pins the admission rule for ids that +// did not come from Allocate. +func TestRangeAllocator_AdmitExplicitIDs(t *testing.T) { + t.Run("inside the local unissued range moves the cursor past it", func(t *testing.T) { + cluster := newFakeCluster() + a := NewRangeAllocator(cluster, time.Second) + mustAllocate(t, a, "t", intMax, 1) + if err := a.Admit("db", "t", intMax, 40); err != nil { + t.Fatal(err) + } + if got := mustAllocate(t, a, "t", intMax, 1); got != 41 { + t.Fatalf("next id after explicit 40 = %d, want 41", got) + } + if cluster.claimCount() != 1 { + t.Fatalf("an id inside the local range made a claim") + } + }) + + t.Run("an id this node already issued is admitted again", func(t *testing.T) { + a := NewRangeAllocator(newFakeCluster(), time.Second) + first := mustAllocate(t, a, "t", intMax, 3) + for id := first; id < first+3; id++ { + if err := a.Admit("db", "t", intMax, id); err != nil { + t.Fatalf("generated id %d refused: %v", id, err) + } + } + if got := mustAllocate(t, a, "t", intMax, 1); got != first+3 { + t.Fatalf("admitting issued ids moved the cursor: next = %d, want %d", got, first+3) + } + }) + + t.Run("above every known base raises the cluster base past it", func(t *testing.T) { + cluster := newFakeCluster() + a := NewRangeAllocator(cluster, time.Second) + mustAllocate(t, a, "t", intMax, 1) + if err := a.Admit("db", "t", intMax, 5000); err != nil { + t.Fatal(err) + } + if base := cluster.base["db.t"]; base < 5000 { + t.Fatalf("cluster base %d after an explicit id 5000; another node could still issue 5000", base) + } + if got := mustAllocate(t, a, "t", intMax, 1); got != 5001 { + t.Fatalf("next id after explicit 5000 = %d, want 5001", got) + } + other := NewRangeAllocator(cluster, time.Second) + if got := mustAllocate(t, other, "t", intMax, 1); got <= 5000 { + t.Fatalf("another node issued %d after explicit 5000 raised the base", got) + } + }) + + t.Run("inside another node's range is refused", func(t *testing.T) { + // Mutation: admit any id at or below the base. "an id inside + // another node's unissued range was admitted" fires. + cluster := newFakeCluster() + owner := NewRangeAllocator(cluster, time.Second) + mustAllocate(t, owner, "t", intMax, 1) // owner holds 1..64 + other := NewRangeAllocator(cluster, time.Second) + mustAllocate(t, other, "t", intMax, 1) // other learns base 64, holds 65..128 + + err := other.Admit("db", "t", intMax, 30) + var below *BelowBaseError + if !errors.As(err, &below) { + t.Fatalf("an id inside another node's unissued range was admitted: err = %v", err) + } + + // A node whose knowledge is stale learns the base through the claim + // protocol and refuses as well. + stale := NewRangeAllocator(cluster, time.Second) + if err := stale.Admit("db", "t", intMax, 100); !errors.As(err, &below) { + t.Fatalf("a node with a stale base admitted id 100 inside another node's range: err = %v", err) + } + }) + + t.Run("above the column maximum is refused", func(t *testing.T) { + a := NewRangeAllocator(newFakeCluster(), time.Second) + if err := a.Admit("db", "t", tinyMax, tinyMax+1); !errors.Is(err, ErrIDOutOfRange) { + t.Fatalf("explicit id 128 for a TINYINT column: err = %v, want ErrIDOutOfRange", err) + } + }) + + t.Run("a mysqldump restore admits every row", func(t *testing.T) { + a := NewRangeAllocator(newFakeCluster(), time.Second) + for id := uint64(1); id <= 1000; id++ { + if err := a.Admit("db", "t", intMax, id); err != nil { + t.Fatalf("dump row %d refused: %v", id, err) + } + } + if got := mustAllocate(t, a, "t", intMax, 1); got <= 1000 { + t.Fatalf("first generated id after the dump = %d, want above 1000", got) + } + }) +} + +// TestRangeAllocator_ForgetDiscardsTheRangeOfAReplacedName is the cross-name +// case the per-name monotone base does not cover on its own: n1 holds an +// unissued range of a dropped t, n2 fills a new t2 from t2's own grants, and +// t2 is renamed to t, inheriting t's base. n1's old range for t overlaps the +// ids the renamed table holds, so the rename makes n1 forget it and claim +// fresh above the inherited base. +// +// Mutation: make Forget a no-op. n1 issues 11, n2's id, and the first +// assertion fires. +func TestRangeAllocator_ForgetDiscardsTheRangeOfAReplacedName(t *testing.T) { + cluster := newFakeCluster() + n1 := NewRangeAllocator(cluster, time.Second) + n2 := NewRangeAllocator(cluster, time.Second) + + for i := 0; i < 10; i++ { + mustAllocate(t, n1, "t", intMax, 1) + } + for i := 0; i < 20; i++ { + mustAllocate(t, n2, "t2", intMax, 1) + } + // RENAME t2 TO t: t inherits t2's base (db.AutoIncClaimStore.Inherit). + cluster.mu.Lock() + cluster.base["db.t"] = max(cluster.base["db.t"], cluster.base["db.t2"]) + cluster.mu.Unlock() + n1.Forget("db", "t") + + if got := mustAllocate(t, n1, "t", intMax, 1); got <= 64 { + t.Fatalf("after the rename n1 issued %d, inside the ids the renamed table holds (1..64)", got) + } + var below *BelowBaseError + if err := n1.Admit("db", "t", intMax, 11); !errors.As(err, &below) { + t.Fatalf("after the rename n1 admitted explicit id 11 from its forgotten range: %v", err) + } +} + +// TestRangeAllocator_ForgetDatabaseDiscardsEveryTableOfIt is reviewer A's DROP +// DATABASE + recreate: n1 holds an unissued range of d.t when d is dropped. +// The claim row survives, so the recreated d.t's grants start above it, and +// the drop makes n1 forget its range so n1 claims fresh too. Another +// database's cursor is kept. +// +// Mutation: make ForgetDatabase a no-op. n1 issues 4 from the dropped +// incarnation's range and the first assertion fires. +func TestRangeAllocator_ForgetDatabaseDiscardsEveryTableOfIt(t *testing.T) { + cluster := newFakeCluster() + n1 := NewRangeAllocator(cluster, time.Second) + for i := 0; i < 3; i++ { + mustAllocate(t, n1, "t", intMax, 1) + } + other, err := n1.Allocate("other", "t", intMax, 1) + if err != nil { + t.Fatal(err) + } + + n1.ForgetDatabase("db") + if got := mustAllocate(t, n1, "t", intMax, 1); got <= 64 { + t.Fatalf("after DROP DATABASE n1 issued %d from the dropped incarnation's range (1..64)", got) + } + if got, err := n1.Allocate("other", "t", intMax, 1); err != nil || got != other+1 { + t.Fatalf("DROP DATABASE db disturbed other.t's cursor: got %d, %v, want %d", got, err, other+1) + } +} + +func TestRangeAllocator_ClaimFailurePropagates(t *testing.T) { + cluster := newFakeCluster() + unavailable := errors.New("quorum not reached") + cluster.fail = unavailable + a := NewRangeAllocator(cluster, time.Second) + _, err := a.Allocate("db", "t", intMax, 1) + if !errors.Is(err, unavailable) { + t.Fatalf("err = %v, want the claimer's error", err) + } + if errors.Is(err, ErrRangeExhausted) { + t.Fatalf("an unavailable quorum was reported as exhaustion") + } +} + +func BenchmarkRangeAllocator_Allocate(b *testing.B) { + a := NewRangeAllocator(newFakeCluster(), time.Second) + b.ReportAllocs() + for b.Loop() { + if _, err := a.Allocate("db", "t", 4294967295, 1); err != nil { + b.Fatal(err) + } + } +} diff --git a/marmot.go b/marmot.go index e4b85412..0fb214a4 100644 --- a/marmot.go +++ b/marmot.go @@ -239,6 +239,9 @@ func main() { } } + // A node whose system database this process initialises holds its + // AUTO_INCREMENT claim votes until it has merged claim bases from its + // peers (db.AutoIncHoldTable), whatever the catch-up decision was. dbMgr, err := db.NewDatabaseManager(cfg.Config.DataDir, cfg.Config.NodeID, clock) if err != nil { log.Fatal().Err(err).Msg("Failed to initialize Database Manager") @@ -399,9 +402,22 @@ func main() { log.Info().Msg("Anti-entropy service initialized") + // A node whose AUTO_INCREMENT claim votes are held (a joiner, or one whose + // system database came from a peer) votes again only after merging claim + // bases from enough members. From then on - at once on any other node - + // it raises its bases to every alive peer's, every interval, so a + // membership change leaves a claim unknown to a new quorum for at most one + // interval. + autoIncMergeCtx, autoIncMergeCancel := context.WithCancel(context.Background()) + defer autoIncMergeCancel() + go marmotgrpc.RunAutoIncBaseMerge(autoIncMergeCtx, cfg.Config.NodeID, dbMgr, grpcServer.GetNodeRegistry(), client, cfg.Config.Cluster.Standalone) + // Phase 7: Setup transaction coordinators for full database replication log.Info().Msg("Setting up transaction coordinators") nodeProvider := marmotgrpc.NewGossipNodeProvider(gossip.GetNodeRegistry(), len(cfg.Config.Cluster.SeedNodes) > 0) + // An AUTO_INCREMENT claim participant counts the cluster exactly as a + // claimant's quorum does. + dbMgr.SetClusterMembership(nodeProvider.GetTotalMembershipSize) replicator := marmotgrpc.NewGRPCReplicator(client) writeTimeout := time.Duration(cfg.Config.Replication.WriteTimeoutMS) * time.Millisecond @@ -418,6 +434,9 @@ func main() { writeTimeout, clock, ) + // AUTO_INCREMENT claims carry this node's schema version, so a voter + // that lags a DDL declines them as it declines DML. + writeCoordinator.SetSchemaVersionSource(schemaVersionMgr.GetSchemaVersion) // Initialize LocalReader for ReadCoordinator localReader := db.NewLocalReader(dbMgr) @@ -453,6 +472,7 @@ func main() { registryAdapter, ) handler.SetVectorEngine(vecEngine) + dbMgr.SetAutoIncIncarnationListener(handler) // Initialize write forwarding for read-only replicas forwardSessionMgr := marmotgrpc.NewForwardSessionManager(60 * time.Second) diff --git a/protocol/autoinc_claim.go b/protocol/autoinc_claim.go index cb05cc8e..bcedcfe6 100644 --- a/protocol/autoinc_claim.go +++ b/protocol/autoinc_claim.go @@ -1,6 +1,7 @@ package protocol import ( + "errors" "fmt" "github.com/maxpert/marmot/encoding" @@ -21,13 +22,31 @@ import ( // participant evaluates at PREPARE. It carries no width: each participant // derives widthMax from its OWN sqlite_master, because trusting the claimant // would let a node with a stale schema be rubber-stamped. +// +// Membership is the claimant's view of the cluster's total membership when it +// sent the claim: the denominator its quorum was computed over. A +// participant whose own view differs rejects the claim, because majorities of +// two different memberships need not intersect. It is a named field, so a +// payload from a binary that did not send it decodes with Membership 0, which +// no participant accepts. type AutoIncClaim struct { - Table string `msgpack:"table"` - PrevBase uint64 `msgpack:"prev_base"` - NewBase uint64 `msgpack:"new_base"` - Size uint64 `msgpack:"size"` + Table string `msgpack:"table"` + PrevBase uint64 `msgpack:"prev_base"` + NewBase uint64 `msgpack:"new_base"` + Size uint64 `msgpack:"size"` + Membership uint32 `msgpack:"membership"` } +// ErrAutoIncClaimNotApplicable reports a COMMIT whose claim a node cannot +// apply: the claim intent written at PREPARE is gone, or the node's committed +// or seed floor has passed the claim's newBase since PREPARE +// (db.AutoIncClaimStore.ApplyClaims). ACKing would let the node count toward +// a quorum for a range it may also have applied for another claimant, so it +// refuses and leaves the transaction to recovery. The refusal is the protocol +// working, not a node that diverged from its quorum. It lives here so the +// coordinator can tell it from a failed local commit without importing db. +var ErrAutoIncClaimNotApplicable = errors.New("auto-increment claim cannot be applied") + // EncodeAutoIncClaim renders a claim payload for an intent's DataSnapshot. func EncodeAutoIncClaim(claim AutoIncClaim) ([]byte, error) { data, err := encoding.Marshal(claim) diff --git a/protocol/autoinc_claim_test.go b/protocol/autoinc_claim_test.go index dc6c72e2..e55e8a5f 100644 --- a/protocol/autoinc_claim_test.go +++ b/protocol/autoinc_claim_test.go @@ -1,6 +1,10 @@ package protocol -import "testing" +import ( + "testing" + + "github.com/maxpert/marmot/encoding" +) // TestAutoIncClaimWireShapeIsPinned pins the exact msgpack encoding of an // AutoIncClaim payload: this is the wire contract a claim intent's @@ -9,18 +13,18 @@ import "testing" // without every node in a mixed-version cluster being able to decode it. // // The byte length below was captured from this exact field set - table -// "users", prev_base 100, new_base 164, size 64 - encoded through +// "users", prev_base 100, new_base 164, size 64, membership 3 - encoded through // encoding.Marshal. A change to this length signals an encoding change that // needs a deliberate wire-compatibility decision, not a silent drift. func TestAutoIncClaimWireShapeIsPinned(t *testing.T) { - claim := AutoIncClaim{Table: "users", PrevBase: 100, NewBase: 164, Size: 64} + claim := AutoIncClaim{Table: "users", PrevBase: 100, NewBase: 164, Size: 64, Membership: 3} data, err := EncodeAutoIncClaim(claim) if err != nil { t.Fatalf("EncodeAutoIncClaim: %v", err) } - const wantLen = 64 + const wantLen = 80 if len(data) != wantLen { t.Fatalf("encoded length = %d, want %d (wire shape changed: %x)", len(data), wantLen, data) } @@ -33,3 +37,29 @@ func TestAutoIncClaimWireShapeIsPinned(t *testing.T) { t.Fatalf("round trip = %+v, want %+v", got, claim) } } + +// TestAutoIncClaimWithoutMembershipDecodesAsZero pins what a participant sees +// from a claimant that predates the membership field: the payload still +// decodes, with Membership 0, which the PREPARE condition refuses +// (db.prepareAutoIncClaim), rather than failing to decode or being read as a +// real size. +func TestAutoIncClaimWithoutMembershipDecodesAsZero(t *testing.T) { + type legacyClaim struct { + Table string `msgpack:"table"` + PrevBase uint64 `msgpack:"prev_base"` + NewBase uint64 `msgpack:"new_base"` + Size uint64 `msgpack:"size"` + } + data, err := encoding.Marshal(legacyClaim{Table: "users", PrevBase: 100, NewBase: 100, Size: 64}) + if err != nil { + t.Fatalf("marshal legacy claim: %v", err) + } + got, err := DecodeAutoIncClaim(data) + if err != nil { + t.Fatalf("a claim without the membership field must still decode: %v", err) + } + want := AutoIncClaim{Table: "users", PrevBase: 100, NewBase: 100, Size: 64} + if got != want { + t.Fatalf("decoded %+v, want %+v with Membership 0", got, want) + } +} diff --git a/protocol/handlers/information_schema.go b/protocol/handlers/information_schema.go index 5a425321..938464b3 100644 --- a/protocol/handlers/information_schema.go +++ b/protocol/handlers/information_schema.go @@ -16,16 +16,20 @@ func (m *MetadataHandler) HandleInformationSchema(currentDB string, stmt protoco Str("filter_table", stmt.ISFilter.TableName). Msg("Handling INFORMATION_SCHEMA query") + var ( + rows [][]interface{} + err error + ) // Route based on pre-parsed table type (no string matching needed) switch stmt.ISTableType { case protocol.ISTableTables: - return m.handleInformationSchemaTables(currentDB, stmt.ISFilter) + rows, err = m.informationSchemaTables(currentDB, stmt.ISFilter) case protocol.ISTableColumns: - return m.handleInformationSchemaColumns(currentDB, stmt.ISFilter) + rows, err = m.informationSchemaColumns(currentDB, stmt.ISFilter) case protocol.ISTableSchemata: - return m.handleInformationSchemaSchemata(stmt.ISFilter) + rows = m.informationSchemaSchemata(stmt.ISFilter) case protocol.ISTableStatistics: - return m.handleInformationSchemaStatistics(currentDB, stmt.ISFilter) + rows, err = m.informationSchemaStatistics(currentDB, stmt.ISFilter) default: // Unsupported INFORMATION_SCHEMA table - return empty result return &protocol.ResultSet{ @@ -33,208 +37,130 @@ func (m *MetadataHandler) HandleInformationSchema(currentDB string, stmt protoco Rows: [][]interface{}{}, }, nil } + if err != nil { + return nil, err + } + if rows == nil { + rows = [][]interface{}{} + } + return &protocol.ResultSet{ + Columns: protocol.InformationSchemaColumns(stmt.ISTableType), + Rows: rows, + }, nil } -// handleInformationSchemaTables handles queries to INFORMATION_SCHEMA.TABLES -func (m *MetadataHandler) handleInformationSchemaTables(currentDB string, filter protocol.InformationSchemaFilter) (*protocol.ResultSet, error) { - // Use filter value or fall back to current database +// visibleDatabase reports whether a client may see name in +// INFORMATION_SCHEMA: it exists and is not Marmot's system database. A schema +// a client cannot see has no tables, exactly as MySQL answers for one that +// does not exist. +func (m *MetadataHandler) visibleDatabase(name string) bool { + return name != "" && name != m.systemDatabaseName && m.db.DatabaseExists(name) +} + +// informationSchemaTables answers INFORMATION_SCHEMA.TABLES: every table of +// the filtered schema (else the session's, else every schema). +func (m *MetadataHandler) informationSchemaTables(currentDB string, filter protocol.InformationSchemaFilter) ([][]interface{}, error) { dbName := filter.SchemaName if dbName == "" { dbName = currentDB } + schemas := []string{dbName} if dbName == "" { - // Return all tables from all databases - return m.getAllTablesFromAllDatabases() - } - - log.Debug(). - Str("database", dbName). - Str("filter_table", filter.TableName). - Msg("Querying INFORMATION_SCHEMA.TABLES") - - sqlDB, err := m.db.GetDatabaseConnection(dbName) - if err != nil { - return nil, err - } - - // Query SQLite's sqlite_master table with optional table name filter - var rows *sql.Rows - if filter.TableName != "" { - // Filter by specific table name - rows, err = sqlDB.Query("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '__marmot__%' AND name = ? ORDER BY name", filter.TableName) - } else { - rows, err = sqlDB.Query("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '__marmot__%' ORDER BY name") - } - if err != nil { - return nil, err + schemas = m.db.ListDatabases() } - defer rows.Close() - var tables [][]interface{} - for rows.Next() { - var tableName string - if err := rows.Scan(&tableName); err != nil { + var rows [][]interface{} + for _, schema := range schemas { + if !m.visibleDatabase(schema) { continue } - - tables = append(tables, []interface{}{ - "def", // TABLE_CATALOG - dbName, // TABLE_SCHEMA - tableName, // TABLE_NAME - "BASE TABLE", // TABLE_TYPE - "SQLite", // ENGINE - 10, // VERSION - "Dynamic", // ROW_FORMAT - nil, // TABLE_ROWS - nil, // AVG_ROW_LENGTH - nil, // DATA_LENGTH - nil, // MAX_DATA_LENGTH - nil, // INDEX_LENGTH - nil, // DATA_FREE - nil, // AUTO_INCREMENT - nil, // CREATE_TIME - nil, // UPDATE_TIME - nil, // CHECK_TIME - "utf8mb4_general_ci", // TABLE_COLLATION - nil, // CHECKSUM - "", // CREATE_OPTIONS - "", // TABLE_COMMENT - }) + tables, err := m.userTables(schema, filter.TableName) + if err != nil { + return nil, err + } + for _, tableName := range tables { + rows = append(rows, informationSchemaTablesRow(schema, tableName)) + } } + return rows, nil +} - return &protocol.ResultSet{ - Columns: []protocol.ColumnDef{ - {Name: "TABLE_CATALOG", Type: 0xFD}, - {Name: "TABLE_SCHEMA", Type: 0xFD}, - {Name: "TABLE_NAME", Type: 0xFD}, - {Name: "TABLE_TYPE", Type: 0xFD}, - {Name: "ENGINE", Type: 0xFD}, - {Name: "VERSION", Type: 0x03}, - {Name: "ROW_FORMAT", Type: 0xFD}, - {Name: "TABLE_ROWS", Type: 0x08}, - {Name: "AVG_ROW_LENGTH", Type: 0x08}, - {Name: "DATA_LENGTH", Type: 0x08}, - {Name: "MAX_DATA_LENGTH", Type: 0x08}, - {Name: "INDEX_LENGTH", Type: 0x08}, - {Name: "DATA_FREE", Type: 0x08}, - {Name: "AUTO_INCREMENT", Type: 0x08}, - {Name: "CREATE_TIME", Type: 0x0C}, - {Name: "UPDATE_TIME", Type: 0x0C}, - {Name: "CHECK_TIME", Type: 0x0C}, - {Name: "TABLE_COLLATION", Type: 0xFD}, - {Name: "CHECKSUM", Type: 0x08}, - {Name: "CREATE_OPTIONS", Type: 0xFD}, - {Name: "TABLE_COMMENT", Type: 0xFD}, - }, - Rows: tables, - }, nil +// informationSchemaTablesRow is one INFORMATION_SCHEMA.TABLES row, in +// protocol.InformationSchemaColumns(ISTableTables) order. +func informationSchemaTablesRow(schema, tableName string) []interface{} { + return []interface{}{ + "def", // TABLE_CATALOG + schema, // TABLE_SCHEMA + tableName, // TABLE_NAME + "BASE TABLE", // TABLE_TYPE + "SQLite", // ENGINE + 10, // VERSION + "Dynamic", // ROW_FORMAT + nil, // TABLE_ROWS + nil, // AVG_ROW_LENGTH + nil, // DATA_LENGTH + nil, // MAX_DATA_LENGTH + nil, // INDEX_LENGTH + nil, // DATA_FREE + nil, // AUTO_INCREMENT + nil, // CREATE_TIME + nil, // UPDATE_TIME + nil, // CHECK_TIME + "utf8mb4_general_ci", // TABLE_COLLATION + nil, // CHECKSUM + "", // CREATE_OPTIONS + "", // TABLE_COMMENT + } } -// handleInformationSchemaColumns handles queries to INFORMATION_SCHEMA.COLUMNS -func (m *MetadataHandler) handleInformationSchemaColumns(currentDB string, filter protocol.InformationSchemaFilter) (*protocol.ResultSet, error) { - // Use filter values or fall back to current database +// informationSchemaColumns answers INFORMATION_SCHEMA.COLUMNS for the +// filtered table, or for every table of the schema. +func (m *MetadataHandler) informationSchemaColumns(currentDB string, filter protocol.InformationSchemaFilter) ([][]interface{}, error) { dbName := filter.SchemaName - tableName := filter.TableName - if dbName == "" { dbName = currentDB } - if dbName == "" { return nil, fmt.Errorf("no database specified") } + if !m.visibleDatabase(dbName) { + return nil, nil + } log.Debug(). Str("database", dbName). - Str("table", tableName). + Str("table", filter.TableName). Msg("Querying INFORMATION_SCHEMA.COLUMNS") - var columns [][]interface{} - - if tableName != "" { - // Query specific table - cols, err := m.getTableColumns(dbName, tableName) - if err != nil { - return nil, err - } - columns = append(columns, cols...) - } else { - // Query all tables - sqlDB, err := m.db.GetDatabaseConnection(dbName) - if err != nil { - return nil, err - } - - rows, err := sqlDB.Query("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '__marmot__%'") + // The table list is read to the end before any column is read, so no + // query runs while another holds a connection. + tables, err := m.userTables(dbName, filter.TableName) + if err != nil { + return nil, err + } + var rows [][]interface{} + for _, tableName := range tables { + cols, err := m.tableColumns(dbName, tableName) if err != nil { return nil, err } - defer rows.Close() - - for rows.Next() { - var tblName string - if err := rows.Scan(&tblName); err != nil { - continue - } - cols, err := m.getTableColumns(dbName, tblName) - if err != nil { - continue - } - columns = append(columns, cols...) - } + rows = append(rows, cols...) } - - return &protocol.ResultSet{ - Columns: []protocol.ColumnDef{ - {Name: "TABLE_CATALOG", Type: 0xFD}, - {Name: "TABLE_SCHEMA", Type: 0xFD}, - {Name: "TABLE_NAME", Type: 0xFD}, - {Name: "COLUMN_NAME", Type: 0xFD}, - {Name: "ORDINAL_POSITION", Type: 0x03}, - {Name: "COLUMN_DEFAULT", Type: 0xFD}, - {Name: "IS_NULLABLE", Type: 0xFD}, - {Name: "DATA_TYPE", Type: 0xFD}, - {Name: "CHARACTER_MAXIMUM_LENGTH", Type: 0x08}, - {Name: "CHARACTER_OCTET_LENGTH", Type: 0x08}, - {Name: "NUMERIC_PRECISION", Type: 0x08}, - {Name: "NUMERIC_SCALE", Type: 0x08}, - {Name: "DATETIME_PRECISION", Type: 0x08}, - {Name: "CHARACTER_SET_NAME", Type: 0xFD}, - {Name: "COLLATION_NAME", Type: 0xFD}, - {Name: "COLUMN_TYPE", Type: 0xFD}, - {Name: "COLUMN_KEY", Type: 0xFD}, - {Name: "EXTRA", Type: 0xFD}, - {Name: "PRIVILEGES", Type: 0xFD}, - {Name: "COLUMN_COMMENT", Type: 0xFD}, - }, - Rows: columns, - }, nil + return rows, nil } -// handleInformationSchemaSchemata handles queries to INFORMATION_SCHEMA.SCHEMATA -func (m *MetadataHandler) handleInformationSchemaSchemata(filter protocol.InformationSchemaFilter) (*protocol.ResultSet, error) { - schemaName := filter.SchemaName - - log.Debug(). - Str("schema", schemaName). - Msg("Querying INFORMATION_SCHEMA.SCHEMATA") - - databases := m.db.ListDatabases() - - var schemas [][]interface{} - for _, dbName := range databases { - // Skip system database +// informationSchemaSchemata answers INFORMATION_SCHEMA.SCHEMATA. +func (m *MetadataHandler) informationSchemaSchemata(filter protocol.InformationSchemaFilter) [][]interface{} { + var rows [][]interface{} + for _, dbName := range m.db.ListDatabases() { if dbName == m.systemDatabaseName { continue } - - // Filter by schema name if specified - if schemaName != "" && dbName != schemaName { + if filter.SchemaName != "" && dbName != filter.SchemaName { continue } - - schemas = append(schemas, []interface{}{ + rows = append(rows, []interface{}{ "def", // CATALOG_NAME dbName, // SCHEMA_NAME "utf8mb4", // DEFAULT_CHARACTER_SET_NAME @@ -242,135 +168,60 @@ func (m *MetadataHandler) handleInformationSchemaSchemata(filter protocol.Inform nil, // SQL_PATH }) } - - return &protocol.ResultSet{ - Columns: []protocol.ColumnDef{ - {Name: "CATALOG_NAME", Type: 0xFD}, - {Name: "SCHEMA_NAME", Type: 0xFD}, - {Name: "DEFAULT_CHARACTER_SET_NAME", Type: 0xFD}, - {Name: "DEFAULT_COLLATION_NAME", Type: 0xFD}, - {Name: "SQL_PATH", Type: 0xFD}, - }, - Rows: schemas, - }, nil + return rows } -// handleInformationSchemaStatistics handles queries to INFORMATION_SCHEMA.STATISTICS -func (m *MetadataHandler) handleInformationSchemaStatistics(currentDB string, filter protocol.InformationSchemaFilter) (*protocol.ResultSet, error) { +// informationSchemaStatistics answers INFORMATION_SCHEMA.STATISTICS for the +// filtered table; without a table it answers no rows. +func (m *MetadataHandler) informationSchemaStatistics(currentDB string, filter protocol.InformationSchemaFilter) ([][]interface{}, error) { dbName := filter.SchemaName - tableName := filter.TableName - if dbName == "" { dbName = currentDB } - - if dbName == "" || tableName == "" { - return &protocol.ResultSet{ - Columns: []protocol.ColumnDef{ - {Name: "TABLE_CATALOG", Type: 0xFD}, - {Name: "TABLE_SCHEMA", Type: 0xFD}, - {Name: "TABLE_NAME", Type: 0xFD}, - {Name: "NON_UNIQUE", Type: 0x03}, - {Name: "INDEX_SCHEMA", Type: 0xFD}, - {Name: "INDEX_NAME", Type: 0xFD}, - {Name: "SEQ_IN_INDEX", Type: 0x03}, - {Name: "COLUMN_NAME", Type: 0xFD}, - }, - Rows: [][]interface{}{}, - }, nil + if filter.TableName == "" || !m.visibleDatabase(dbName) { + return nil, nil } - // Use SHOW INDEXES handler - result, err := m.HandleShowIndexes(dbName, tableName) + result, err := m.HandleShowIndexes(dbName, filter.TableName) if err != nil { return nil, err } - // Convert to INFORMATION_SCHEMA.STATISTICS format - var statistics [][]interface{} + var rows [][]interface{} for _, row := range result.Rows { - if len(row) >= 5 { - statistics = append(statistics, []interface{}{ - "def", // TABLE_CATALOG - dbName, // TABLE_SCHEMA - row[0], // TABLE_NAME - row[1], // NON_UNIQUE - dbName, // INDEX_SCHEMA - row[2], // INDEX_NAME - row[3], // SEQ_IN_INDEX - row[4], // COLUMN_NAME - }) - } + rows = append(rows, []interface{}{ + "def", // TABLE_CATALOG + dbName, // TABLE_SCHEMA + row[0], // TABLE_NAME + row[1], // NON_UNIQUE + dbName, // INDEX_SCHEMA + row[2], // INDEX_NAME + row[3], // SEQ_IN_INDEX + row[4], // COLUMN_NAME + }) } - - return &protocol.ResultSet{ - Columns: []protocol.ColumnDef{ - {Name: "TABLE_CATALOG", Type: 0xFD}, - {Name: "TABLE_SCHEMA", Type: 0xFD}, - {Name: "TABLE_NAME", Type: 0xFD}, - {Name: "NON_UNIQUE", Type: 0x03}, - {Name: "INDEX_SCHEMA", Type: 0xFD}, - {Name: "INDEX_NAME", Type: 0xFD}, - {Name: "SEQ_IN_INDEX", Type: 0x03}, - {Name: "COLUMN_NAME", Type: 0xFD}, - }, - Rows: statistics, - }, nil + return rows, nil } -// getTableColumns retrieves column information for a specific table -func (m *MetadataHandler) getTableColumns(dbName, tableName string) ([][]interface{}, error) { - sqlDB, err := m.db.GetDatabaseConnection(dbName) +// tableColumns returns a table's INFORMATION_SCHEMA.COLUMNS rows. +func (m *MetadataHandler) tableColumns(dbName, tableName string) ([][]interface{}, error) { + infos, err := m.columnInfos(dbName, tableName) if err != nil { return nil, err } - rows, err := sqlDB.Query(fmt.Sprintf("PRAGMA table_info(%s)", tableName)) - if err != nil { - return nil, err - } - defer rows.Close() - - var columns [][]interface{} - for rows.Next() { - var cid int - var name, colType string - var notNull int - var dfltValue sql.NullString - var pk int - - if err := rows.Scan(&cid, &name, &colType, ¬Null, &dfltValue, &pk); err != nil { - continue - } - - // Convert SQLite types to MySQL-compatible format - mysqlType := sqliteToMySQLType(colType) - dataType := extractDataType(mysqlType) - - nullable := "YES" - if notNull == 1 { - nullable = "NO" - } - - key := "" - if pk > 0 { - key = "PRI" - } - - defaultVal := interface{}(nil) - if dfltValue.Valid { - defaultVal = dfltValue.String - } - + columns := make([][]interface{}, 0, len(infos)) + for _, info := range infos { + mysqlType := sqliteToMySQLType(info.declType) columns = append(columns, []interface{}{ "def", // TABLE_CATALOG dbName, // TABLE_SCHEMA tableName, // TABLE_NAME - name, // COLUMN_NAME - cid + 1, // ORDINAL_POSITION - defaultVal, // COLUMN_DEFAULT - nullable, // IS_NULLABLE - dataType, // DATA_TYPE + info.name, // COLUMN_NAME + info.cid + 1, // ORDINAL_POSITION + info.defaultValue(), // COLUMN_DEFAULT + info.nullable(), // IS_NULLABLE + extractDataType(mysqlType), // DATA_TYPE nil, // CHARACTER_MAXIMUM_LENGTH nil, // CHARACTER_OCTET_LENGTH nil, // NUMERIC_PRECISION @@ -379,59 +230,42 @@ func (m *MetadataHandler) getTableColumns(dbName, tableName string) ([][]interfa "utf8mb4", // CHARACTER_SET_NAME "utf8mb4_general_ci", // COLLATION_NAME mysqlType, // COLUMN_TYPE - key, // COLUMN_KEY + info.key(), // COLUMN_KEY "", // EXTRA "select,insert,update,references", // PRIVILEGES "", // COLUMN_COMMENT }) } - - return columns, rows.Err() + return columns, nil } -// getAllTablesFromAllDatabases returns tables from all databases -func (m *MetadataHandler) getAllTablesFromAllDatabases() (*protocol.ResultSet, error) { - databases := m.db.ListDatabases() - var allTables [][]interface{} - - for _, dbName := range databases { - if dbName == m.systemDatabaseName { - continue - } - - sqlDB, err := m.db.GetDatabaseConnection(dbName) - if err != nil { - continue - } - - rows, err := sqlDB.Query("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '__marmot__%'") - if err != nil { - continue - } +// columnInfo is one row of SQLite's table_info pragma. +type columnInfo struct { + cid int + name string + declType string + notNull int + dflt sql.NullString + pk int +} - for rows.Next() { - var tableName string - if err := rows.Scan(&tableName); err != nil { - continue - } +func (c columnInfo) nullable() string { + if c.notNull == 1 { + return "NO" + } + return "YES" +} - allTables = append(allTables, []interface{}{ - "def", // TABLE_CATALOG - dbName, // TABLE_SCHEMA - tableName, // TABLE_NAME - "BASE TABLE", // TABLE_TYPE - }) - } - rows.Close() +func (c columnInfo) key() string { + if c.pk > 0 { + return "PRI" } + return "" +} - return &protocol.ResultSet{ - Columns: []protocol.ColumnDef{ - {Name: "TABLE_CATALOG", Type: 0xFD}, - {Name: "TABLE_SCHEMA", Type: 0xFD}, - {Name: "TABLE_NAME", Type: 0xFD}, - {Name: "TABLE_TYPE", Type: 0xFD}, - }, - Rows: allTables, - }, nil +func (c columnInfo) defaultValue() interface{} { + if c.dflt.Valid { + return c.dflt.String + } + return nil } diff --git a/protocol/handlers/information_schema_test.go b/protocol/handlers/information_schema_test.go new file mode 100644 index 00000000..6120ba00 --- /dev/null +++ b/protocol/handlers/information_schema_test.go @@ -0,0 +1,208 @@ +package handlers + +import ( + "database/sql" + "path/filepath" + "testing" + "time" + + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// metadataDeadline bounds every metadata call in these tests. A metadata read +// that waits on a connection never returns, so a missed deadline is the +// failure being tested for, not slowness. +const metadataDeadline = 5 * time.Second + +// onePoolProvider serves one file database whose read pool has exactly one +// connection, the smallest pool a node can be configured with. A metadata +// handler that issues a query while another's rows are open waits forever for +// that connection, which is the F-P1 wedge. +type onePoolProvider struct { + read *sql.DB +} + +func (p onePoolProvider) ListDatabases() []string { return []string{"testdb"} } +func (p onePoolProvider) DatabaseExists(name string) bool { return name == "testdb" } +func (p onePoolProvider) GetDatabaseReadConnection(name string) (*sql.DB, error) { + return p.read, nil +} + +func newOnePoolHandler(t *testing.T) *MetadataHandler { + t.Helper() + path := filepath.Join(t.TempDir(), "testdb.db") + + setup, err := sql.Open("sqlite3", path+"?_journal_mode=WAL") + require.NoError(t, err) + for _, ddl := range []string{ + "CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL, email TEXT DEFAULT 'none')", + "CREATE INDEX idx_users_email ON users(email)", + "CREATE TABLE posts (id INTEGER PRIMARY KEY, body TEXT)", + "CREATE TABLE __marmot__bookkeeping (k TEXT)", + } { + _, err := setup.Exec(ddl) + require.NoError(t, err) + } + require.NoError(t, setup.Close()) + + read, err := sql.Open("sqlite3", path+"?_journal_mode=WAL") + require.NoError(t, err) + read.SetMaxOpenConns(1) + t.Cleanup(func() { _ = read.Close() }) + + return NewMetadataHandler(onePoolProvider{read: read}, "__marmot_system") +} + +// answerWithin runs one metadata call and fails the test if it has not +// answered by metadataDeadline. +func answerWithin(t *testing.T, name string, call func() (*protocol.ResultSet, error)) *protocol.ResultSet { + t.Helper() + type answer struct { + rs *protocol.ResultSet + err error + } + done := make(chan answer, 1) + go func() { + rs, err := call() + done <- answer{rs, err} + }() + select { + case a := <-done: + require.NoError(t, a.err, name) + return a.rs + case <-time.After(metadataDeadline): + t.Fatalf("%s did not answer within %s: a metadata query is waiting on a connection another query holds", name, metadataDeadline) + return nil + } +} + +func isStatement(table protocol.InformationSchemaTableType, schema, tableName string) protocol.Statement { + return protocol.Statement{ + Type: protocol.StatementInformationSchema, + ISTableType: table, + ISFilter: protocol.InformationSchemaFilter{SchemaName: schema, TableName: tableName}, + } +} + +// TestMetadataNeverWaitsOnItself pins the root cause of F-P1: every +// INFORMATION_SCHEMA table and SHOW command answers on a single connection, +// with and without filters, and keeps answering afterwards. Each answer's +// shape is the one the text and prepared paths both send. +// +// Mutations: in informationSchemaColumns, query each table's columns while +// the table list's rows are still open (the old all-tables loop); or in +// HandleShowIndexes, read index_info inside the index_list rows loop. The +// unfiltered COLUMNS row, or the STATISTICS and SHOW INDEX rows, then miss +// their deadline. +func TestMetadataNeverWaitsOnItself(t *testing.T) { + h := newOnePoolHandler(t) + + tests := []struct { + name string + call func() (*protocol.ResultSet, error) + wantCols int + wantRows int + }{ + {"TABLES of the session db", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("testdb", isStatement(protocol.ISTableTables, "", "")) + }, 21, 2}, + {"TABLES filtered by schema and table", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("", isStatement(protocol.ISTableTables, "testdb", "users")) + }, 21, 1}, + {"TABLES with no database: every schema, same columns", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("", isStatement(protocol.ISTableTables, "", "")) + }, 21, 2}, + {"TABLES of an unknown schema is empty", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("testdb", isStatement(protocol.ISTableTables, "nosuch", "")) + }, 21, 0}, + {"TABLES of the system database is empty", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("testdb", isStatement(protocol.ISTableTables, "__marmot_system", "")) + }, 21, 0}, + {"COLUMNS of every table", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("testdb", isStatement(protocol.ISTableColumns, "", "")) + }, 20, 5}, + {"COLUMNS of one table", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("", isStatement(protocol.ISTableColumns, "testdb", "users")) + }, 20, 3}, + {"COLUMNS of an unknown table", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("testdb", isStatement(protocol.ISTableColumns, "", "nosuch")) + }, 20, 0}, + {"SCHEMATA", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("", isStatement(protocol.ISTableSchemata, "", "")) + }, 5, 1}, + {"SCHEMATA filtered", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("", isStatement(protocol.ISTableSchemata, "nosuch", "")) + }, 5, 0}, + {"STATISTICS of one table", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("testdb", isStatement(protocol.ISTableStatistics, "", "users")) + }, 8, 1}, + {"STATISTICS without a table", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("testdb", isStatement(protocol.ISTableStatistics, "", "")) + }, 8, 0}, + {"SHOW TABLES", func() (*protocol.ResultSet, error) { return h.HandleShowTables("testdb", "") }, 1, 2}, + {"SHOW TABLES LIKE", func() (*protocol.ResultSet, error) { return h.HandleShowTables("testdb", "u%") }, 1, 1}, + {"SHOW COLUMNS", func() (*protocol.ResultSet, error) { return h.HandleShowColumns("testdb", "users") }, 6, 3}, + {"SHOW INDEX", func() (*protocol.ResultSet, error) { return h.HandleShowIndexes("testdb", "users") }, 13, 1}, + {"SHOW TABLE STATUS", func() (*protocol.ResultSet, error) { return h.HandleShowTableStatus("testdb", "") }, 18, 2}, + } + + // Twice over: a wedged connection shows as the first answer passing and + // every later one timing out. + for round := 0; round < 2; round++ { + for _, tt := range tests { + rs := answerWithin(t, tt.name, tt.call) + require.Len(t, rs.Columns, tt.wantCols, tt.name) + require.Len(t, rs.Rows, tt.wantRows, tt.name) + for _, row := range rs.Rows { + require.Len(t, row, tt.wantCols, "%s: row width must match its columns", tt.name) + } + } + } +} + +// TestInformationSchemaRowValues pins the values a client reads from the +// rows the tests above count. +func TestInformationSchemaRowValues(t *testing.T) { + h := newOnePoolHandler(t) + + cols := answerWithin(t, "COLUMNS users", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("", isStatement(protocol.ISTableColumns, "testdb", "users")) + }) + require.Equal(t, []interface{}{"def", "testdb", "users", "id", 1, nil, "YES", "int", nil, nil, nil, nil, nil, + "utf8mb4", "utf8mb4_general_ci", "int", "PRI", "", "select,insert,update,references", ""}, cols.Rows[0]) + require.Equal(t, "name", cols.Rows[1][3]) + require.Equal(t, "NO", cols.Rows[1][6]) + require.Equal(t, "'none'", cols.Rows[2][5]) + + idx := answerWithin(t, "STATISTICS users", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("testdb", isStatement(protocol.ISTableStatistics, "", "users")) + }) + require.Equal(t, []interface{}{"def", "testdb", "users", 1, "testdb", "idx_users_email", 1, "email"}, idx.Rows[0]) + + tables := answerWithin(t, "TABLES", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema("testdb", isStatement(protocol.ISTableTables, "", "")) + }) + require.Equal(t, "posts", tables.Rows[0][2]) + require.Equal(t, "users", tables.Rows[1][2]) + require.Equal(t, "BASE TABLE", tables.Rows[1][3]) +} + +// TestTableNamesAreBoundNotSpliced pins that a table name reaches SQLite as a +// bound value, never as SQL text: a filter value is client input, and every +// prepared INFORMATION_SCHEMA query now delivers one. A name SQLite cannot +// parse as SQL answers as the unknown table it is. +// +// Mutation: build "PRAGMA table_info(%s)" with fmt.Sprintf again; the +// statement fails to parse and the call errors. +func TestTableNamesAreBoundNotSpliced(t *testing.T) { + h := newOnePoolHandler(t) + for _, name := range []string{"o'brien", "users); DROP TABLE posts; --"} { + cols := answerWithin(t, "SHOW COLUMNS "+name, func() (*protocol.ResultSet, error) { return h.HandleShowColumns("testdb", name) }) + require.Empty(t, cols.Rows, name) + idx := answerWithin(t, "SHOW INDEX "+name, func() (*protocol.ResultSet, error) { return h.HandleShowIndexes("testdb", name) }) + require.Empty(t, idx.Rows, name) + } + tables := answerWithin(t, "SHOW TABLES", func() (*protocol.ResultSet, error) { return h.HandleShowTables("testdb", "") }) + require.Len(t, tables.Rows, 2) +} diff --git a/protocol/handlers/metadata.go b/protocol/handlers/metadata.go index a5e65bfa..83efb9e2 100644 --- a/protocol/handlers/metadata.go +++ b/protocol/handlers/metadata.go @@ -3,17 +3,25 @@ package handlers import ( "database/sql" "fmt" - "github.com/maxpert/marmot/protocol/query/transform/intmarker" "github.com/maxpert/marmot/protocol" + "github.com/maxpert/marmot/protocol/query/transform/intmarker" "github.com/rs/zerolog/log" ) -// DatabaseProvider abstracts database access for metadata queries +// userTablesQuery selects the names of a database's user tables: Marmot's own +// bookkeeping tables and SQLite's are never shown to a client. +const userTablesQuery = "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '__marmot__%'" + +// DatabaseProvider abstracts database access for metadata queries. +// +// Metadata is read through the database's read pool, never its write handle: +// the write handle has one connection, and a metadata read waiting on it would +// wait behind every write, or behind itself. type DatabaseProvider interface { ListDatabases() []string DatabaseExists(name string) bool - GetDatabaseConnection(name string) (*sql.DB, error) + GetDatabaseReadConnection(name string) (*sql.DB, error) } // MetadataHandler provides shared SHOW command implementations @@ -61,43 +69,34 @@ func (m *MetadataHandler) HandleShowTables(dbName string, likeFilter string) (*p log.Debug().Str("database", dbName).Str("filter", likeFilter).Msg("Handling SHOW TABLES") - sqlDB, err := m.db.GetDatabaseConnection(dbName) + sqlDB, err := m.db.GetDatabaseReadConnection(dbName) if err != nil { return nil, err } - // Build query with optional LIKE filter - query := "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '__marmot__%'" + // MySQL LIKE is case-insensitive by default, as SQLite's LIKE is, and both + // use the same % and _ wildcards. + query := userTablesQuery var args []interface{} if likeFilter != "" { - // SQLite uses GLOB for case-sensitive matching or LIKE for case-insensitive - // MySQL LIKE is case-insensitive by default, so we use SQLite LIKE - // Convert MySQL LIKE pattern (% and _) to be used directly (SQLite supports same syntax) query += " AND name LIKE ?" args = append(args, likeFilter) } query += " ORDER BY name" - rows, err := sqlDB.Query(query, args...) + tables, err := queryStrings(sqlDB, query, args...) if err != nil { return nil, err } - defer rows.Close() result := &protocol.ResultSet{ Columns: []protocol.ColumnDef{{Name: fmt.Sprintf("Tables_in_%s", dbName)}}, - Rows: make([][]interface{}, 0), + Rows: make([][]interface{}, 0, len(tables)), } - - for rows.Next() { - var tableName string - if err := rows.Scan(&tableName); err != nil { - return nil, err - } + for _, tableName := range tables { result.Rows = append(result.Rows, []interface{}{tableName}) } - - return result, rows.Err() + return result, nil } // HandleShowColumns returns column information for a table @@ -115,16 +114,10 @@ func (m *MetadataHandler) HandleShowColumns(dbName, tableName string) (*protocol Str("table", tableName). Msg("Handling SHOW COLUMNS") - sqlDB, err := m.db.GetDatabaseConnection(dbName) - if err != nil { - return nil, err - } - - rows, err := sqlDB.Query(fmt.Sprintf("PRAGMA table_info(%s)", tableName)) + infos, err := m.columnInfos(dbName, tableName) if err != nil { return nil, err } - defer rows.Close() result := &protocol.ResultSet{ Columns: []protocol.ColumnDef{ @@ -135,44 +128,14 @@ func (m *MetadataHandler) HandleShowColumns(dbName, tableName string) (*protocol {Name: "Default", Type: 0xFD}, {Name: "Extra", Type: 0xFD}, }, - Rows: make([][]interface{}, 0), + Rows: make([][]interface{}, 0, len(infos)), } - - for rows.Next() { - var cid int - var name, colType string - var notNull int - var dfltValue sql.NullString - var pk int - - if err := rows.Scan(&cid, &name, &colType, ¬Null, &dfltValue, &pk); err != nil { - return nil, err - } - - // Convert SQLite types to MySQL-compatible format - mysqlType := sqliteToMySQLType(colType) - - nullStr := "YES" - if notNull == 1 { - nullStr = "NO" - } - - keyStr := "" - if pk > 0 { - keyStr = "PRI" - } - - defaultVal := interface{}(nil) - if dfltValue.Valid { - defaultVal = dfltValue.String - } - + for _, info := range infos { result.Rows = append(result.Rows, []interface{}{ - name, mysqlType, nullStr, keyStr, defaultVal, "", + info.name, sqliteToMySQLType(info.declType), info.nullable(), info.key(), info.defaultValue(), "", }) } - - return result, rows.Err() + return result, nil } // HandleShowCreateTable returns the CREATE TABLE statement @@ -190,7 +153,7 @@ func (m *MetadataHandler) HandleShowCreateTable(dbName, tableName string) (*prot Str("table", tableName). Msg("Handling SHOW CREATE TABLE") - sqlDB, err := m.db.GetDatabaseConnection(dbName) + sqlDB, err := m.db.GetDatabaseReadConnection(dbName) if err != nil { return nil, err } @@ -241,16 +204,36 @@ func (m *MetadataHandler) HandleShowIndexes(dbName, tableName string) (*protocol Str("table", tableName). Msg("Handling SHOW INDEXES") - sqlDB, err := m.db.GetDatabaseConnection(dbName) + sqlDB, err := m.db.GetDatabaseReadConnection(dbName) if err != nil { return nil, err } - rows, err := sqlDB.Query(fmt.Sprintf("PRAGMA index_list(%s)", tableName)) + // The index list is read to the end before any index's columns are read, + // so no query runs while another holds a connection. + type indexEntry struct { + name string + unique int + } + var indexes []indexEntry + err = func() error { + rows, err := sqlDB.Query("SELECT name, \"unique\" FROM pragma_index_list(?)", tableName) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var e indexEntry + if err := rows.Scan(&e.name, &e.unique); err != nil { + return err + } + indexes = append(indexes, e) + } + return rows.Err() + }() if err != nil { return nil, err } - defer rows.Close() result := &protocol.ResultSet{ Columns: []protocol.ColumnDef{ @@ -268,44 +251,28 @@ func (m *MetadataHandler) HandleShowIndexes(dbName, tableName string) (*protocol {Name: "Comment", Type: 0xFD}, {Name: "Index_comment", Type: 0xFD}, }, - Rows: make([][]interface{}, 0), + Rows: make([][]interface{}, 0, len(indexes)), } - for rows.Next() { - var seq int - var name string - var unique int - var origin, partial string - - if err := rows.Scan(&seq, &name, &unique, &origin, &partial); err != nil { - continue - } - + for _, idx := range indexes { nonUnique := 1 - if unique == 1 { + if idx.unique == 1 { nonUnique = 0 } - // Get index column info - var columnName string - indexRows, err := sqlDB.Query(fmt.Sprintf("PRAGMA index_info(%s)", name)) - if err == nil { - if indexRows.Next() { - var seqno, cid int - var colName sql.NullString - if err := indexRows.Scan(&seqno, &cid, &colName); err == nil && colName.Valid { - columnName = colName.String - } - } - indexRows.Close() + // An expression index has no column name; it shows an empty one. + var columnName sql.NullString + err := sqlDB.QueryRow("SELECT name FROM pragma_index_info(?) ORDER BY seqno LIMIT 1", idx.name).Scan(&columnName) + if err != nil && err != sql.ErrNoRows { + return nil, err } result.Rows = append(result.Rows, []interface{}{ - tableName, nonUnique, name, 1, columnName, "A", nil, nil, nil, "YES", "BTREE", "", "", + tableName, nonUnique, idx.name, 1, columnName.String, "A", nil, nil, nil, "YES", "BTREE", "", "", }) } - return result, rows.Err() + return result, nil } // HandleShowTableStatus returns table status information @@ -319,13 +286,13 @@ func (m *MetadataHandler) HandleShowTableStatus(dbName, tableName string) (*prot Str("table", tableName). Msg("Handling SHOW TABLE STATUS") - sqlDB, err := m.db.GetDatabaseConnection(dbName) + sqlDB, err := m.db.GetDatabaseReadConnection(dbName) if err != nil { return nil, err } // Build query to get table info - query := "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '__marmot__%'" + query := userTablesQuery var args []interface{} if tableName != "" { query = "SELECT name FROM sqlite_master WHERE type='table' AND name = ?" @@ -392,3 +359,65 @@ func (m *MetadataHandler) HandleShowTableStatus(dbName, tableName string) (*prot return result, rows.Err() } + +// userTables lists a database's user tables by name, or only name when it is +// set. +func (m *MetadataHandler) userTables(dbName, name string) ([]string, error) { + sqlDB, err := m.db.GetDatabaseReadConnection(dbName) + if err != nil { + return nil, err + } + query := userTablesQuery + var args []interface{} + if name != "" { + query += " AND name = ?" + args = append(args, name) + } + return queryStrings(sqlDB, query+" ORDER BY name", args...) +} + +// columnInfos reads a table's columns from SQLite's table_info pragma. The +// table name is bound, never spliced into the statement. +func (m *MetadataHandler) columnInfos(dbName, tableName string) ([]columnInfo, error) { + sqlDB, err := m.db.GetDatabaseReadConnection(dbName) + if err != nil { + return nil, err + } + rows, err := sqlDB.Query(`SELECT cid, name, type, "notnull", dflt_value, pk FROM pragma_table_info(?)`, tableName) + if err != nil { + return nil, err + } + defer rows.Close() + + var infos []columnInfo + for rows.Next() { + var c columnInfo + if err := rows.Scan(&c.cid, &c.name, &c.declType, &c.notNull, &c.dflt, &c.pk); err != nil { + return nil, err + } + infos = append(infos, c) + } + return infos, rows.Err() +} + +// queryStrings runs a query selecting one text column and reads it to the +// end. The rows are closed before it returns, so the connection is free for +// the caller's next query: metadata handlers never issue a query while +// another one's rows are open. +func queryStrings(db *sql.DB, query string, args ...interface{}) ([]string, error) { + rows, err := db.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []string + for rows.Next() { + var s string + if err := rows.Scan(&s); err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} diff --git a/protocol/handlers/metadata_marker_test.go b/protocol/handlers/metadata_marker_test.go index aaf75e53..c05be63d 100644 --- a/protocol/handlers/metadata_marker_test.go +++ b/protocol/handlers/metadata_marker_test.go @@ -13,7 +13,7 @@ type fakeDatabaseProvider struct{ db *sql.DB } func (f fakeDatabaseProvider) ListDatabases() []string { return []string{"testdb"} } func (f fakeDatabaseProvider) DatabaseExists(name string) bool { return name == "testdb" } -func (f fakeDatabaseProvider) GetDatabaseConnection(string) (*sql.DB, error) { +func (f fakeDatabaseProvider) GetDatabaseReadConnection(string) (*sql.DB, error) { return f.db, nil } diff --git a/protocol/handlers/metadata_readpool_test.go b/protocol/handlers/metadata_readpool_test.go new file mode 100644 index 00000000..bc1589d4 --- /dev/null +++ b/protocol/handlers/metadata_readpool_test.go @@ -0,0 +1,67 @@ +package handlers_test + +import ( + "testing" + "time" + + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/maxpert/marmot/protocol/handlers" + "github.com/stretchr/testify/require" +) + +// TestMetadataAnswersWhileTheWriterIsBusy pins the other half of F-P1 on a +// real DatabaseManager: metadata is read through the read pool, so it answers +// while the database's single write connection is taken by an open write +// transaction, as it is during every DDL and every 2PC apply. +// +// Mutation: DatabaseManager.GetDatabaseReadConnection returns GetDB() (the +// write handle); the first call misses its deadline. +func TestMetadataAnswersWhileTheWriterIsBusy(t *testing.T) { + mgr, err := db.NewDatabaseManager(t.TempDir(), 1, hlc.NewClock(1)) + require.NoError(t, err) + t.Cleanup(func() { mgr.Close() }) + + const dbName = "busy" + require.NoError(t, mgr.CreateDatabase(dbName)) + writer, err := mgr.GetDatabaseConnection(dbName) + require.NoError(t, err) + _, err = writer.Exec("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)") + require.NoError(t, err) + + // Take the write handle's only connection and keep it. + tx, err := writer.Begin() + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + _, err = tx.Exec("INSERT INTO t (v) VALUES ('held')") + require.NoError(t, err) + + h := handlers.NewMetadataHandler(mgr, db.SystemDatabaseName) + calls := []struct { + name string + call func() (*protocol.ResultSet, error) + }{ + {"TABLES", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema(dbName, protocol.Statement{ISTableType: protocol.ISTableTables}) + }}, + {"COLUMNS", func() (*protocol.ResultSet, error) { + return h.HandleInformationSchema(dbName, protocol.Statement{ISTableType: protocol.ISTableColumns}) + }}, + {"SHOW INDEX", func() (*protocol.ResultSet, error) { return h.HandleShowIndexes(dbName, "t") }}, + {"SHOW CREATE TABLE", func() (*protocol.ResultSet, error) { return h.HandleShowCreateTable(dbName, "t") }}, + } + for _, c := range calls { + done := make(chan error, 1) + go func() { + _, err := c.call() + done <- err + }() + select { + case err := <-done: + require.NoError(t, err, c.name) + case <-time.After(5 * time.Second): + t.Fatalf("%s did not answer while the write connection was held: metadata is waiting on the writer", c.name) + } + } +} diff --git a/protocol/information_schema.go b/protocol/information_schema.go new file mode 100644 index 00000000..fd68cac5 --- /dev/null +++ b/protocol/information_schema.go @@ -0,0 +1,98 @@ +package protocol + +import "slices" + +// Column types used by the INFORMATION_SCHEMA column sets. +const ( + isTypeString byte = 0xFD // MYSQL_TYPE_VAR_STRING + isTypeLong byte = 0x03 // MYSQL_TYPE_LONG + isTypeLongLong byte = 0x08 // MYSQL_TYPE_LONGLONG + isTypeDatetime byte = 0x0C // MYSQL_TYPE_DATETIME +) + +var ( + isTablesColumns = []ColumnDef{ + {Name: "TABLE_CATALOG", Type: isTypeString}, + {Name: "TABLE_SCHEMA", Type: isTypeString}, + {Name: "TABLE_NAME", Type: isTypeString}, + {Name: "TABLE_TYPE", Type: isTypeString}, + {Name: "ENGINE", Type: isTypeString}, + {Name: "VERSION", Type: isTypeLong}, + {Name: "ROW_FORMAT", Type: isTypeString}, + {Name: "TABLE_ROWS", Type: isTypeLongLong}, + {Name: "AVG_ROW_LENGTH", Type: isTypeLongLong}, + {Name: "DATA_LENGTH", Type: isTypeLongLong}, + {Name: "MAX_DATA_LENGTH", Type: isTypeLongLong}, + {Name: "INDEX_LENGTH", Type: isTypeLongLong}, + {Name: "DATA_FREE", Type: isTypeLongLong}, + {Name: "AUTO_INCREMENT", Type: isTypeLongLong}, + {Name: "CREATE_TIME", Type: isTypeDatetime}, + {Name: "UPDATE_TIME", Type: isTypeDatetime}, + {Name: "CHECK_TIME", Type: isTypeDatetime}, + {Name: "TABLE_COLLATION", Type: isTypeString}, + {Name: "CHECKSUM", Type: isTypeLongLong}, + {Name: "CREATE_OPTIONS", Type: isTypeString}, + {Name: "TABLE_COMMENT", Type: isTypeString}, + } + + isColumnsColumns = []ColumnDef{ + {Name: "TABLE_CATALOG", Type: isTypeString}, + {Name: "TABLE_SCHEMA", Type: isTypeString}, + {Name: "TABLE_NAME", Type: isTypeString}, + {Name: "COLUMN_NAME", Type: isTypeString}, + {Name: "ORDINAL_POSITION", Type: isTypeLong}, + {Name: "COLUMN_DEFAULT", Type: isTypeString}, + {Name: "IS_NULLABLE", Type: isTypeString}, + {Name: "DATA_TYPE", Type: isTypeString}, + {Name: "CHARACTER_MAXIMUM_LENGTH", Type: isTypeLongLong}, + {Name: "CHARACTER_OCTET_LENGTH", Type: isTypeLongLong}, + {Name: "NUMERIC_PRECISION", Type: isTypeLongLong}, + {Name: "NUMERIC_SCALE", Type: isTypeLongLong}, + {Name: "DATETIME_PRECISION", Type: isTypeLongLong}, + {Name: "CHARACTER_SET_NAME", Type: isTypeString}, + {Name: "COLLATION_NAME", Type: isTypeString}, + {Name: "COLUMN_TYPE", Type: isTypeString}, + {Name: "COLUMN_KEY", Type: isTypeString}, + {Name: "EXTRA", Type: isTypeString}, + {Name: "PRIVILEGES", Type: isTypeString}, + {Name: "COLUMN_COMMENT", Type: isTypeString}, + } + + isSchemataColumns = []ColumnDef{ + {Name: "CATALOG_NAME", Type: isTypeString}, + {Name: "SCHEMA_NAME", Type: isTypeString}, + {Name: "DEFAULT_CHARACTER_SET_NAME", Type: isTypeString}, + {Name: "DEFAULT_COLLATION_NAME", Type: isTypeString}, + {Name: "SQL_PATH", Type: isTypeString}, + } + + isStatisticsColumns = []ColumnDef{ + {Name: "TABLE_CATALOG", Type: isTypeString}, + {Name: "TABLE_SCHEMA", Type: isTypeString}, + {Name: "TABLE_NAME", Type: isTypeString}, + {Name: "NON_UNIQUE", Type: isTypeLong}, + {Name: "INDEX_SCHEMA", Type: isTypeString}, + {Name: "INDEX_NAME", Type: isTypeString}, + {Name: "SEQ_IN_INDEX", Type: isTypeLong}, + {Name: "COLUMN_NAME", Type: isTypeString}, + } +) + +// InformationSchemaColumns returns the columns Marmot answers an +// INFORMATION_SCHEMA table with, or nil for a table it does not serve. The +// text query, the prepared statement's execution and its PREPARE description +// all use this one set, so the three can never disagree. +func InformationSchemaColumns(table InformationSchemaTableType) []ColumnDef { + switch table { + case ISTableTables: + return slices.Clone(isTablesColumns) + case ISTableColumns: + return slices.Clone(isColumnsColumns) + case ISTableSchemata: + return slices.Clone(isSchemataColumns) + case ISTableStatistics: + return slices.Clone(isStatisticsColumns) + default: + return nil + } +} diff --git a/protocol/information_schema_test.go b/protocol/information_schema_test.go new file mode 100644 index 00000000..70701b3e --- /dev/null +++ b/protocol/information_schema_test.go @@ -0,0 +1,191 @@ +package protocol + +import ( + "encoding/binary" + "io" + "net" + "testing" + + "github.com/stretchr/testify/require" +) + +// isAnswerHandler answers every query with a fixed INFORMATION_SCHEMA row set, +// as the metadata handler does, and records the params it was handed. +type isAnswerHandler struct { + params []interface{} +} + +func (h *isAnswerHandler) HandleQuery(_ *ConnectionSession, _ string, params []interface{}) (*ResultSet, error) { + h.params = params + return &ResultSet{ + Columns: InformationSchemaColumns(ISTableSchemata), + Rows: [][]interface{}{{"def", "marmot", "utf8mb4", "utf8mb4_general_ci", nil}}, + }, nil +} + +// readAllPackets runs serve against one end of a pipe and returns every +// packet it wrote, in order. +func readAllPackets(t *testing.T, serve func(conn net.Conn)) [][]byte { + t.Helper() + clientConn, serverConn := net.Pipe() + defer clientConn.Close() + go func() { + serve(serverConn) + _ = serverConn.Close() + }() + + var packets [][]byte + for { + header := make([]byte, 4) + if _, err := io.ReadFull(clientConn, header); err != nil { + return packets + } + size := int(uint32(header[0]) | uint32(header[1])<<8 | uint32(header[2])<<16) + payload := make([]byte, size) + _, err := io.ReadFull(clientConn, payload) + require.NoError(t, err) + packets = append(packets, payload) + } +} + +// TestParseResolvesBoundInformationSchemaFilters pins F-P1's trigger: a bound +// table_schema/table_name must filter exactly as the literal does. Without it +// a prepared COLUMNS query takes the every-table branch and answers rows of +// tables the client never asked about. +// +// Mutation: filterValues.value ignores *sqlparser.Argument; the first row +// fails. +func TestParseResolvesBoundInformationSchemaFilters(t *testing.T) { + tests := []struct { + name string + sql string + bound []interface{} + wantType InformationSchemaTableType + wantSchema string + wantTable string + }{ + { + name: "both bound", + sql: "SELECT column_name FROM information_schema.columns WHERE table_schema = ? AND table_name = ?", + bound: []interface{}{"marmot", "pn"}, + wantType: ISTableColumns, + wantSchema: "marmot", + wantTable: "pn", + }, + { + name: "bound as bytes", + sql: "SELECT table_name FROM information_schema.tables WHERE table_schema = ?", + bound: []interface{}{[]byte("p2")}, + wantType: ISTableTables, + wantSchema: "p2", + }, + { + name: "literal and bound mixed", + sql: "SELECT * FROM information_schema.columns WHERE table_schema = 'marmot' AND table_name = ?", + bound: []interface{}{"pk"}, + wantType: ISTableColumns, + wantSchema: "marmot", + wantTable: "pk", + }, + { + name: "placeholder before the filter", + sql: "SELECT ?, table_name FROM information_schema.tables WHERE table_schema = ?", + bound: []interface{}{int64(1), "p2"}, + wantType: ISTableTables, + wantSchema: "p2", + }, + { + name: "schemata", + sql: "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?", + bound: []interface{}{"p2"}, + wantType: ISTableSchemata, + wantSchema: "p2", + }, + { + name: "non-text bound value filters nothing", + sql: "SELECT * FROM information_schema.tables WHERE table_schema = ?", + bound: []interface{}{int64(7)}, + wantType: ISTableTables, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stmt := ParseStatementWithOptions(tt.sql, ParseOptions{BoundParams: tt.bound}) + require.Equal(t, StatementInformationSchema, stmt.Type) + require.Equal(t, tt.wantType, stmt.ISTableType) + require.Equal(t, tt.wantSchema, stmt.ISFilter.SchemaName, "schema filter") + require.Equal(t, tt.wantTable, stmt.ISFilter.TableName, "table filter") + }) + } +} + +// TestPreparedInformationSchemaAnswersWithItsRows pins F-P2: a prepared +// INFORMATION_SCHEMA statement answers with the handler's result set, exactly +// as its text form does, never with an OK packet. +// +// Mutation: answersRows := stmt.OriginalType == StatementSelect; the column +// count assertion fires on an OK packet (0x00). +func TestPreparedInformationSchemaAnswersWithItsRows(t *testing.T) { + handler := &isAnswerHandler{} + server := NewMySQLServer("127.0.0.1:0", "", 0, handler) + + stmt := &PreparedStatement{ + ID: 9, + Query: "SELECT * FROM information_schema.schemata WHERE schema_name = ?", + ParamCount: 1, + OriginalType: StatementInformationSchema, + } + session := &ConnectionSession{ + ConnID: 1, + preparedStmts: map[uint32]*PreparedStatement{stmt.ID: stmt}, + } + + value := "marmot" + execPayload := make([]byte, 14+len(value)) + binary.LittleEndian.PutUint32(execPayload[0:4], stmt.ID) // statement_id + execPayload[4] = 0 // flags + binary.LittleEndian.PutUint32(execPayload[5:9], 1) // iteration_count + execPayload[9] = 0x00 // NULL bitmap + execPayload[10] = 0x01 // new_params_bound_flag + execPayload[11] = 0xFD // MYSQL_TYPE_VAR_STRING + execPayload[12] = 0x00 // param flags + execPayload[13] = byte(len(value)) // lenenc length + copy(execPayload[14:], value) + + packets := readAllPackets(t, func(conn net.Conn) { server.handleStmtExecute(conn, session, execPayload) }) + require.NotEmpty(t, packets) + require.Equal(t, byte(len(isSchemataColumns)), packets[0][0], + "a prepared INFORMATION_SCHEMA query must answer with its column count, not an OK packet") + // Column count, 5 definitions, EOF, one row, EOF. + require.Len(t, packets, 1+len(isSchemataColumns)+1+1+1) + require.Equal(t, []interface{}{"marmot"}, handler.params) +} + +// TestPrepareDescribesInformationSchemaColumns pins that COM_STMT_PREPARE of +// an INFORMATION_SCHEMA statement reports the columns its execution returns: +// clients index result columns by name from this response. +// +// Mutation: drop the StatementInformationSchema branch in handleStmtPrepare; +// the column count is 0. +func TestPrepareDescribesInformationSchemaColumns(t *testing.T) { + server := NewMySQLServer("127.0.0.1:0", "", 0, &isAnswerHandler{}) + session := &ConnectionSession{ConnID: 1, preparedStmts: map[uint32]*PreparedStatement{}} + + packets := readAllPackets(t, func(conn net.Conn) { + server.handleStmtPrepare(conn, session, "SELECT * FROM information_schema.tables WHERE table_schema = ? AND table_name = ?") + }) + require.NotEmpty(t, packets) + ok := packets[0] + require.Equal(t, byte(0x00), ok[0], "PREPARE_OK") + require.Equal(t, uint16(len(isTablesColumns)), binary.LittleEndian.Uint16(ok[5:7]), "column count") + require.Equal(t, uint16(2), binary.LittleEndian.Uint16(ok[7:9]), "param count") +} + +// TestInformationSchemaColumnsAreCopies pins that a caller cannot corrupt the +// shared column sets. +func TestInformationSchemaColumnsAreCopies(t *testing.T) { + cols := InformationSchemaColumns(ISTableTables) + cols[0].Name = "mutated" + require.Equal(t, "TABLE_CATALOG", InformationSchemaColumns(ISTableTables)[0].Name) + require.Nil(t, InformationSchemaColumns(ISTableUnknown)) +} diff --git a/protocol/load_data_local.go b/protocol/load_data_local.go index 8800698e..7892be9d 100644 --- a/protocol/load_data_local.go +++ b/protocol/load_data_local.go @@ -13,10 +13,13 @@ const maxLoadDataLocalBytes = 64 << 20 // 64 MiB safety cap for phase-1 LOCAL IN const capabilityClientLocalFiles uint32 = 1 << 7 const loadDataInsertBatchSize = 500 +// loadDataIdent matches one identifier, backquoted or bare. +const loadDataIdent = "`(?:[^`]|``)+`|[A-Za-z0-9_$]+" + var ( loadDataAnyPattern = regexp.MustCompile(`(?is)^\s*LOAD\s+DATA\s+`) loadDataLocalPattern = regexp.MustCompile(`(?is)^\s*LOAD\s+DATA\s+(?:LOW_PRIORITY|CONCURRENT\s+)?LOCAL\s+INFILE\s+`) - loadDataTablePattern = regexp.MustCompile(`(?is)\bINTO\s+TABLE\s+((?:` + "`" + `[^` + "`" + `]+` + "`" + `)|(?:[A-Za-z0-9_\.]+))`) + loadDataTablePattern = regexp.MustCompile(`(?is)\bINTO\s+TABLE\s+(?:(` + loadDataIdent + `)\s*\.\s*)?(` + loadDataIdent + `)`) loadDataLocalPathPattern = regexp.MustCompile(`(?is)\bLOCAL\s+INFILE\s+'((?:\\.|[^'])*)'`) loadDataFieldsTerminated = regexp.MustCompile(`(?is)\bFIELDS\s+TERMINATED\s+BY\s+'((?:\\.|[^'])*)'`) loadDataLinesTerminated = regexp.MustCompile(`(?is)\bLINES\s+TERMINATED\s+BY\s+'((?:\\.|[^'])*)'`) @@ -26,6 +29,9 @@ var ( ) type loadDataLocalSpec struct { + // Database is the target's qualifier, "" when the statement names none; + // Database and Table are unquoted. + Database string Table string Columns []string FieldTerminator string @@ -50,12 +56,13 @@ func parseLoadDataLocalSpec(sql string) (*loadDataLocalSpec, error) { } matches := loadDataTablePattern.FindStringSubmatch(sql) - if len(matches) < 2 { + if len(matches) < 3 { return nil, fmt.Errorf("LOAD DATA LOCAL missing INTO TABLE target") } spec := &loadDataLocalSpec{ - Table: strings.TrimSpace(matches[1]), + Database: unquoteLoadDataIdent(matches[1]), + Table: unquoteLoadDataIdent(matches[2]), FieldTerminator: "\t", LineTerminator: "\n", } @@ -102,6 +109,32 @@ func parseLoadDataLocalSpec(sql string) (*loadDataLocalSpec, error) { return spec, nil } +// unquoteLoadDataIdent removes an identifier's backquotes, undoing a doubled +// backquote inside it. +func unquoteLoadDataIdent(s string) string { + if len(s) >= 2 && s[0] == '`' && s[len(s)-1] == '`' { + return strings.ReplaceAll(s[1:len(s)-1], "``", "`") + } + return s +} + +// quoteLoadDataIdent backquotes an identifier. +func quoteLoadDataIdent(s string) string { + return "`" + strings.ReplaceAll(s, "`", "``") + "`" +} + +// target renders the table the INSERTs name. It is qualified only when the +// statement names a database other than the session's: a session with no +// current database is one applying a replicated LOAD DATA on the target +// database's own connection (db.ApplyLoadData), where a qualifier would name +// an attached schema instead. +func (s *loadDataLocalSpec) target(session *ConnectionSession) string { + if s.Database == "" || session.CurrentDatabase == "" || s.Database == session.CurrentDatabase { + return quoteLoadDataIdent(s.Table) + } + return quoteLoadDataIdent(s.Database) + "." + quoteLoadDataIdent(s.Table) +} + func unescapeLoadDataToken(s string) (string, error) { if !strings.Contains(s, `\`) { return s, nil @@ -114,7 +147,7 @@ func unescapeLoadDataToken(s string) (string, error) { return v, nil } -func buildInsertForLoadData(spec *loadDataLocalSpec, fieldCount, rowCount int) string { +func buildInsertForLoadData(spec *loadDataLocalSpec, target string, fieldCount, rowCount int) string { rowPlaceholder := make([]string, fieldCount) for i := range rowPlaceholder { rowPlaceholder[i] = "?" @@ -128,9 +161,9 @@ func buildInsertForLoadData(spec *loadDataLocalSpec, fieldCount, rowCount int) s cols = "(" + strings.Join(spec.Columns, ", ") + ")" } if cols != "" { - return fmt.Sprintf("INSERT INTO %s %s VALUES %s", spec.Table, cols, strings.Join(rows, ", ")) + return fmt.Sprintf("INSERT INTO %s %s VALUES %s", target, cols, strings.Join(rows, ", ")) } - return fmt.Sprintf("INSERT INTO %s VALUES %s", spec.Table, strings.Join(rows, ", ")) + return fmt.Sprintf("INSERT INTO %s VALUES %s", target, strings.Join(rows, ", ")) } func splitLoadDataRows(data []byte, spec *loadDataLocalSpec) [][]string { @@ -212,6 +245,16 @@ func (s *MySQLServer) processLoadDataQuery(conn io.ReadWriter, session *Connecti return s.writeOK(conn, lastSeq+1, session, rows, lastInsertID) } +// LoadDataTarget returns the database qualifier ("" when the statement names +// none) and the table a LOAD DATA LOCAL statement loads into, unquoted. +func LoadDataTarget(sql string) (database, table string, err error) { + spec, err := parseLoadDataLocalSpec(sql) + if err != nil { + return "", "", err + } + return spec.Database, spec.Table, nil +} + // ExecuteLoadDataLocal applies LOAD DATA LOCAL payload by translating to parameterized INSERT batches. func ExecuteLoadDataLocal(session *ConnectionSession, handler ConnectionHandler, query string, data []byte) (*ResultSet, error) { spec, err := parseLoadDataLocalSpec(query) @@ -267,7 +310,7 @@ func executeLocalLoadDataRowsWithHandler(session *ConnectionSession, handler Con } } - stmt := buildInsertForLoadData(spec, fieldCount, batchEnd-i) + stmt := buildInsertForLoadData(spec, spec.target(session), fieldCount, batchEnd-i) params := make([]interface{}, 0, (batchEnd-i)*fieldCount) for _, row := range rows[i:batchEnd] { for _, col := range row { diff --git a/protocol/load_data_local_test.go b/protocol/load_data_local_test.go index 5e61cf72..74ea7795 100644 --- a/protocol/load_data_local_test.go +++ b/protocol/load_data_local_test.go @@ -96,7 +96,7 @@ func TestProcessQueryLoadDataLocal(t *testing.T) { require.Equal(t, []string{ "BEGIN", - "INSERT INTO users (name, email) VALUES (?, ?), (?, ?)", + "INSERT INTO `users` (name, email) VALUES (?, ?), (?, ?)", "COMMIT", }, handler.queries) require.Len(t, handler.params, 1) @@ -307,3 +307,30 @@ func TestHandshakeOmitsLocalInfileCapabilityWhenDisabled(t *testing.T) { capsLower := binary.LittleEndian.Uint16(payload[offset : offset+2]) require.Zero(t, capsLower&0x0080, "CLIENT_LOCAL_FILES should not be advertised") } + +// TestLoadDataLocalSpecQualifiedTarget is L3: the target's database +// qualifier is parsed, bare or backquoted, and the INSERTs name it only when +// it is not the session's database - never on a session with no current +// database, which is a replicated apply on the target's own connection. +// +// Mutation: go back to one capture for the whole target. "`hq`.`t`" parses +// as table "hq" and the first assertion fires. +func TestLoadDataLocalSpecQualifiedTarget(t *testing.T) { + for _, tc := range []struct{ sql, database, table string }{ + {"LOAD DATA LOCAL INFILE 'f' INTO TABLE `hq`.`t` (v)", "hq", "t"}, + {"LOAD DATA LOCAL INFILE 'f' INTO TABLE hq.t (v)", "hq", "t"}, + {"LOAD DATA LOCAL INFILE 'f' INTO TABLE `h``q`.t (v)", "h`q", "t"}, + {"LOAD DATA LOCAL INFILE 'f' INTO TABLE `t` (v)", "", "t"}, + } { + database, table, err := LoadDataTarget(tc.sql) + require.NoError(t, err, tc.sql) + require.Equal(t, tc.database, database, tc.sql) + require.Equal(t, tc.table, table, tc.sql) + } + + spec, err := parseLoadDataLocalSpec("LOAD DATA LOCAL INFILE 'f' INTO TABLE hq.t (v)") + require.NoError(t, err) + require.Equal(t, "`hq`.`t`", spec.target(&ConnectionSession{CurrentDatabase: "marmot"})) + require.Equal(t, "`t`", spec.target(&ConnectionSession{CurrentDatabase: "hq"})) + require.Equal(t, "`t`", spec.target(&ConnectionSession{}), "a replicated apply must not name an attached schema") +} diff --git a/protocol/merge_exec_params_test.go b/protocol/merge_exec_params_test.go index f32c15eb..ba415c0b 100644 --- a/protocol/merge_exec_params_test.go +++ b/protocol/merge_exec_params_test.go @@ -89,3 +89,24 @@ func TestStatement_WithResolvedParams_LeavesOriginalUntouched(t *testing.T) { require.Equal(t, []interface{}{"active"}, original.ExtractedParams) require.Equal(t, []bool{true, false}, original.ParamOrder) } + +// TestMergeExecParams_BoundIDsReplaceTheCallersValues: a narrow +// AUTO_INCREMENT placeholder the caller bound to NULL executes with the id +// the server generated for it, at its own position, interleaved with a +// pipeline-extracted literal; the caller's slice is not modified. +// +// Mutation: skip the BoundIDs loop. NULL reaches SQLite and the first +// assertion fires. +func TestMergeExecParams_BoundIDsReplaceTheCallersValues(t *testing.T) { + stmt := Statement{ + SQL: "INSERT INTO g (group_id, name, kind) VALUES (?, ?, ?)", + ExtractedParams: []interface{}{"k"}, + ParamOrder: []bool{true, true, false}, + BoundIDs: map[int]uint64{0: 42}, + } + wire := []interface{}{nil, "a"} + require.Equal(t, []interface{}{int64(42), "a", "k"}, stmt.MergeExecParams(wire)) + require.Nil(t, wire[0], "MergeExecParams modified the caller's values") + + require.Nil(t, stmt.WithResolvedParams("SELECT 1", nil).BoundIDs, "resolved params kept stale bound ids") +} diff --git a/protocol/parser.go b/protocol/parser.go index 5768da82..11aa5d17 100644 --- a/protocol/parser.go +++ b/protocol/parser.go @@ -6,6 +6,7 @@ import ( "github.com/maxpert/marmot/id" "github.com/maxpert/marmot/protocol/query" + "github.com/maxpert/marmot/protocol/query/rules" "github.com/maxpert/marmot/protocol/query/transform" "github.com/rs/zerolog/log" ) @@ -76,7 +77,7 @@ func InitializePipeline(cacheSize int, idGen id.Generator) error { // SchemaLookupFunc returns the schema facts an INSERT needs for auto-increment // id injection, or nil if the table is unknown. -type SchemaLookupFunc func(table string) *transform.SchemaInfo +type SchemaLookupFunc func(database, table string) *transform.SchemaInfo // ParseOptions holds options for parsing SQL statements. type ParseOptions struct { @@ -84,6 +85,14 @@ type ParseOptions struct { SchemaProvider transform.SchemaProvider // For ON CONFLICT target resolution SkipTranspilation bool ExtractLiterals bool // Enable literal extraction for parameterized execution + + // NarrowIDs mints ids for AUTO_INCREMENT columns declared narrower than + // BIGINT. Nil refuses any INSERT that needs one. + NarrowIDs rules.NarrowAllocator + + // BoundParams are the statement's bound values, nil for a text query + // (query.QueryContext.BoundParams). + BoundParams []interface{} } // ParseStatement analyzes a SQL statement and returns its type and metadata. @@ -97,6 +106,8 @@ func ParseStatement(sql string) Statement { func ParseStatementWithOptions(sql string, opts ParseOptions) Statement { ctx := query.NewContext(sql, nil) ctx.SchemaLookup = opts.SchemaLookup + ctx.NarrowIDs = opts.NarrowIDs + ctx.BoundParams = opts.BoundParams ctx.SchemaProvider = opts.SchemaProvider ctx.SkipTranspilation = opts.SkipTranspilation ctx.ExtractLiterals = opts.ExtractLiterals @@ -119,12 +130,14 @@ func ParseStatementWithOptions(sql string, opts ParseOptions) Statement { transpiledSQL := "" var extractedParams []interface{} var paramOrder []bool + var boundIDs map[int]uint64 if len(ctx.Output.Statements) > 0 { transpiledSQL = ctx.Output.Statements[0].SQL if len(ctx.Output.Statements[0].Params) > 0 { extractedParams = ctx.Output.Statements[0].Params } paramOrder = ctx.Output.Statements[0].ParamOrder + boundIDs = ctx.Output.Statements[0].BoundIDs } stmt := Statement{ @@ -134,6 +147,7 @@ func ParseStatementWithOptions(sql string, opts ParseOptions) Statement { Error: errorString(ctx.Output.ValidationErr), ExtractedParams: extractedParams, ParamOrder: paramOrder, + BoundIDs: boundIDs, } // Extract MySQL-specific metadata (if available) @@ -169,12 +183,14 @@ func ParseStatementWithSchema(sql string, schemaLookup SchemaLookupFunc) Stateme transpiledSQL := "" var extractedParams []interface{} var paramOrder []bool + var boundIDs map[int]uint64 if len(ctx.Output.Statements) > 0 { transpiledSQL = ctx.Output.Statements[0].SQL if len(ctx.Output.Statements[0].Params) > 0 { extractedParams = ctx.Output.Statements[0].Params } paramOrder = ctx.Output.Statements[0].ParamOrder + boundIDs = ctx.Output.Statements[0].BoundIDs } stmt := Statement{ @@ -184,6 +200,7 @@ func ParseStatementWithSchema(sql string, schemaLookup SchemaLookupFunc) Stateme Error: errorString(ctx.Output.ValidationErr), ExtractedParams: extractedParams, ParamOrder: paramOrder, + BoundIDs: boundIDs, } // Extract MySQL-specific metadata (if available) @@ -237,6 +254,7 @@ func buildStatement(ctx query.QueryContext, ts query.TranspiledStatement) Statem Error: errorString(ctx.Output.ValidationErr), ExtractedParams: extractedParams, ParamOrder: ts.ParamOrder, + BoundIDs: ts.BoundIDs, } // Extract MySQL-specific metadata (if available) diff --git a/protocol/query/context.go b/protocol/query/context.go index 63897fbb..487a0bec 100644 --- a/protocol/query/context.go +++ b/protocol/query/context.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/maxpert/marmot/common" + "github.com/maxpert/marmot/protocol/query/rules" "github.com/maxpert/marmot/protocol/query/transform" "vitess.io/vitess/go/vt/sqlparser" ) @@ -124,6 +125,10 @@ type TranspiledStatement struct { // server-injected auto-increment id) - see transform.ExtractLiterals and // protocol.MergeExecParams. nil means Params is the only source needed. ParamOrder []bool + // BoundIDs are server-generated ids that replace the caller's bound + // values, keyed by the value's position: a narrow AUTO_INCREMENT + // placeholder bound to NULL or 0 (rules.AutoIncrementIDRule.ApplyAST). + BoundIDs map[int]uint64 } // MySQLParseState holds MySQL-specific parsing state and metadata. @@ -166,9 +171,20 @@ type QueryContext struct { Input QueryInput Output QueryOutput MySQLState *MySQLParseState // nil for SQLite dialect - SchemaLookup func(table string) *transform.SchemaInfo + SchemaLookup func(database, table string) *transform.SchemaInfo SchemaProvider transform.SchemaProvider + // NarrowIDs mints ids for AUTO_INCREMENT columns declared narrower than + // BIGINT. It belongs to the node's write coordinator, which claims the + // ranges, so it travels with the query rather than living in the shared + // pipeline. Nil refuses any INSERT that needs one. + NarrowIDs rules.NarrowAllocator + + // BoundParams are a prepared statement's bound values, nil for a text + // query. Only AUTO_INCREMENT id injection reads them, to generate the id + // of a placeholder bound to NULL or 0. + BoundParams []interface{} + // SkipTranspilation bypasses MySQL→SQLite transpilation when true. // SQL is passed through unchanged, only statement type classification is performed. SkipTranspilation bool diff --git a/protocol/query/parser.go b/protocol/query/parser.go index 11b6ef9b..c82f999e 100644 --- a/protocol/query/parser.go +++ b/protocol/query/parser.go @@ -293,7 +293,7 @@ func classifyStatement(ctx *QueryContext, stmt sqlparser.Statement) { // Check for INFORMATION_SCHEMA queries ctx.Output.StatementType = StatementInformationSchema ctx.MySQLState.ISTableType = isTableType - ctx.MySQLState.ISFilter = extractInformationSchemaFilter(parsed) + ctx.MySQLState.ISFilter = extractInformationSchemaFilter(parsed, newFilterValues(stmt, ctx.BoundParams)) } case *sqlparser.CreateTable: @@ -379,7 +379,7 @@ func classifyStatement(ctx *QueryContext, stmt sqlparser.Statement) { } else if isTableType := detectInformationSchemaTable(leftSelect); isTableType != ISTableUnknown { ctx.Output.StatementType = StatementInformationSchema ctx.MySQLState.ISTableType = isTableType - ctx.MySQLState.ISFilter = extractInformationSchemaFilter(leftSelect) + ctx.MySQLState.ISFilter = extractInformationSchemaFilter(leftSelect, newFilterValues(stmt, ctx.BoundParams)) } } @@ -543,46 +543,89 @@ func detectInformationSchemaTable(sel *sqlparser.Select) InformationSchemaTableT return ISTableUnknown } +// filterValues resolves the value an INFORMATION_SCHEMA filter compares a +// column with: a literal, or a prepared statement's bound value, found by its +// placeholder's position in the whole statement. +type filterValues struct { + bound []interface{} + args map[*sqlparser.Argument]int +} + +// newFilterValues numbers stmt's placeholders only when values are bound; a +// text query has none to resolve. +func newFilterValues(stmt sqlparser.Statement, bound []interface{}) filterValues { + if len(bound) == 0 { + return filterValues{} + } + return filterValues{bound: bound, args: transform.ArgumentPositions(stmt)} +} + +// value returns expr's string value, or "" when it is neither a string +// literal nor a placeholder bound to text. +func (v filterValues) value(expr sqlparser.Expr) string { + switch e := expr.(type) { + case *sqlparser.Literal: + // Remove surrounding quotes + val := e.Val + if len(val) >= 2 && (val[0] == '\'' || val[0] == '"') { + return val[1 : len(val)-1] + } + return val + case *sqlparser.Argument: + pos, ok := v.args[e] + if !ok || pos >= len(v.bound) { + return "" + } + switch b := v.bound[pos].(type) { + case string: + return b + case []byte: + return string(b) + } + } + return "" +} + // extractInformationSchemaFilter extracts WHERE clause filter values from INFORMATION_SCHEMA queries -func extractInformationSchemaFilter(sel *sqlparser.Select) InformationSchemaFilter { +func extractInformationSchemaFilter(sel *sqlparser.Select, values filterValues) InformationSchemaFilter { filter := InformationSchemaFilter{} if sel.Where == nil { return filter } - extractFiltersFromExpr(sel.Where.Expr, &filter) + extractFiltersFromExpr(sel.Where.Expr, values, &filter) return filter } // extractFiltersFromExpr recursively walks WHERE expression to find equality comparisons -func extractFiltersFromExpr(expr sqlparser.Expr, filter *InformationSchemaFilter) { +func extractFiltersFromExpr(expr sqlparser.Expr, values filterValues, filter *InformationSchemaFilter) { switch e := expr.(type) { case *sqlparser.AndExpr: - extractFiltersFromExpr(e.Left, filter) - extractFiltersFromExpr(e.Right, filter) + extractFiltersFromExpr(e.Left, values, filter) + extractFiltersFromExpr(e.Right, values, filter) case *sqlparser.OrExpr: // For OR expressions, we can't reliably extract filters // but we still walk both sides in case there's an AND somewhere - extractFiltersFromExpr(e.Left, filter) - extractFiltersFromExpr(e.Right, filter) + extractFiltersFromExpr(e.Left, values, filter) + extractFiltersFromExpr(e.Right, values, filter) case *sqlparser.ComparisonExpr: if e.Operator == sqlparser.EqualOp { - extractEqualityFilter(e, filter) + extractEqualityFilter(e, values, filter) } } } -// extractEqualityFilter extracts column = 'value' patterns -func extractEqualityFilter(cmp *sqlparser.ComparisonExpr, filter *InformationSchemaFilter) { +// extractEqualityFilter extracts column = 'value' and column = ? patterns +func extractEqualityFilter(cmp *sqlparser.ComparisonExpr, values filterValues, filter *InformationSchemaFilter) { // Get column name (could be on left or right side) var colName string var value string if col, ok := cmp.Left.(*sqlparser.ColName); ok { colName = strings.ToUpper(col.Name.String()) - value = extractStringValue(cmp.Right) + value = values.value(cmp.Right) } else if col, ok := cmp.Right.(*sqlparser.ColName); ok { colName = strings.ToUpper(col.Name.String()) - value = extractStringValue(cmp.Left) + value = values.value(cmp.Left) } if value == "" { @@ -599,20 +642,6 @@ func extractEqualityFilter(cmp *sqlparser.ComparisonExpr, filter *InformationSch } } -// extractStringValue extracts string literal value from expression -func extractStringValue(expr sqlparser.Expr) string { - switch v := expr.(type) { - case *sqlparser.Literal: - // Remove surrounding quotes - val := v.Val - if len(val) >= 2 && (val[0] == '\'' || val[0] == '"') { - return val[1 : len(val)-1] - } - return val - } - return "" -} - func extractMetadata(ctx *QueryContext, stmt sqlparser.Statement) { switch parsed := stmt.(type) { case *sqlparser.Insert: diff --git a/protocol/query/parser_test.go b/protocol/query/parser_test.go index 57c4bfb1..5b693b13 100644 --- a/protocol/query/parser_test.go +++ b/protocol/query/parser_test.go @@ -433,7 +433,7 @@ func TestExtractInformationSchemaFilter(t *testing.T) { t.Fatalf("expected SELECT statement") } - filter := extractInformationSchemaFilter(sel) + filter := extractInformationSchemaFilter(sel, filterValues{}) if filter.SchemaName != tt.wantSchemaName { t.Errorf("SchemaName = %q, want %q", filter.SchemaName, tt.wantSchemaName) diff --git a/protocol/query/pipeline_autoincrement_test.go b/protocol/query/pipeline_autoincrement_test.go index 209d935d..5c8b25e5 100644 --- a/protocol/query/pipeline_autoincrement_test.go +++ b/protocol/query/pipeline_autoincrement_test.go @@ -13,15 +13,18 @@ type mockIDGenerator struct { counter atomic.Uint64 } -func (m *mockIDGenerator) NextID() uint64 { - return m.counter.Add(1) +func (m *mockIDGenerator) NextIDs(ids []uint64) error { + for i := range ids { + ids[i] = m.counter.Add(1) + } + return nil } // mockSchemaLookup returns a schema lookup that knows about specific tables. // The ordinal is the auto-increment column's position in the table's own column // order, which is what a column-less INSERT's VALUES tuple is indexed by. -func mockSchemaLookup(tables map[string]transform.SchemaInfo) func(string) *transform.SchemaInfo { - return func(table string) *transform.SchemaInfo { +func mockSchemaLookup(tables map[string]transform.SchemaInfo) func(string, string) *transform.SchemaInfo { + return func(_, table string) *transform.SchemaInfo { info, ok := tables[table] if !ok { return nil diff --git a/protocol/query/rules/autoincrement_id.go b/protocol/query/rules/autoincrement_id.go index 9f7c2f01..caa1cb87 100644 --- a/protocol/query/rules/autoincrement_id.go +++ b/protocol/query/rules/autoincrement_id.go @@ -1,17 +1,33 @@ package rules import ( + "errors" + "slices" "strconv" "strings" "github.com/maxpert/marmot/id" + "github.com/maxpert/marmot/protocol/mysqlcode" "github.com/maxpert/marmot/protocol/query/transform" + "github.com/maxpert/marmot/protocol/query/transform/intmarker" "vitess.io/vitess/go/vt/sqlparser" ) -// SchemaLookup returns the schema facts an INSERT needs for id injection. -// It returns nil when the table is unknown. -type SchemaLookup func(table string) *transform.SchemaInfo +// SchemaLookup returns the schema facts an INSERT needs for id injection, for +// the table in database - the statement's qualifier, or "" for the +// session's current database. It returns nil when the table is unknown. +type SchemaLookup func(database, table string) *transform.SchemaInfo + +// NarrowAllocator mints ids for AUTO_INCREMENT columns declared narrower than +// BIGINT, from ranges claimed cluster-wide so every id fits the column. It is +// id.RangeAllocator. +type NarrowAllocator interface { + // Allocate returns the first of n contiguous ids. + Allocate(database, table string, widthMax uint64, n int) (uint64, error) + // Admit decides whether the table may hold an id that did not come from + // Allocate (see id.RangeAllocator.Admit). + Admit(database, table string, widthMax, explicit uint64) error +} // AutoIncrementIDRule injects distributed IDs for auto-increment columns. // Detection is based on the SQLite schema: a single INTEGER/BIGINT PRIMARY KEY. @@ -22,11 +38,20 @@ type SchemaLookup func(table string) *transform.SchemaInfo // generated id wherever the id's position in the statement is knowable, and // refuses the statement only where safe substitution is structurally // impossible. +// +// Which ids a table gets depends on how its key was declared. A column +// declared AUTO_INCREMENT narrower than BIGINT gets the narrow allocator's +// ids, which fit the declared width, and every id a client supplies for it is +// admitted (id.RangeAllocator.Admit) before any row is written. Every other +// key - BIGINT, a table created before width markers existed, and a narrow +// INTEGER PRIMARY KEY not declared AUTO_INCREMENT, whose ids MySQL never +// generates - keeps the wide generator's 53- or 64-bit ids for a row that +// leaves the key out, as always, and its client-supplied ids untouched. type AutoIncrementIDRule struct { generator id.Generator } -// NewAutoIncrementIDRule creates a new rule with the given ID generator. +// NewAutoIncrementIDRule creates a new rule with the given wide ID generator. func NewAutoIncrementIDRule(gen id.Generator) *AutoIncrementIDRule { return &AutoIncrementIDRule{ generator: gen, @@ -36,6 +61,13 @@ func NewAutoIncrementIDRule(gen id.Generator) *AutoIncrementIDRule { func (r *AutoIncrementIDRule) Name() string { return "AutoIncrementID" } func (r *AutoIncrementIDRule) Priority() int { return 5 } +// IsNarrow reports whether a table's auto-increment column was declared +// AUTO_INCREMENT with a width marker, and so takes its ids from the narrow +// allocator. +func IsNarrow(info *transform.SchemaInfo) bool { + return info.HasAutoIncrement() && info.AutoIncrementWidth != 0 && info.AutoIncrementExplicit +} + // insertPlan is what analyzeInsert decided ApplyAST must do to an INSERT. // Exactly one of appendColumn and valueIdx is meaningful: appendColumn names a // column to add to an explicit column list (with one generated value per row), @@ -44,8 +76,59 @@ func (r *AutoIncrementIDRule) Priority() int { return 5 } type insertPlan struct { insert *sqlparser.Insert rows sqlparser.Values + info *transform.SchemaInfo + table string appendColumn string valueIdx int + // bound holds a prepared statement's bound values, and args the + // position among them of each of the statement's placeholders. + bound []interface{} + args map[*sqlparser.Argument]int +} + +// needsID reports whether expr, in the auto-increment column's position, +// asks the server for an id. A narrow column treats a placeholder bound to +// NULL or 0 exactly as the literal: SQLite must never assign its rowid. +func (p *insertPlan) needsID(expr sqlparser.Expr) bool { + if needsIDInjection(expr) { + return true + } + _, ok := p.boundIDRequest(expr) + return ok +} + +// boundIDRequest returns the bound-value position of a placeholder bound to +// NULL or 0 in a narrow column. +func (p *insertPlan) boundIDRequest(expr sqlparser.Expr) (int, bool) { + arg, ok := expr.(*sqlparser.Argument) + if !ok || !IsNarrow(p.info) { + return 0, false + } + i, ok := p.args[arg] + if !ok || i >= len(p.bound) || !boundRequestsID(p.bound[i]) { + return 0, false + } + return i, true +} + +// boundRequestsID reports whether a bound value asks for a generated id, as +// NULL and 0 do in MySQL. +func boundRequestsID(v interface{}) bool { + switch x := v.(type) { + case nil: + return true + case int64: + return x == 0 + case uint64: + return x == 0 + case int: + return x == 0 + case []byte: + return string(x) == "0" + case string: + return x == "0" + } + return false } // analyzeInsert decides what the rule must do with a statement, without @@ -53,18 +136,18 @@ type insertPlan struct { // the statement can be rewritten, and an error when the statement must be // refused. NeedsIDInjection and ApplyAST both go through it so the transpiler's // gate and the rewrite can never disagree. -func analyzeInsert(stmt sqlparser.Statement, schemaLookup SchemaLookup) (*insertPlan, error) { +func analyzeInsert(stmt sqlparser.Statement, schemaLookup SchemaLookup, bound []interface{}) (*insertPlan, error) { insert, ok := stmt.(*sqlparser.Insert) if !ok { return nil, nil } - tableName := extractInsertTableName(insert) + database, tableName := extractInsertTableName(insert) if tableName == "" { return nil, nil } - info := schemaLookup(tableName) + info := schemaLookup(database, tableName) if !info.HasAutoIncrement() { return nil, nil } @@ -97,105 +180,277 @@ func analyzeInsert(stmt sqlparser.Statement, schemaLookup SchemaLookup) (*insert tableName, autoIncCol) } - if colIdx >= 0 { - // The column list names the auto-increment column: substitute wherever - // a row left the value for the server to fill in. - if rowsNeedID(rows, colIdx) { - return &insertPlan{insert: insert, rows: rows, valueIdx: colIdx}, nil - } - return nil, nil + plan := &insertPlan{insert: insert, rows: rows, info: info, table: tableName, valueIdx: -1, bound: bound} + if len(bound) > 0 && IsNarrow(info) { + plan.args = transform.ArgumentPositions(insert) } - if len(insert.Columns) > 0 { + if colIdx < 0 && len(insert.Columns) > 0 { // Explicit column list that omits the auto-increment column: add it, // with one generated id per row. - return &insertPlan{insert: insert, rows: rows, appendColumn: autoIncCol, valueIdx: -1}, nil + plan.appendColumn = autoIncCol + return plan, nil } - // Column-less INSERT. Every column gets a value in the table's own order, - // so the id sits at the auto-increment column's ordinal. A row of the - // wrong length is SQLite's error to report, exactly as it is today, so - // rows that cannot hold the ordinal are left untouched. - ordinal := info.AutoIncrementOrdinal - if ordinal < 0 { - return nil, nil + // A column list naming the auto-increment column puts its values at + // colIdx. A column-less INSERT gives every column a value in the table's + // own order, so the id sits at the auto-increment column's ordinal; a row + // of the wrong length is SQLite's error to report, exactly as it is + // today, so rows that cannot hold the ordinal are left untouched. + idx := colIdx + if idx < 0 { + idx = info.AutoIncrementOrdinal + if idx < 0 { + return nil, nil + } } - if rowsNeedID(rows, ordinal) { - return &insertPlan{insert: insert, rows: rows, valueIdx: ordinal}, nil + plan.valueIdx = idx + + // A narrow column's explicit ids are admitted even when no row asks for + // a generated one, so that no node issues them later. + if plan.rowsNeedID() || (IsNarrow(info) && rowsHaveExplicitID(rows, idx)) { + return plan, nil } return nil, nil } -// rowsNeedID reports whether any row leaves position idx for the server to -// fill in. -func rowsNeedID(rows sqlparser.Values, idx int) bool { - for _, row := range rows { - if idx < len(row) && needsIDInjection(row[idx]) { +// rowsNeedID reports whether any row leaves the auto-increment position for +// the server to fill in. +func (p *insertPlan) rowsNeedID() bool { + for _, row := range p.rows { + if p.valueIdx < len(row) && p.needsID(row[p.valueIdx]) { return true } } return false } +// rowsHaveExplicitID reports whether any row supplies a positive integer +// literal at position idx. +func rowsHaveExplicitID(rows sqlparser.Values, idx int) bool { + for _, row := range rows { + if idx < len(row) { + if _, ok := explicitID(row[idx]); ok { + return true + } + } + } + return false +} + +// explicitID returns the id a client supplied as a positive integer literal. +// Anything else - a negative number, an expression, a bound parameter - is +// not an id the allocator could ever issue or cannot be read before +// execution, and is left to the database as it always was. +func explicitID(expr sqlparser.Expr) (uint64, bool) { + lit, ok := expr.(*sqlparser.Literal) + if !ok || lit.Type != sqlparser.IntVal { + return 0, false + } + v, err := strconv.ParseUint(strings.TrimSpace(lit.Val), 10, 64) + if err != nil || v == 0 { + return 0, false + } + return v, true +} + // NeedsIDInjection reports whether the statement needs this rule to run. The // transpiler uses it both to decide whether to call ApplyAST and to bypass its // SQL cache, so a statement this rule would refuse must answer true as well. -func (r *AutoIncrementIDRule) NeedsIDInjection(stmt sqlparser.Statement, schemaLookup SchemaLookup) bool { +// bound is a prepared statement's bound values, nil for a text query. +func (r *AutoIncrementIDRule) NeedsIDInjection(stmt sqlparser.Statement, schemaLookup SchemaLookup, bound []interface{}) bool { if r.generator == nil || schemaLookup == nil { return false } - plan, err := analyzeInsert(stmt, schemaLookup) + plan, err := analyzeInsert(stmt, schemaLookup, bound) return err != nil || plan != nil } // ApplyAST rewrites the statement in place, or returns an error when the // statement must be refused. The error is returned to the caller verbatim; a // *transform.CodedError carries the MySQL error code the client must see. -func (r *AutoIncrementIDRule) ApplyAST(stmt sqlparser.Statement, schemaLookup SchemaLookup) (sqlparser.Statement, bool, error) { +// narrow mints the ids of a column declared narrower than BIGINT; nil refuses +// such an INSERT. bound is a prepared statement's bound values, nil for a +// text query. A generated id for a placeholder bound to NULL or 0 is not +// written into the statement: it is returned in boundIDs, keyed by the +// placeholder's position among the bound values, to be bound in its place +// (protocol.Statement.MergeExecParams). +func (r *AutoIncrementIDRule) ApplyAST(stmt sqlparser.Statement, schemaLookup SchemaLookup, narrow NarrowAllocator, bound []interface{}) (out sqlparser.Statement, applied bool, boundIDs map[int]uint64, err error) { if r.generator == nil || schemaLookup == nil { - return stmt, false, nil + return stmt, false, nil, nil } - plan, err := analyzeInsert(stmt, schemaLookup) + plan, err := analyzeInsert(stmt, schemaLookup, bound) if err != nil { - return stmt, false, err + return stmt, false, nil, err } if plan == nil { - return stmt, false, nil + return stmt, false, nil, nil + } + + var targets []int + if plan.appendColumn == "" { + for i, row := range plan.rows { + if plan.valueIdx < len(row) && plan.needsID(row[plan.valueIdx]) { + targets = append(targets, i) + } + } + } + count := len(targets) + if plan.appendColumn != "" { + count = len(plan.rows) + } + + ids, err := r.mint(plan, count, narrow) + if err != nil { + return stmt, false, nil, err } if plan.appendColumn != "" { plan.insert.Columns = append(plan.insert.Columns, sqlparser.NewIdentifierCI(plan.appendColumn)) for i := range plan.rows { - plan.rows[i] = append(plan.rows[i], r.newIDLiteral()) + plan.rows[i] = append(plan.rows[i], idLiteral(ids[i])) + } + return plan.insert, true, nil, nil + } + for i, row := range targets { + if pos, ok := plan.boundIDRequest(plan.rows[row][plan.valueIdx]); ok { + if boundIDs == nil { + boundIDs = make(map[int]uint64) + } + boundIDs[pos] = ids[i] + continue } - return plan.insert, true, nil + plan.rows[row][plan.valueIdx] = idLiteral(ids[i]) } + return plan.insert, count > 0, boundIDs, nil +} - // analyzeInsert only returns a substitution plan when at least one row - // needs one, so this loop always replaces something. - for _, row := range plan.rows { - if plan.valueIdx < len(row) && needsIDInjection(row[plan.valueIdx]) { - row[plan.valueIdx] = r.newIDLiteral() +// mint returns count ids for the plan's rows, in row order. For a narrow +// column it first observes every id the client supplied, so the ids it then +// generates can never equal one of them. +func (r *AutoIncrementIDRule) mint(plan *insertPlan, count int, narrow NarrowAllocator) ([]uint64, error) { + if !IsNarrow(plan.info) { + ids := make([]uint64, count) + if count > 0 { + if err := r.generator.NextIDs(ids); err != nil { + return nil, AllocationError(err, plan.table, WidthMax(plan.info)) + } } + return ids, nil + } + + if narrow == nil { + return nil, transform.NewCodedError(mysqlcode.ErrCodeUnknown, + "no allocator for AUTO_INCREMENT column `%s` of table `%s`", plan.info.AutoIncrementColumn, plan.table) + } + ceiling := WidthMax(plan.info) + if err := admitExplicitIDs(plan, ceiling, narrow); err != nil { + return nil, err + } + + ids := make([]uint64, count) + if count == 0 { + return ids, nil + } + first, err := narrow.Allocate(plan.info.Database, plan.table, ceiling, count) + if err != nil { + return nil, AllocationError(err, plan.table, WidthMax(plan.info)) + } + for i := range ids { + ids[i] = first + uint64(i) } - return plan.insert, true, nil + return ids, nil +} + +// admitExplicitIDs admits every id the plan's rows supply themselves, in +// ascending order, before any row is written, so a refused id refuses the +// statement and a mixed statement's generated ids can never equal one of +// them. Ids known only once the statement has run are admitted from its CDC +// rows by the same allocator (coordinator.admitNarrowIDs). +func admitExplicitIDs(plan *insertPlan, ceiling uint64, narrow NarrowAllocator) error { + if plan.valueIdx < 0 { + return nil + } + var explicit []uint64 + for i, row := range plan.rows { + if plan.valueIdx >= len(row) { + continue + } + v, ok := explicitID(row[plan.valueIdx]) + if !ok { + continue + } + if v > ceiling { + return transform.NewCodedError(mysqlcode.ErrCodeDataOutOfRange, + "Out of range value for column '%s' at row %d", plan.info.AutoIncrementColumn, i+1) + } + explicit = append(explicit, v) + } + slices.Sort(explicit) + for _, v := range explicit { + if err := narrow.Admit(plan.info.Database, plan.table, ceiling, v); err != nil { + return AdmissionError(err, plan.table, plan.info.AutoIncrementColumn, ceiling) + } + } + return nil +} + +// AdmissionError turns a refused id for table's narrow column into the error +// a client sees: an id the column cannot hold is ER_WARN_DATA_OUT_OF_RANGE, +// as MySQL's strict mode reports it; an id at or below the cluster's +// allocation base that no range of this node holds is ER_NOT_SUPPORTED_YET, +// since it may lie in another node's unissued range; anything else is an +// allocation failure (AllocationError). +func AdmissionError(err error, table, column string, widthMax uint64) error { + var below *id.BelowBaseError + switch { + case errors.Is(err, id.ErrIDOutOfRange): + return transform.NewCodedError(mysqlcode.ErrCodeDataOutOfRange, + "Out of range value for column '%s'", column) + case errors.As(err, &below): + return transform.NewCodedError(mysqlcode.ErrCodeNotSupportedYet, + "This version of MySQL doesn't yet support 'an explicit id %d for AUTO_INCREMENT column `%s` of table `%s` at or below the cluster's allocation base %d'", + below.ID, column, table, below.Base) + default: + return AllocationError(err, table, widthMax) + } +} + +// AllocationError turns a failure to mint or observe narrow ids for table, +// whose column holds at most widthMax, into the error a client sees: a full +// column is ER_DUP_ENTRY on the column's maximum, as MySQL reports it, and +// never retryable; anything else is the retryable lock-wait timeout. +func AllocationError(err error, table string, widthMax uint64) error { + if errors.Is(err, id.ErrRangeExhausted) { + return transform.NewCodedError(mysqlcode.ErrCodeDupEntry, + "Duplicate entry '%d' for key '%s.PRIMARY'", widthMax, table) + } + return transform.NewCodedError(mysqlcode.ErrCodeLockTimeout, + "Lock wait timeout exceeded; try restarting transaction (auto-increment ids for `%s`: %v)", table, err) +} + +// WidthMax is the largest id a table's narrow auto-increment column can hold; +// zero for a column with no width marker. +func WidthMax(info *transform.SchemaInfo) uint64 { + return intmarker.Attributes{Bits: info.AutoIncrementWidth, Unsigned: info.AutoIncrementUnsigned}.WidthMax() } -// newIDLiteral mints one id and renders it as an integer literal. -func (r *AutoIncrementIDRule) newIDLiteral() *sqlparser.Literal { - return sqlparser.NewIntLiteral(strconv.FormatUint(r.generator.NextID(), 10)) +// idLiteral renders an id as an integer literal. +func idLiteral(v uint64) *sqlparser.Literal { + return sqlparser.NewIntLiteral(strconv.FormatUint(v, 10)) } -// extractInsertTableName extracts the table name from an INSERT statement. -func extractInsertTableName(insert *sqlparser.Insert) string { +// extractInsertTableName returns the database qualifier ("" when absent) and +// the table name of an INSERT statement. +func extractInsertTableName(insert *sqlparser.Insert) (database, table string) { if insert.Table == nil { - return "" + return "", "" } if tn, ok := insert.Table.Expr.(sqlparser.TableName); ok { - return tn.Name.String() + return tn.Qualifier.String(), tn.Name.String() } - return "" + return "", "" } // findColumnIndex returns the index of a column in the columns list, or -1 if not found. diff --git a/protocol/query/rules/autoincrement_id_narrow_test.go b/protocol/query/rules/autoincrement_id_narrow_test.go new file mode 100644 index 00000000..af0591af --- /dev/null +++ b/protocol/query/rules/autoincrement_id_narrow_test.go @@ -0,0 +1,299 @@ +package rules + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/maxpert/marmot/id" + "github.com/maxpert/marmot/protocol/mysqlcode" + "github.com/maxpert/marmot/protocol/query/transform" + "vitess.io/vitess/go/vt/sqlparser" +) + +// fakeNarrow records what the rule asks of the narrow allocator. Allocate +// hands out ids from next upward; Admit moves next past an explicit id, or +// refuses it with admitErr. +type fakeNarrow struct { + next uint64 + allocs []int + observed []uint64 + allocErr error + admitErr error + database string + lastWidth uint64 +} + +func (f *fakeNarrow) Allocate(database, table string, widthMax uint64, n int) (uint64, error) { + f.database, f.lastWidth = database, widthMax + f.allocs = append(f.allocs, n) + if f.allocErr != nil { + return 0, f.allocErr + } + first := f.next + f.next += uint64(n) + return first, nil +} + +func (f *fakeNarrow) Admit(database, table string, widthMax, explicit uint64) error { + f.observed = append(f.observed, explicit) + if f.admitErr != nil { + return f.admitErr + } + if explicit >= f.next { + f.next = explicit + 1 + } + return nil +} + +// narrowSchemas: a signed INT AUTO_INCREMENT table with the id first, one +// with it third, a TINYINT one, and a narrow INTEGER PRIMARY KEY not declared +// AUTO_INCREMENT. +var narrowSchemas = map[string]transform.SchemaInfo{ + "alias": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 0, AutoIncrementWidth: 32, Database: "lldap"}, + "groups": {AutoIncrementColumn: "group_id", AutoIncrementOrdinal: 0, AutoIncrementWidth: 32, AutoIncrementExplicit: true, Database: "lldap"}, + "late": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 2, AutoIncrementWidth: 32, AutoIncrementExplicit: true, Database: "lldap"}, + "tiny": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 0, AutoIncrementWidth: 8, AutoIncrementExplicit: true, Database: "lldap"}, +} + +func applyNarrow(t *testing.T, sql string, narrow NarrowAllocator) (string, bool, error) { + t.Helper() + rule := NewAutoIncrementIDRule(&seqGenerator{}) + stmt := parseOne(t, sql) + got, applied, _, err := rule.ApplyAST(stmt, lookupFor(narrowSchemas), narrow, nil) + if err != nil { + return "", applied, err + } + return sqlparser.String(got), applied, nil +} + +// TestNarrowInjectionShapes pins every injection shape on a marked column: +// the ids come from the narrow allocator, one Allocate per statement for +// exactly the rows that need one, contiguous in row order. +// +// Mutation: route marked columns to the wide generator. The 1000-series ids +// of seqGenerator appear instead of 1-series ids and every case fires. +func TestNarrowInjectionShapes(t *testing.T) { + cases := []struct { + name string + sql string + want string + wantAllocs []int + }{ + {"column list omitting the id", "insert into groups (name) values ('a'), ('b'), ('c')", + "insert into `groups`(`name`, group_id) values ('a', 1), ('b', 2), ('c', 3)", []int{3}}, + {"column-less at ordinal 0, NULL 0 and DEFAULT alike", "insert into groups values (null, 'a'), (0, 'b'), (default, 'c')", + "insert into `groups` values (1, 'a'), (2, 'b'), (3, 'c')", []int{3}}, + {"column-less at a non-zero ordinal", "insert into late values ('x', 'y', null)", + "insert into late values ('x', 'y', 1)", []int{1}}, + {"listed id with DEFAULT", "insert into groups (group_id, name) values (default, 'a')", + "insert into `groups`(group_id, `name`) values (1, 'a')", []int{1}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + narrow := &fakeNarrow{next: 1} + got, applied, err := applyNarrow(t, tc.sql, narrow) + if err != nil { + t.Fatalf("ApplyAST: %v", err) + } + if !applied || got != tc.want { + t.Fatalf("got %q (applied=%v), want %q", got, applied, tc.want) + } + if fmt.Sprint(narrow.allocs) != fmt.Sprint(tc.wantAllocs) { + t.Fatalf("Allocate calls %v, want %v: a statement takes its ids in one call", narrow.allocs, tc.wantAllocs) + } + if narrow.database != "lldap" || narrow.lastWidth != 2147483647 { + t.Fatalf("allocated for %q with ceiling %d, want lldap / 2147483647", narrow.database, narrow.lastWidth) + } + }) + } +} + +// TestNarrowExplicitIDsAreAdmittedBeforeAllocating pins explicit literal ids: +// every one is admitted, in ascending order, before any id is generated, so a +// statement mixing explicit and generated ids never generates one it also +// supplies, and a statement of explicit ids only is still admitted. +// +// Mutation: skip Admit when a statement carries explicit ids. "explicit ids +// were not admitted" fires, and the mixed statement collides. +func TestNarrowExplicitIDsAreAdmittedBeforeAllocating(t *testing.T) { + narrow := &fakeNarrow{next: 1} + got, applied, err := applyNarrow(t, "insert into groups values (null, 'a'), (2, 'b'), (null, 'c')", narrow) + if err != nil { + t.Fatalf("ApplyAST: %v", err) + } + if want := "insert into `groups` values (3, 'a'), (2, 'b'), (4, 'c')"; !applied || got != want { + t.Fatalf("got %q, want %q", got, want) + } + + dump := &fakeNarrow{next: 1} + got, applied, err = applyNarrow(t, "insert into groups values (900, 'b'), (7, 'a')", dump) + if err != nil { + t.Fatalf("ApplyAST: %v", err) + } + if fmt.Sprint(dump.observed) != "[7 900]" { + t.Fatalf("explicit ids were not admitted in ascending order: %v", dump.observed) + } + if len(dump.allocs) != 0 || applied { + t.Fatalf("a statement of explicit ids generated ids: allocs=%v applied=%v", dump.allocs, applied) + } + if want := "insert into `groups` values (900, 'b'), (7, 'a')"; got != want { + t.Fatalf("explicit ids were rewritten: %q", got) + } +} + +// TestNarrowErrorsReachTheClientWithMySQLCodes pins the three refusals: an +// explicit id past the column is 1264, a full column is 1062 on the column's +// maximum, and an allocator that cannot reach a quorum is the retryable 1205. +// +// Mutation: map every allocator error to one code. One of the arms fires. +func TestNarrowErrorsReachTheClientWithMySQLCodes(t *testing.T) { + cases := []struct { + name string + sql string + allocErr error + wantCode uint16 + wantMsg string + }{ + {"explicit id above the column", "insert into tiny values (128, 'a')", nil, mysqlcode.ErrCodeDataOutOfRange, + "Out of range value for column 'id' at row 1"}, + {"column exhausted", "insert into tiny values (null, 'a')", fmt.Errorf("claim: %w", id.ErrRangeExhausted), + mysqlcode.ErrCodeDupEntry, "Duplicate entry '127' for key 'tiny.PRIMARY'"}, + {"quorum unavailable", "insert into tiny values (null, 'a')", errors.New("prepare quorum not achieved"), + mysqlcode.ErrCodeLockTimeout, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, _, err := applyNarrow(t, tc.sql, &fakeNarrow{next: 1, allocErr: tc.allocErr}) + var coded *transform.CodedError + if !errors.As(err, &coded) || coded.Code != tc.wantCode { + t.Fatalf("err = %v, want MySQL code %d", err, tc.wantCode) + } + if tc.wantMsg != "" && coded.Message != tc.wantMsg { + t.Fatalf("message %q, want %q", coded.Message, tc.wantMsg) + } + }) + } +} + +// TestNarrowWithoutAllocatorIsRefused pins that a path with no narrow +// allocator never falls back to wide ids for a marked column. +// +// Mutation: fall back to the wide generator when narrow is nil. The 53-bit +// shape of the bug this work fixes returns and "a marked column got a wide +// id" fires. +func TestNarrowWithoutAllocatorIsRefused(t *testing.T) { + got, applied, err := applyNarrow(t, "insert into groups (name) values ('a')", nil) + if err == nil { + t.Fatalf("a marked column got a wide id: %q (applied=%v)", got, applied) + } +} + +// TestWideTableKeepsTheWideGenerator pins that an unmarked table - BIGINT, +// and every table created before markers existed - never touches the narrow +// allocator. +// +// Mutation: route every auto-increment table to the narrow allocator. +// "an unmarked table used the narrow allocator" fires. +func TestWideTableKeepsTheWideGenerator(t *testing.T) { + narrow := &fakeNarrow{next: 1} + rule := NewAutoIncrementIDRule(&seqGenerator{}) + stmt := parseOne(t, "insert into idfirst (v) values ('a')") + got, applied, _, err := rule.ApplyAST(stmt, lookupFor(injectionSchemas), narrow, nil) + if err != nil || !applied { + t.Fatalf("ApplyAST: applied=%v err=%v", applied, err) + } + if len(narrow.allocs) != 0 || len(narrow.observed) != 0 { + t.Fatalf("an unmarked table used the narrow allocator") + } + if want := "insert into idfirst(v, id) values ('a', 1000)"; sqlparser.String(got) != want { + t.Fatalf("got %q, want %q", sqlparser.String(got), want) + } +} + +// TestNarrowExplicitIDAtOrBelowTheBaseIsRefused pins the refusal the rule +// gives an id the allocator will not admit: ER_NOT_SUPPORTED_YET naming the +// column and the base, before any row is written. +// +// Mutation: map a *id.BelowBaseError like any allocation failure. The code +// becomes 1205 and this fires. +func TestNarrowExplicitIDAtOrBelowTheBaseIsRefused(t *testing.T) { + narrow := &fakeNarrow{next: 1, admitErr: &id.BelowBaseError{ID: 30, Base: 128}} + _, _, err := applyNarrow(t, "insert into groups values (30, 'x')", narrow) + var coded *transform.CodedError + if !errors.As(err, &coded) || coded.Code != mysqlcode.ErrCodeNotSupportedYet { + t.Fatalf("err = %v, want MySQL code %d", err, mysqlcode.ErrCodeNotSupportedYet) + } + if !strings.Contains(coded.Message, "group_id") || !strings.Contains(coded.Message, "128") { + t.Fatalf("refusal %q does not name the column and the base", coded.Message) + } + if len(narrow.allocs) != 0 { + t.Fatalf("ids were generated for a refused statement") + } +} + +// TestNarrowKeyNotDeclaredAutoIncrementKeepsTheWidePath pins that a narrow +// INTEGER PRIMARY KEY not declared AUTO_INCREMENT never touches the narrow +// allocator: MySQL never generates its ids, so the ids clients write there +// are theirs, and the admission rule for AUTO_INCREMENT columns does not +// apply. +// +// Mutation: make IsNarrow ignore AutoIncrementExplicit. "a key not declared +// AUTO_INCREMENT used the narrow allocator" fires. +func TestNarrowKeyNotDeclaredAutoIncrementKeepsTheWidePath(t *testing.T) { + narrow := &fakeNarrow{next: 1} + got, applied, err := applyNarrow(t, "insert into alias values (7, 'x')", narrow) + if err != nil || applied { + t.Fatalf("a client id in a key not declared AUTO_INCREMENT was rewritten: %q applied=%v err=%v", got, applied, err) + } + if _, _, err := applyNarrow(t, "insert into alias values (null, 'x')", narrow); err != nil { + t.Fatalf("ApplyAST: %v", err) + } + if len(narrow.allocs) != 0 || len(narrow.observed) != 0 { + t.Fatalf("a key not declared AUTO_INCREMENT used the narrow allocator") + } +} + +// TestNarrowBoundNullOrZeroIDIsGenerated is F3: a prepared INSERT whose id +// placeholder is bound to NULL or 0 gets a generated id, exactly as the +// literal NULL does, and never reaches SQLite, which would assign its own +// rowid. The id is bound in the placeholder's place rather than written into +// the statement, so the caller's other values keep their positions. An +// explicit bound id, and a wide table, are left alone. +// +// Mutation: make boundRequestsID return false for nil. The first row's +// placeholder gets no id and "a placeholder bound to NULL got no id" fires. +func TestNarrowBoundNullOrZeroIDIsGenerated(t *testing.T) { + apply := func(sql string, schemas map[string]transform.SchemaInfo, bound ...interface{}) (string, map[int]uint64, *fakeNarrow) { + t.Helper() + narrow := &fakeNarrow{next: 100} + got, _, ids, err := NewAutoIncrementIDRule(&seqGenerator{}).ApplyAST(parseOne(t, sql), lookupFor(schemas), narrow, bound) + if err != nil { + t.Fatalf("%s: %v", sql, err) + } + return sqlparser.String(got), ids, narrow + } + + sql, ids, narrow := apply("INSERT INTO groups (group_id, name) VALUES (?, ?), (?, ?), (?, ?)", narrowSchemas, + nil, "a", int64(7), "b", int64(0), "c") + if want := map[int]uint64{0: 100, 4: 101}; fmt.Sprint(ids) != fmt.Sprint(want) { + t.Fatalf("a placeholder bound to NULL got no id: bound ids %v, want %v", ids, want) + } + if !strings.Contains(sql, "values (:v1, :v2), (:v3, :v4), (:v5, :v6)") { + t.Fatalf("the statement's placeholders were rewritten: %s", sql) + } + if fmt.Sprint(narrow.allocs) != "[2]" { + t.Fatalf("allocations %v, want one of 2", narrow.allocs) + } + + if _, ids, _ := apply("INSERT INTO groups (group_id, name) VALUES (?, ?)", narrowSchemas, []byte("0"), "a"); ids[0] != 100 { + t.Fatalf("a placeholder bound to '0' got no id: %v", ids) + } + if _, ids, narrow := apply("INSERT INTO groups (group_id, name) VALUES (?, ?)", narrowSchemas, int64(9), "a"); len(ids) != 0 || len(narrow.allocs) != 0 { + t.Fatalf("an explicit bound id was replaced: %v, allocations %v", ids, narrow.allocs) + } + if _, ids, _ := apply("INSERT INTO idfirst (id, name) VALUES (?, ?)", injectionSchemas, nil, "a"); len(ids) != 0 { + t.Fatalf("a wide table's bound NULL was replaced: %v", ids) + } +} diff --git a/protocol/query/rules/autoincrement_id_test.go b/protocol/query/rules/autoincrement_id_test.go index 73065aec..51894385 100644 --- a/protocol/query/rules/autoincrement_id_test.go +++ b/protocol/query/rules/autoincrement_id_test.go @@ -14,11 +14,16 @@ import ( // the serialized statement and the order of injection is observable. type seqGenerator struct{ n atomic.Uint64 } -func (g *seqGenerator) NextID() uint64 { return 1000 + g.n.Add(1) - 1 } +func (g *seqGenerator) NextIDs(ids []uint64) error { + for i := range ids { + ids[i] = 1000 + g.n.Add(1) - 1 + } + return nil +} // lookupFor builds the schema lookup the rule receives. func lookupFor(tables map[string]transform.SchemaInfo) SchemaLookup { - return func(table string) *transform.SchemaInfo { + return func(_, table string) *transform.SchemaInfo { info, ok := tables[table] if !ok { return nil @@ -140,8 +145,8 @@ func TestAutoIncrementIDRuleInjectionShapes(t *testing.T) { stmt := parseOne(t, tc.sql) before := sqlparser.String(stmt) - needs := rule.NeedsIDInjection(stmt, lookup) - got, applied, err := rule.ApplyAST(stmt, lookup) + needs := rule.NeedsIDInjection(stmt, lookup, nil) + got, applied, _, err := rule.ApplyAST(stmt, lookup, nil, nil) if err != nil { t.Fatalf("ApplyAST returned an error for %q: %v", tc.sql, err) } @@ -181,11 +186,11 @@ func TestAutoIncrementIDRuleRejectsUnprojectedInsertSelect(t *testing.T) { // Mutation: restore the `insert.Rows.(sqlparser.Values)` early return // in analyzeInsert. The statement is then accepted, SQLite assigns the // rowid, and two nodes both mint 1, 2, 3. - if !rule.NeedsIDInjection(stmt, lookup) { + if !rule.NeedsIDInjection(stmt, lookup, nil) { t.Fatalf("NeedsIDInjection = false for %q: the transpiler would cache it and never call ApplyAST", sql) } - _, applied, err := rule.ApplyAST(stmt, lookup) + _, applied, _, err := rule.ApplyAST(stmt, lookup, nil, nil) if err == nil { t.Fatalf("ApplyAST accepted %q; want a rejection", sql) } @@ -236,10 +241,10 @@ func TestAutoIncrementIDRuleRejectsRowConstructor(t *testing.T) { // Mutation: drop the *sqlparser.ValuesStatement arm in analyzeInsert. // The column-listed form then returns nothing-to-do and this fires. - if !rule.NeedsIDInjection(stmt, lookup) { + if !rule.NeedsIDInjection(stmt, lookup, nil) { t.Fatalf("NeedsIDInjection = false for %q: the transpiler would cache it and never call ApplyAST", sql) } - _, applied, err := rule.ApplyAST(stmt, lookup) + _, applied, _, err := rule.ApplyAST(stmt, lookup, nil, nil) if err == nil { t.Fatalf("ApplyAST accepted %q; want a 1235 rejection", sql) } @@ -270,18 +275,18 @@ func TestAutoIncrementIDRuleNoLookupOrGenerator(t *testing.T) { // Mutation: drop either nil guard in ApplyAST/NeedsIDInjection; the rule // dereferences a nil generator or a nil lookup and panics. rule := NewAutoIncrementIDRule(nil) - if rule.NeedsIDInjection(stmt, lookupFor(injectionSchemas)) { + if rule.NeedsIDInjection(stmt, lookupFor(injectionSchemas), nil) { t.Error("NeedsIDInjection = true with a nil generator") } - if _, applied, err := rule.ApplyAST(stmt, lookupFor(injectionSchemas)); applied || err != nil { + if _, applied, _, err := rule.ApplyAST(stmt, lookupFor(injectionSchemas), nil, nil); applied || err != nil { t.Errorf("ApplyAST with a nil generator: applied=%v err=%v", applied, err) } rule = NewAutoIncrementIDRule(&seqGenerator{}) - if rule.NeedsIDInjection(stmt, nil) { + if rule.NeedsIDInjection(stmt, nil, nil) { t.Error("NeedsIDInjection = true with a nil schema lookup") } - if _, applied, err := rule.ApplyAST(stmt, nil); applied || err != nil { + if _, applied, _, err := rule.ApplyAST(stmt, nil, nil, nil); applied || err != nil { t.Errorf("ApplyAST with a nil schema lookup: applied=%v err=%v", applied, err) } } @@ -295,10 +300,10 @@ func TestAutoIncrementIDRuleUnknownTable(t *testing.T) { lookup := lookupFor(injectionSchemas) stmt := parseOne(t, "insert into nosuchtable values (null, 'a')") - if rule.NeedsIDInjection(stmt, lookup) { + if rule.NeedsIDInjection(stmt, lookup, nil) { t.Error("NeedsIDInjection = true for a table the lookup does not know") } - if _, applied, err := rule.ApplyAST(stmt, lookup); applied || err != nil { + if _, applied, _, err := rule.ApplyAST(stmt, lookup, nil, nil); applied || err != nil { t.Errorf("ApplyAST on an unknown table: applied=%v err=%v", applied, err) } } diff --git a/protocol/query/transform/literal_extractor.go b/protocol/query/transform/literal_extractor.go index 0ceef970..e9468284 100644 --- a/protocol/query/transform/literal_extractor.go +++ b/protocol/query/transform/literal_extractor.go @@ -6,6 +6,21 @@ import ( "vitess.io/vitess/go/vt/sqlparser" ) +// ArgumentPositions numbers the statement's placeholders in the order its +// bound values apply to them: the walk order ExtractLiterals and +// protocol.Statement.MergeExecParams rely on. Every reader of a bound value +// finds its placeholder's value through this one numbering. +func ArgumentPositions(stmt sqlparser.SQLNode) map[*sqlparser.Argument]int { + args := make(map[*sqlparser.Argument]int) + sqlparser.Rewrite(stmt, func(cursor *sqlparser.Cursor) bool { + if arg, ok := cursor.Node().(*sqlparser.Argument); ok { + args[arg] = len(args) + } + return true + }, nil) + return args +} + // ExtractLiterals walks the AST and replaces literal values with argument placeholders. // Returns the extracted values in order of appearance, suitable for parameterized execution. // diff --git a/protocol/query/transform/rule.go b/protocol/query/transform/rule.go index 9281f08c..9b6f72af 100644 --- a/protocol/query/transform/rule.go +++ b/protocol/query/transform/rule.go @@ -65,6 +65,13 @@ type SchemaInfo struct { // AutoIncrementUnsigned mirrors the MySQL UNSIGNED modifier on that column. // Meaningless when AutoIncrementWidth is zero. AutoIncrementUnsigned bool + // AutoIncrementExplicit reports that the column was declared + // AUTO_INCREMENT, as opposed to being a narrow INTEGER PRIMARY KEY that + // SQLite treats as a rowid alias. Meaningless when AutoIncrementWidth is + // zero. + AutoIncrementExplicit bool + // Database is the database the table was resolved in. + Database string } // HasAutoIncrement returns true if table has an auto-increment column diff --git a/protocol/query/transpiler.go b/protocol/query/transpiler.go index 141ec16a..05efea09 100644 --- a/protocol/query/transpiler.go +++ b/protocol/query/transpiler.go @@ -73,7 +73,7 @@ func NewTranspiler(cacheSize int, idGen id.Generator) (*Transpiler, error) { func (t *Transpiler) Transpile(ctx *QueryContext) error { // Check if this statement needs ID injection needsIDInjection := t.autoIncRule != nil && ctx.SchemaLookup != nil && - t.autoIncRule.NeedsIDInjection(ctx.MySQLState.AST, ctx.SchemaLookup) + t.autoIncRule.NeedsIDInjection(ctx.MySQLState.AST, ctx.SchemaLookup, ctx.BoundParams) // Check if literal extraction will be applied needsLiteralExtraction := ctx.ExtractLiterals && @@ -102,17 +102,19 @@ func (t *Transpiler) Transpile(ctx *QueryContext) error { transformations := []Transformation{} ast := ctx.MySQLState.AST var conflictColumns []string + var boundIDs map[int]uint64 ruleApplied := false // Apply ID injection FIRST if needed if needsIDInjection { - newAST, applied, err := t.autoIncRule.ApplyAST(ast, ctx.SchemaLookup) + newAST, applied, ids, err := t.autoIncRule.ApplyAST(ast, ctx.SchemaLookup, ctx.NarrowIDs, ctx.BoundParams) if err != nil { // A rule that refuses a statement must reach the client. Returning // the error unchanged keeps whatever MySQL error code the rule // chose (see transform.CodedError). return err } + boundIDs = ids if applied { ast = newAST transformations = append(transformations, Transformation{ @@ -191,7 +193,7 @@ func (t *Transpiler) Transpile(ctx *QueryContext) error { params = extractedParams } - ctx.Output.Statements = []TranspiledStatement{{SQL: sql, Params: params, ParamOrder: paramOrder}} + ctx.Output.Statements = []TranspiledStatement{{SQL: sql, Params: params, ParamOrder: paramOrder, BoundIDs: boundIDs}} ctx.MySQLState.AST = ast ctx.MySQLState.Transformations = transformations diff --git a/protocol/server.go b/protocol/server.go index c5d24d1d..04c3cd25 100644 --- a/protocol/server.go +++ b/protocol/server.go @@ -213,9 +213,10 @@ type LoadDataHandler interface { // column-name index from that response; a server that reports no columns leaves // strict clients unable to address any column by name, even though the rows // themselves arrive intact. Handlers that cannot describe a statement return no -// columns and the prepare response omits the definitions. +// columns and the prepare response omits the definitions. database is the +// database the statement runs in: the one it names, or the session's. type ResultColumnDescriber interface { - DescribeResultColumns(session *ConnectionSession, sql string) ([]ColumnDef, error) + DescribeResultColumns(session *ConnectionSession, database, sql string) ([]ColumnDef, error) } // SessionCloser is an optional extension for handlers that need to release @@ -238,6 +239,12 @@ type ResultSet struct { CommittedTxnId uint64 // For write forwarding: actual CDC txnID } +// HasResultSet reports whether rs is answered to a client as a result set +// rather than an OK packet: it carries columns or rows. A nil rs has none. +func (rs *ResultSet) HasResultSet() bool { + return rs != nil && (len(rs.Columns) > 0 || len(rs.Rows) > 0) +} + // ColumnDef represents a column definition type ColumnDef struct { Name string @@ -562,7 +569,7 @@ func (s *MySQLServer) processQuery(conn net.Conn, session *ConnectionSession, qu _ = s.writeMySQLErr(conn, 1, err) return } - if rs == nil || (len(rs.Columns) == 0 && len(rs.Rows) == 0) { + if !rs.HasResultSet() { // OK response for non-SELECT (INSERT/UPDATE/DELETE/etc) rowsAffected := int64(0) lastInsertId := int64(0) @@ -1103,8 +1110,16 @@ func (s *MySQLServer) handleStmtPrepare(conn net.Conn, session *ConnectionSessio // Describe the result set now: clients index columns by name from this // response, so omitting the definitions makes every by-name lookup fail. var resultColumns []ColumnDef - if describer, ok := s.handler.(ResultColumnDescriber); ok { - cols, err := describer.DescribeResultColumns(session, transpiledSQL) + if StatementCode(ctx.Output.StatementType) == StatementInformationSchema { + // Marmot answers INFORMATION_SCHEMA itself, with a fixed column set + // per table; the database has no such table to describe. + resultColumns = InformationSchemaColumns(InformationSchemaTableType(ctx.MySQLState.ISTableType)) + } else if describer, ok := s.handler.(ResultColumnDescriber); ok { + database := ctx.Output.Database + if database == "" { + database = session.CurrentDatabase + } + cols, err := describer.DescribeResultColumns(session, database, transpiledSQL) switch { case err == nil: resultColumns = cols @@ -1143,9 +1158,13 @@ func (s *MySQLServer) handleStmtPrepare(conn net.Conn, session *ConnectionSessio defer session.preparedStmtLock.Unlock() stmtID = session.nextStmtID session.nextStmtID++ + // The client's own SQL is kept, not the transpiled text: execution + // runs it through the handler exactly as a text query, and the + // transpiled text has already lost what the handler needs from the + // original, such as a table's database qualifier. session.preparedStmts[stmtID] = &PreparedStatement{ ID: stmtID, - Query: transpiledSQL, + Query: sql, ParamCount: paramCount, OriginalType: StatementCode(ctx.Output.StatementType), Context: ctx, @@ -1402,23 +1421,25 @@ func (s *MySQLServer) handleStmtExecute(conn net.Conn, session *ConnectionSessio Int("param_count", len(params)). Msg("Executing prepared statement") - // Execute the query directly (it's already transpiled SQLite syntax) - // We need to determine if it's a SELECT to know how to format the response - // Use the original statement type we stored during PREPARE - isSelect := stmt.OriginalType == StatementSelect - // Execute query with params passed to handler (no string interpolation!) + // The handler transpiles the client's SQL as it does a text query. rs, err := s.handler.HandleQuery(session, stmt.Query, params) if err != nil { _ = s.writeMySQLErr(conn, 1, err) return } + // A prepared statement answers as its text form does: a result set when + // the answer carries one (INFORMATION_SCHEMA, SHOW and the like), an OK + // packet otherwise. A SELECT always answers with a result set, even an + // empty one. + answersRows := stmt.OriginalType == StatementSelect || rs.HasResultSet() + log.Debug(). Uint64("conn_id", session.ConnID). Uint32("stmt_id", stmtID). Int("original_type", int(stmt.OriginalType)). - Bool("is_select", isSelect). + Bool("answers_rows", answersRows). Int64("rows_affected", func() int64 { if rs != nil { return rs.RowsAffected @@ -1427,9 +1448,7 @@ func (s *MySQLServer) handleStmtExecute(conn net.Conn, session *ConnectionSessio }()). Msg("Prepared statement result") - // Use the original statement type to determine response format - if isSelect { - // Write binary result set for SELECT queries + if answersRows { if rs != nil { _ = s.writeBinaryResultSet(conn, 1, session, rs) } else { diff --git a/protocol/transaction.go b/protocol/transaction.go index 9da98591..3d7fd3ed 100644 --- a/protocol/transaction.go +++ b/protocol/transaction.go @@ -2,6 +2,7 @@ package protocol import ( "fmt" + "slices" "sync" "github.com/maxpert/marmot/common" @@ -197,6 +198,12 @@ type Statement struct { // one of the two sources is in play - see MergeExecParams. ParamOrder []bool `msgpack:"-"` + // BoundIDs are server-generated ids that replace the caller's bound + // values, keyed by the value's position among them: a narrow + // AUTO_INCREMENT placeholder bound to NULL or 0 gets an id exactly as the + // literal would. MergeExecParams binds them in place. + BoundIDs map[int]uint64 `msgpack:"-"` + // ParsedAST carries the Vitess AST produced during MySQL-dialect parse. // Non-nil only when the pipeline parsed the statement via Vitess (SELECT, // DML, and most DDL). Downstream components that need the AST — notably @@ -224,8 +231,17 @@ type Statement struct { // pipeline-extracted ones interleaved in serialization order; ParamOrder // records that order (true = next wireParams value, false = next // ExtractedParams value) so the two sources are threaded back together -// positionally instead of one being silently dropped. +// positionally instead of one being silently dropped. Any BoundIDs replace +// the caller's values at their positions first. func (s Statement) MergeExecParams(wireParams []interface{}) []interface{} { + if len(s.BoundIDs) > 0 { + wireParams = slices.Clone(wireParams) + for pos, id := range s.BoundIDs { + if pos < len(wireParams) { + wireParams[pos] = int64(id) + } + } + } if len(s.ParamOrder) == 0 { if len(wireParams) == 0 && len(s.ExtractedParams) > 0 { return s.ExtractedParams @@ -262,6 +278,7 @@ func (s Statement) WithResolvedParams(sql string, params []interface{}) Statemen s.SQL = sql s.ExtractedParams = params s.ParamOrder = nil + s.BoundIDs = nil return s } diff --git a/replica/database_discovery_test.go b/replica/database_discovery_test.go index cca17c88..91225917 100644 --- a/replica/database_discovery_test.go +++ b/replica/database_discovery_test.go @@ -114,6 +114,22 @@ func (m *mockMarmotServiceClient) ForwardLoadData(ctx context.Context, req *marm return args.Get(0).(*marmotgrpc.ForwardQueryResponse), args.Error(1) } +func (m *mockMarmotServiceClient) GetAutoIncBases(ctx context.Context, req *marmotgrpc.AutoIncBasesRequest, opts ...grpc.CallOption) (*marmotgrpc.AutoIncBasesResponse, error) { + args := m.Called(ctx, req) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*marmotgrpc.AutoIncBasesResponse), args.Error(1) +} + +func (m *mockMarmotServiceClient) SyncAutoIncBases(ctx context.Context, req *marmotgrpc.AutoIncSyncRequest, opts ...grpc.CallOption) (*marmotgrpc.AutoIncSyncResponse, error) { + args := m.Called(ctx, req) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*marmotgrpc.AutoIncSyncResponse), args.Error(1) +} + func (m *mockMarmotServiceClient) GetLoadDataChunk(ctx context.Context, req *marmotgrpc.LoadDataChunkRequest, opts ...grpc.CallOption) (*marmotgrpc.LoadDataChunkResponse, error) { args := m.Called(ctx, req) if args.Get(0) == nil { diff --git a/replica/handler.go b/replica/handler.go index 4415db08..1a4b993c 100644 --- a/replica/handler.go +++ b/replica/handler.go @@ -83,6 +83,7 @@ func (h *ReadOnlyHandler) HandleQuery(session *protocol.ConnectionSession, sql s stmt := protocol.ParseStatementWithOptions(sql, protocol.ParseOptions{ SkipTranspilation: !session.TranspilationEnabled, SchemaProvider: schemaProvider, + BoundParams: params, }) // Handle system variable queries (@@version, DATABASE(), etc.) @@ -101,7 +102,7 @@ func (h *ReadOnlyHandler) HandleQuery(session *protocol.ConnectionSession, sql s // Reject all mutations with MySQL read-only error if protocol.IsMutation(stmt) { if h.forwardWrites { - return h.forwardMutation(session, stmt, params) + return h.forwardMutation(session, sql, stmt, params) } log.Debug(). Uint64("conn_id", session.ConnID). @@ -409,8 +410,11 @@ func (h *ReadOnlyHandler) handleMarmotCommand(session *protocol.ConnectionSessio return nil, fmt.Errorf("unrecognized marmot command: %s", sql) } -// forwardMutation forwards a mutation to the leader -func (h *ReadOnlyHandler) forwardMutation(session *protocol.ConnectionSession, stmt protocol.Statement, params []interface{}) (*protocol.ResultSet, error) { +// forwardMutation forwards a mutation to the leader. It forwards the client's +// own SQL, never stmt.SQL: the leader transpiles the statement exactly once, +// as it would a client's, and the transpiled text has already lost what the +// leader needs from the original, such as a table's database qualifier. +func (h *ReadOnlyHandler) forwardMutation(session *protocol.ConnectionSession, sql string, stmt protocol.Statement, params []interface{}) (*protocol.ResultSet, error) { client := h.replica.streamClient.GetClient() if client == nil { return nil, protocol.NewMySQLError(protocol.ErrCodeUnknown, protocol.SQLStateGeneral, "Not connected to leader") @@ -428,7 +432,7 @@ func (h *ReadOnlyHandler) forwardMutation(session *protocol.ConnectionSession, s SessionId: session.ConnID, RequestId: requestID, Database: session.CurrentDatabase, - Sql: stmt.SQL, + Sql: sql, TxnControl: marmotgrpc.ForwardTxnControl_FWD_TXN_NONE, WaitForReplication: session.WaitForReplication, Params: serializedParams, diff --git a/replica/handler_test.go b/replica/handler_test.go index 64f06000..4801b1a8 100644 --- a/replica/handler_test.go +++ b/replica/handler_test.go @@ -849,6 +849,55 @@ func (f fakeLeaderClient) ForwardLoadData(context.Context, *marmotgrpc.ForwardLo return nil, f.err } +// capturingLeaderClient accepts every forwarded query and records it. +type capturingLeaderClient struct { + marmotgrpc.MarmotServiceClient + got *[]*marmotgrpc.ForwardQueryRequest +} + +func (c capturingLeaderClient) ForwardQuery(_ context.Context, req *marmotgrpc.ForwardQueryRequest, _ ...grpc.CallOption) (*marmotgrpc.ForwardQueryResponse, error) { + *c.got = append(*c.got, req) + return &marmotgrpc.ForwardQueryResponse{Success: true, RowsAffected: 1}, nil +} + +// TestForwardedMutationCarriesTheClientSQL pins R3c-13: a replica forwards the +// client's own SQL, so a qualified write keeps its database through a +// replica, and the leader transpiles it once. Forwarding the transpiled text +// stripped the qualifier, so the leader applied the write in the session's +// database: 1146 there, or a same-named table silently written. +// +// Mutation: forward stmt.SQL again. "the forwarded SQL lost the client's +// qualifier" fires. +func TestForwardedMutationCarriesTheClientSQL(t *testing.T) { + var got []*marmotgrpc.ForwardQueryRequest + h := handlerWithLeaderClient(t, capturingLeaderClient{got: &got}) + session := &protocol.ConnectionSession{ConnID: 1, CurrentDatabase: "testdb", TranspilationEnabled: true} + + for _, tc := range []struct { + sql string + params []interface{} + }{ + {"INSERT INTO other.users (name) VALUES ('x')", nil}, + {"INSERT INTO `other`.`users` (name) VALUES (?)", []interface{}{"y"}}, + {"UPDATE other.users SET name = ? WHERE id = ?", []interface{}{"z", int64(1)}}, + } { + got = nil + _, err := h.HandleQuery(session, tc.sql, tc.params) + if err != nil { + t.Fatalf("%q: %v", tc.sql, err) + } + if len(got) != 1 { + t.Fatalf("%q: forwarded %d requests, want 1", tc.sql, len(got)) + } + if got[0].Sql != tc.sql { + t.Errorf("the forwarded SQL lost the client's qualifier: forwarded %q, client sent %q", got[0].Sql, tc.sql) + } + if got[0].Database != "testdb" { + t.Errorf("%q: forwarded with database %q, want the session's", tc.sql, got[0].Database) + } + } +} + // handlerWithLeaderClient builds a forwarding-enabled handler whose leader link // is the given client. A nil client reproduces "not connected"; a client that // returns an error reproduces "lost connection". Neither needs a live leader. @@ -902,7 +951,7 @@ func TestReplicaEmittedErrorCodesAreServerSide(t *testing.T) { { name: "forwardMutation, not connected", client: nil, call: func(h *ReadOnlyHandler, s *protocol.ConnectionSession) error { - _, err := h.forwardMutation(s, insert, nil) + _, err := h.forwardMutation(s, insert.SQL, insert, nil) return err }, wantCode: protocol.ErrCodeUnknown, wantSQLState: protocol.SQLStateGeneral, @@ -929,7 +978,7 @@ func TestReplicaEmittedErrorCodesAreServerSide(t *testing.T) { { name: "forwardMutation, lost connection", client: fakeLeaderClient{err: errors.New("boom")}, call: func(h *ReadOnlyHandler, s *protocol.ConnectionSession) error { - _, err := h.forwardMutation(s, insert, nil) + _, err := h.forwardMutation(s, insert.SQL, insert, nil) return err }, wantCode: protocol.ErrCodeUnknown, wantSQLState: protocol.SQLStateGeneral, diff --git a/replica/replica.go b/replica/replica.go index 66bffb38..b5a577da 100644 --- a/replica/replica.go +++ b/replica/replica.go @@ -174,7 +174,7 @@ func Run() { // Initialize read-only handler replica.handler = NewReadOnlyHandler(dbMgr, replica.clock, replica) - // Initialize query pipeline (nil ID generator - replicas are read-only) + // Initialize query pipeline (no ID generators - replicas are read-only) if err := protocol.InitializePipeline( cfg.Config.QueryPipeline.TranspilerCacheSize, nil, diff --git a/test/autoinc_backstop_cluster_test.go b/test/autoinc_backstop_cluster_test.go new file mode 100644 index 00000000..64c3b447 --- /dev/null +++ b/test/autoinc_backstop_cluster_test.go @@ -0,0 +1,128 @@ +package test + +import ( + "fmt" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// Log lines a claim COMMIT that could not be applied leaves: the participant's +// refusal (db ReplicationEngine.Commit) and, on the claimant, a local commit +// that failed after the remote quorum committed (coordinator +// commitLocalAfterRemoteQuorum). +const ( + claimCommitRefusedLog = "COMMIT REFUSED: auto-increment claim could not be applied" + localCommitFailedLog = "Local commit failed after remote quorum achieved" +) + +// TestNarrowAutoInc_ClaimsCommitUnderConstantBaseSync is reviewer rr-s3cf2's +// H1 hunt, amplified without touching the backstop's interval: while every +// node inserts into two SMALLINT tables (a claim every 31 ids), the test runs +// the operator's sync command back to back on every node, so each node pulls +// its peers' bases many times a second. A peer that committed a claim first +// reports exactly its end; a sync landing between this node's PREPARE and +// COMMIT of that claim used to raise the stored base past the claim's newBase +// and refuse the COMMIT, leaving the claim key locked until the +// pending-transaction GC. Every claim COMMIT must now apply, and no id may be +// acknowledged twice. +// +// Mutation: test the merged floor in applyClaimTx's condition (or write merges +// to the committed floor). "claim COMMITs were refused" fires. +func TestNarrowAutoInc_ClaimsCommitUnderConstantBaseSync(t *testing.T) { + tables := []string{"bs1", "bs2"} + harness := startNarrowCluster(t, tables[0], "CREATE TABLE "+tables[0]+" (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + defer harness.Cleanup() + if _, err := execTimed(openTimedNodeDatabase(t, harness, 1, "marmot"), + "CREATE TABLE "+tables[1]+" (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)"); err != nil { + t.Fatalf("CREATE TABLE %s: %v", tables[1], err) + } + if err := harness.WaitForTableExists(tables[1], []int{1, 2, 3}, 30*time.Second); err != nil { + t.Fatal(err) + } + + const load = 45 * time.Second + stop := time.Now().Add(load) + var syncs, syncErrs atomic.Int64 + var wg sync.WaitGroup + for n := 1; n <= 3; n++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + for time.Now().Before(stop) { + if _, err := runAutoIncSync(harness, n); err != nil { + syncErrs.Add(1) + time.Sleep(100 * time.Millisecond) + continue + } + syncs.Add(1) + } + }(n) + } + + var mu sync.Mutex + ledger := map[string]string{} + var dups []string + inserted := make([]int, 4) + failures := make([]error, 4) + for n := 1; n <= 3; n++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + conn := openTimedNodeDatabase(t, harness, n, "marmot") + for i := 0; time.Now().Before(stop); i++ { + table := tables[i%len(tables)] + who := fmt.Sprintf("node %d #%d", n, i) + id, err := insertNarrowWithin(conn, 30*time.Second, "INSERT INTO "+table+" (v) VALUES (?)", who) + mu.Lock() + if err != nil { + failures[n] = err + mu.Unlock() + return + } + inserted[n]++ + key := fmt.Sprintf("%s/%d", table, id) + if prev, dup := ledger[key]; dup { + dups = append(dups, fmt.Sprintf("%s: %s and %s", key, prev, who)) + } + ledger[key] = who + mu.Unlock() + } + }(n) + } + wg.Wait() + + t.Logf("syncs=%d sync errors=%d inserted per node=%v ids=%d", syncs.Load(), syncErrs.Load(), inserted[1:], len(ledger)) + if len(dups) > 0 { + t.Fatalf("DUPLICATE ids: %v", dups) + } + var refusals []string + for n := 1; n <= 3; n++ { + logText, err := os.ReadFile(harness.Nodes[n-1].LogFile) + if err != nil { + t.Fatalf("read node %d log: %v", n, err) + } + refused := strings.Count(string(logText), claimCommitRefusedLog) + failed := strings.Count(string(logText), localCommitFailedLog) + t.Logf("node %d: commit-refused=%d local-commit-failed=%d", n, refused, failed) + if refused+failed > 0 { + refusals = append(refusals, fmt.Sprintf("node %d: %d refused, %d local commits failed", n, refused, failed)) + } + } + if len(refusals) > 0 { + harness.dumpNodeLogs("base_sync_refusals") + t.Fatalf("claim COMMITs were refused under constant base syncs: %v", refusals) + } + for n := 1; n <= 3; n++ { + if failures[n] != nil { + harness.dumpNodeLogs("base_sync_inserts") + t.Fatalf("node %d stopped inserting after %d rows: %v", n, inserted[n], failures[n]) + } + } + if syncs.Load() < 100 { + t.Fatalf("only %d syncs ran in %s; the hunt did not amplify the backstop", syncs.Load(), load) + } +} diff --git a/test/autoinc_claim_cluster_test.go b/test/autoinc_claim_cluster_test.go index 7ff074d4..d42c4d7d 100644 --- a/test/autoinc_claim_cluster_test.go +++ b/test/autoinc_claim_cluster_test.go @@ -6,26 +6,15 @@ package test // over the real MySQL wire protocol against a 3-node ClusterHarness // (test/crash_recovery_test.go). // -// coordinator.WriteCoordinator.ClaimRange itself has no caller reachable -// over the MySQL wire protocol or any admin endpoint yet - the range -// allocator that will wire an INSERT against a narrow AUTO_INCREMENT table -// to ClaimRange has not landed. Its only callers today are in-process Go -// tests (coordinator/autoinc_claim_test.go). This ClusterHarness only talks -// to a node over its MySQL listener (database/sql + the MySQL wire -// protocol) and a small JSON admin API for gossip membership - neither can -// drive a claim - so the claim-protocol scenarios (sequential claim safety, -// concurrent boundary claims, restart and crash recovery, meta-store wipe, -// full-cluster snapshot restore) are covered instead in -// test/autoinc_claim_inprocess_test.go, which drives the real coordinator -// and participant engines directly, in-process, replacing only the gRPC -// transport. Standing up an ad hoc coordinator inside test/ that dials each -// node's gRPC PrepareTransaction/CommitTransaction RPCs directly was -// considered and rejected: WriteCoordinator.localReplicator is written -// assuming in-process co-location with the participant it calls "local" -// (coordinator/write_coordinator.go dispatches with wc.nodeID against it), -// so reproducing it faithfully from outside the process would mean -// re-deriving quorum membership, retry and rejection-base bookkeeping that -// belongs to the coordinator itself, not to a test fixture. +// The claim protocol's own scenarios (sequential claim safety, concurrent +// boundary claims, restart and crash recovery, meta-store wipe, full-cluster +// snapshot restore, membership growth) are covered in +// test/autoinc_claim_inprocess_test.go and +// test/autoinc_membership_inprocess_test.go, which drive the real coordinator +// and participant engines in-process, replacing only the gRPC transport, so +// they can script which node sees which round. Inserts that reach +// ClaimRange over the MySQL wire protocol are covered in +// test/autoinc_narrow_cluster_test.go. import ( "database/sql" @@ -53,8 +42,8 @@ func systemDBFile(dataDir string) string { } // readAutoIncBase opens a node's system database file directly, outside the -// MySQL wire protocol, and reads the committed base for (database, table) -// from __marmot__autoinc. This is the only channel available to a cluster +// MySQL wire protocol, and reads the base for (database, table) - the +// largest of its committed, seed and merged floors - from __marmot__autoinc. This is the only channel available to a cluster // test: the table is deliberately hidden from client SQL (see // common.IsInternalTableName and protocol/query/rules/claim_table_guard.go), // and nothing on the wire path reveals it. found is false, with no error, @@ -71,7 +60,7 @@ func readAutoIncBase(dataDir, database, table string) (base uint64, found bool, var got int64 scanErr := conn.QueryRow( - "SELECT base FROM "+common.AutoIncClaimTableName+" WHERE db = ? AND tbl = ?", + "SELECT MAX(committed, seed, merged) FROM "+common.AutoIncClaimTableName+" WHERE db = ? AND tbl = ?", database, table).Scan(&got) switch { case scanErr == sql.ErrNoRows: @@ -215,13 +204,14 @@ func TestAutoIncSeed_CreateTableWithFloorOption(t *testing.T) { } } -// TestAutoIncSeed_AlterSeedsFromExistingMax exercises DDL-time seeding's -// derivation from a table's actual MAX(id), not just a declared floor -// option: rows carrying explicit ids are inserted directly (DML never seeds -// - only a DDL that changes the table's CREATE text does, db/autoinc_seed.go), -// so the claim base must still reflect CREATE TABLE's own empty-table seed -// right up until an unrelated ALTER TABLE ADD COLUMN re-triggers seeding and -// picks up MAX(id) fresh. +// TestAutoIncSeed_AlterSeedsFromExistingMax: a table filled with explicit +// ids 1..rowCount holds a base at or above rowCount on every node, both +// before and after an unrelated ALTER TABLE ADD COLUMN re-triggers DDL-time +// seeding. The explicit ids raise the base as they are inserted (the range +// allocator observes every explicit id), and the ALTER's seed from MAX(id) +// can only raise it further. That the seed itself derives from MAX(id) is +// pinned without the allocator in the way by +// db/autoinc_seed_test.go TestSeedAutoIncBasesForDDL_ExistingRowsRaiseTheFloor. func TestAutoIncSeed_AlterSeedsFromExistingMax(t *testing.T) { harness := NewClusterHarness(t) defer harness.Cleanup() @@ -261,9 +251,9 @@ func TestAutoIncSeed_AlterSeedsFromExistingMax(t *testing.T) { t.Fatalf("explicit-id rows did not replicate: %v", err) } - // Before the ALTER: proves the assertion below is the ALTER's own - // seeding at work, not a base that was already >= rowCount for some - // other reason (e.g. DML itself seeding, which it must not). + // Before the ALTER: explicit ids above the base raise it through the + // claim protocol as they are inserted (id.RangeAllocator.Observe), so no + // node can later issue one of them. preDataDir := harness.Nodes[0].DataDir preBase, found, err := readAutoIncBase(preDataDir, db.DefaultDatabaseName, table) if err != nil { @@ -272,8 +262,8 @@ func TestAutoIncSeed_AlterSeedsFromExistingMax(t *testing.T) { if !found { t.Fatalf("no claim row for %s before ALTER", table) } - if preBase >= rowCount { - t.Fatalf("pre-ALTER base = %d, want < %d (DML must not have seeded the claim row)", preBase, rowCount) + if preBase < rowCount { + t.Fatalf("pre-ALTER base = %d, want >= %d: explicit ids must raise the base as they are inserted", preBase, rowCount) } if _, err := harness.ExecNode(1, fmt.Sprintf("ALTER TABLE %s ADD COLUMN w INT", table)); err != nil { @@ -283,7 +273,7 @@ func TestAutoIncSeed_AlterSeedsFromExistingMax(t *testing.T) { for _, nodeID := range allNodes { dataDir := harness.Nodes[nodeID-1].DataDir base := waitForAutoIncBaseAtLeast(t, dataDir, db.DefaultDatabaseName, table, rowCount, 15*time.Second) - t.Logf("node %d: ALTER seeded base = %d (want >= %d)", nodeID, base, rowCount) + t.Logf("node %d: base after ALTER = %d (want >= %d)", nodeID, base, rowCount) } } @@ -341,22 +331,18 @@ func TestAutoIncSeed_AlterFloorWithoutAColumnIsRefused(t *testing.T) { } } -// TestAutoIncSeed_RefusesWhenExistingMaxExceedsWidth: a TINYINT -// AUTO_INCREMENT column (widthMax = 127, intmarker.Attributes.WidthMax) whose -// table already holds a row above that ceiling must refuse the next DDL that -// touches that table's schema, with MySQL error 1264 -// (ER_WARN_DATA_OUT_OF_RANGE) / SQLSTATE 22003 -// (db/ddl_prepare_validate.go checkAutoIncWidthCeilings, -// AutoIncWidthExceededError). -// -// SQLite does not enforce MySQL's column width, so INSERT INTO ... (id, ...) -// VALUES (200, ...) against a TINYINT column succeeds; the refusal has to -// come from a later DDL. checkAutoIncWidthCeilings scopes to every table -// whose sqlite_master CREATE TABLE text this DDL statement changed - not -// only DDL that redeclares AUTO_INCREMENT - so an unrelated ADD COLUMN on -// the same table is enough to re-trigger the check and is the shape used -// here. -func TestAutoIncSeed_RefusesWhenExistingMaxExceedsWidth(t *testing.T) { +// TestAutoIncSeed_RefusesAnIDBeyondTheWidth: a TINYINT AUTO_INCREMENT column +// (widthMax = 127, intmarker.Attributes.WidthMax) refuses an explicit id +// above its ceiling at INSERT, with MySQL error 1264 +// (ER_WARN_DATA_OUT_OF_RANGE) / SQLSTATE 22003, as MySQL does in strict +// mode. SQLite would store it; letting it in would leave the table holding an +// id its declared width cannot, so it is refused before it reaches SQLite +// (protocol/query/rules/autoincrement_id.go). A DDL that declares a width the +// existing data does not fit is refused with the same code at PREPARE +// (db/ddl_prepare_validate.go checkAutoIncWidthCeilings), pinned by +// db/ddl_prepare_validate_autoinc_code_test.go and +// grpc/autoinc_claim_handler_test.go. +func TestAutoIncSeed_RefusesAnIDBeyondTheWidth(t *testing.T) { harness := NewClusterHarness(t) defer harness.Cleanup() @@ -376,23 +362,17 @@ func TestAutoIncSeed_RefusesWhenExistingMaxExceedsWidth(t *testing.T) { t.Fatalf("DDL did not replicate: %v", err) } - if _, err := harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, v) VALUES (200, 'beyond_tinyint')", table)); err != nil { - t.Fatalf("explicit-id INSERT beyond widthMax: %v", err) - } - if err := harness.WaitForRowCount(table, allNodes, 1, 15*time.Second); err != nil { - t.Fatalf("row did not replicate: %v", err) - } - - _, err := harness.ExecNode(1, fmt.Sprintf("ALTER TABLE %s ADD COLUMN extra INT", table)) - mysqlErr := asMySQLError(t, err, "ALTER TABLE ADD COLUMN on a table whose MAX(id) exceeds widthMax") - + _, err := harness.ExecNode(1, fmt.Sprintf("INSERT INTO %s (id, v) VALUES (200, 'beyond_tinyint')", table)) + mysqlErr := asMySQLError(t, err, "explicit-id INSERT beyond widthMax") if mysqlErr.Number != mysqlcode.ErrCodeDataOutOfRange { t.Errorf("error code = %d, want %d (ER_WARN_DATA_OUT_OF_RANGE)", mysqlErr.Number, mysqlcode.ErrCodeDataOutOfRange) } if got := string(mysqlErr.SQLState[:]); got != mysqlcode.SQLStateDataOutOfRange { t.Errorf("SQLSTATE = %q, want %q", got, mysqlcode.SQLStateDataOutOfRange) } + if n := harness.getRowCount(1, table); n != 0 { + t.Errorf("the refused row reached the table: %d rows", n) + } t.Logf("refused as required: %v", mysqlErr) } @@ -485,8 +465,8 @@ func TestAutoIncHiddenTable_RejectsClientWrites(t *testing.T) { table := common.AutoIncClaimTableName statements := map[string]string{ "INSERT": fmt.Sprintf( - "INSERT INTO %s (db, tbl, base, owner, granted_at) VALUES ('marmot', 'x', 1, 1, 1)", table), - "UPDATE": fmt.Sprintf("UPDATE %s SET base = 999 WHERE tbl = 'x'", table), + "INSERT INTO %s (db, tbl, committed, owner, granted_at) VALUES ('marmot', 'x', 1, 1, 1)", table), + "UPDATE": fmt.Sprintf("UPDATE %s SET committed = 999 WHERE tbl = 'x'", table), "DELETE": fmt.Sprintf("DELETE FROM %s WHERE tbl = 'x'", table), "DROP": fmt.Sprintf("DROP TABLE %s", table), } diff --git a/test/autoinc_claim_inprocess_test.go b/test/autoinc_claim_inprocess_test.go index ca32dc9d..7dc17ed3 100644 --- a/test/autoinc_claim_inprocess_test.go +++ b/test/autoinc_claim_inprocess_test.go @@ -23,6 +23,7 @@ import ( "github.com/maxpert/marmot/db" "github.com/maxpert/marmot/db/snapshot" "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/id" "github.com/maxpert/marmot/protocol" "github.com/stretchr/testify/require" ) @@ -65,15 +66,25 @@ type fanoutReplicator struct { mu sync.Mutex nodes map[uint64]*db.LocalReplicator prepareCalls map[uint64][]prepareRecord + unreachable map[uint64]bool } func newFanoutReplicator() *fanoutReplicator { return &fanoutReplicator{ nodes: make(map[uint64]*db.LocalReplicator), prepareCalls: make(map[uint64][]prepareRecord), + unreachable: make(map[uint64]bool), } } +// setUnreachable makes every call to nodeID fail as a transport error, so the +// node misses whatever the cluster decides meanwhile. +func (f *fanoutReplicator) setUnreachable(nodeID uint64, unreachable bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.unreachable[nodeID] = unreachable +} + func (f *fanoutReplicator) register(nodeID uint64, lr *db.LocalReplicator) { f.mu.Lock() defer f.mu.Unlock() @@ -83,7 +94,11 @@ func (f *fanoutReplicator) register(nodeID uint64, lr *db.LocalReplicator) { func (f *fanoutReplicator) ReplicateTransaction(ctx context.Context, nodeID uint64, req *coordinator.ReplicationRequest) (*coordinator.ReplicationResponse, error) { f.mu.Lock() lr := f.nodes[nodeID] + unreachable := f.unreachable[nodeID] f.mu.Unlock() + if unreachable { + return nil, fmt.Errorf("node %d unreachable", nodeID) + } resp, err := lr.ReplicateTransaction(ctx, nodeID, req) @@ -117,26 +132,39 @@ func (f *fanoutReplicator) prepareCallsSince(nodeID uint64, since int) []prepare } // fixedNodeProvider implements coordinator.NodeProvider (coordinator/node_provider.go:6) -// with a static, never-mutated node set - this fixture never simulates -// membership changes, only claim-protocol behaviour, so the interface's -// four methods are all that is needed. HasSeedNodes is false: this is a +// with a node set that changes only when a test grows the cluster +// (setNodes); the interface's four methods are all that is needed. HasSeedNodes is false: this is a // legitimate fixed-size deployment, not a node still learning its peers // (coordinator/cluster.go:90's quorum-of-one guard), matching // coordinator's own mockNodeProvider zero value // (coordinator/full_replication_test.go:15-18). type fixedNodeProvider struct { + mu sync.Mutex nodes []uint64 } func (p *fixedNodeProvider) GetAliveNodes() ([]uint64, error) { - out := make([]uint64, len(p.nodes)) - copy(out, p.nodes) - return out, nil + p.mu.Lock() + defer p.mu.Unlock() + return append([]uint64(nil), p.nodes...), nil } -func (p *fixedNodeProvider) GetClusterSize() int { return len(p.nodes) } -func (p *fixedNodeProvider) GetTotalMembershipSize() int { return len(p.nodes) } -func (p *fixedNodeProvider) HasSeedNodes() bool { return false } +func (p *fixedNodeProvider) GetClusterSize() int { return p.GetTotalMembershipSize() } + +func (p *fixedNodeProvider) GetTotalMembershipSize() int { + p.mu.Lock() + defer p.mu.Unlock() + return len(p.nodes) +} + +func (p *fixedNodeProvider) HasSeedNodes() bool { return false } + +// setNodes replaces the membership every node sees. +func (p *fixedNodeProvider) setNodes(nodes []uint64) { + p.mu.Lock() + defer p.mu.Unlock() + p.nodes = append([]uint64(nil), nodes...) +} // inprocNode is one participant: its own data directory, clock, DatabaseManager, // LocalReplicator (db/local_replicator.go:20, which owns the real @@ -183,6 +211,7 @@ func newInprocCluster(t *testing.T, nodeIDs []uint64) *inprocCluster { for _, id := range nodeIDs { c.nodes[id] = c.buildNode(id, t.TempDir()) } + c.releaseNewCluster() t.Cleanup(func() { for _, n := range c.nodes { _ = n.dm.Close() @@ -196,12 +225,33 @@ func (c *inprocCluster) buildNode(id uint64, dataDir string) *inprocNode { clock := hlc.NewClock(id) dm, err := db.NewDatabaseManager(dataDir, id, clock) require.NoError(c.t, err) + // Every node counts the cluster exactly as every claimant does. + dm.SetClusterMembership(c.provider.GetTotalMembershipSize) lr := db.NewLocalReplicator(id, dm, clock) wc := coordinator.NewWriteCoordinator(id, c.provider, c.fanout, lr, inprocCoordinatorTimeout, clock) c.fanout.register(id, lr) return &inprocNode{id: id, dataDir: dataDir, clock: clock, dm: dm, lr: lr, wc: wc} } +// releaseNewCluster releases the votes of a cluster whose every node is held +// because it initialised its system database: once every member answers +// held, each releases with no bases (grpc.autoIncMergeSafe, all-held case), +// which is what the merge loop does in a node process. A cluster reopened +// over existing claim history has no held node and is left alone. +func (c *inprocCluster) releaseNewCluster() { + c.t.Helper() + for _, n := range c.nodes { + held, err := n.dm.AutoIncVotesHeld() + require.NoError(c.t, err) + if !held { + return + } + } + for _, n := range c.nodes { + require.NoError(c.t, n.dm.MergeAutoIncBasesAndReleaseVotes(nil)) + } +} + // closeNode stops a node's DatabaseManager without reopening it. func (c *inprocCluster) closeNode(id uint64) { c.t.Helper() @@ -257,6 +307,7 @@ func newInprocClusterFromDirs(t *testing.T, dataDirs map[uint64]string) *inprocC for _, id := range nodeIDs { c.nodes[id] = c.buildNode(id, dataDirs[id]) } + c.releaseNewCluster() t.Cleanup(func() { for _, n := range c.nodes { _ = n.dm.Close() @@ -391,7 +442,7 @@ func setupClaimCluster(t *testing.T, database, table string) *inprocCluster { for _, id := range []uint64{1, 2, 3} { require.NoError(t, c.nodes[id].dm.CreateDatabase(database)) } - assertAbsentClaimRowRejected(t, c, database, table+"_absent") + assertAbsentClaimRowBackfilled(t, c, database, table+"_absent") assertWidthCeilingRejected(t, c, database, table+"_narrow") driveDDL(t, c.nodes[1], database, table, sprintfDDL(table)) for _, id := range []uint64{1, 2, 3} { @@ -410,7 +461,7 @@ const narrowMarkedTableDDL = `CREATE TABLE %s (id INTEGER /*M:8a*/ PRIMARY KEY, // asserts a claim whose range exceeds that ceiling is rejected. // // This is run ahead of every one of the five tests for the same reason -// assertAbsentClaimRowRejected is: every one of prepareAutoIncClaim's +// assertAbsentClaimRowBackfilled is: every one of prepareAutoIncClaim's // rejection clauses needs at least one assertion that isolates it, or a // mutation to that specific clause can slip through five tests whose own // claim sizes never approach the (very large, 32-bit) ceiling their main @@ -429,7 +480,7 @@ func assertWidthCeilingRejected(t *testing.T, c *inprocCluster, database, table } mark2 := c.fanout.prepareCallCount(2) - _, _, err := c.nodes[1].wc.ClaimRange(context.Background(), database, table, 0, 200) + _, _, err := c.nodes[1].wc.ClaimRange(context.Background(), database, table, 0, fixedClaimSize(200)) require.Error(t, err, "a claim whose range exceeds the column's width ceiling must be rejected") round1Node2 := c.fanout.prepareCallsSince(2, mark2) @@ -444,46 +495,51 @@ func sprintfDDL(table string) string { return strings.Replace(markedTableDDL, "%s", table, 1) } -// assertAbsentClaimRowRejected creates `table` directly on every node's own -// SQLite connection - bypassing the coordinator's replicated DDL and its -// automatic claim-row seeding (db/autoinc_seed.go's -// seedAutoIncBasesForDDL) - so the table has a valid, width-resolvable -// schema on every node but NO claim row anywhere, then asserts a claim -// against it is rejected. +// assertAbsentClaimRowBackfilled creates `table` directly on every node's +// own SQLite connection - bypassing the coordinator's replicated DDL and its +// automatic claim-row seeding (db/autoinc_seed.go's seedAutoIncBasesForDDL) +// - with ids 1..5 already in it, so the table has a valid, width-resolvable +// schema on every node but NO claim row anywhere. A claim proposing base 0 +// must then be rejected with the base each participant backfills from its +// own table (5), and the retry above it granted. // -// This isolates the absent-row rejection -// (db/replication_engine.go:337-343, db.ErrAutoIncBaseAbsent's doc at -// db/autoinc_claim.go:90-97) from the width check that runs right after it: -// with a real schema present, AutoIncWidthMax succeeds regardless, so only -// the absent-row check itself stands between this claim and acceptance. It -// is run ahead of every one of the five tests (setupClaimCluster is their -// shared entry point) because "the table has not been created yet" is a -// precondition every one of them briefly passes through. +// It is run ahead of every one of the five tests (setupClaimCluster is their +// shared entry point) because "the claim row does not exist yet" is a state +// a table created before claim rows existed passes through. // // Mutation: a regression that treats an absent claim row as base 0 instead -// of rejecting would make prepareAutoIncClaim fall through here - with a -// real schema present, nothing else in the condition would stop the claim, -// so it would be wrongly accepted where this assertion expects rejection. -func assertAbsentClaimRowRejected(t *testing.T, c *inprocCluster, database, table string) { +// of backfilling it would accept the round-1 proposal at base 0, over ids +// 1..5, where this assertion expects a rejection carrying base 5. +func assertAbsentClaimRowBackfilled(t *testing.T, c *inprocCluster, database, table string) { t.Helper() for _, id := range []uint64{1, 2, 3} { mdb, err := c.nodes[id].dm.GetDatabase(database) require.NoError(t, err) _, err = mdb.GetWriteDB().Exec(sprintfDDL(table)) require.NoError(t, err) + for row := 1; row <= 5; row++ { + _, err = mdb.GetWriteDB().Exec(fmt.Sprintf("INSERT INTO %s (id, v) VALUES (?, 'x')", table), row) + require.NoError(t, err) + } require.NoError(t, mdb.ReloadSchema()) } mark2 := c.fanout.prepareCallCount(2) - _, _, err := c.nodes[1].wc.ClaimRange(context.Background(), database, table, 0, 1) - require.Error(t, err, "a claim against a table with a real schema but no claim row must be rejected, not silently accepted as base 0") + newBase, _, err := c.nodes[1].wc.ClaimRange(context.Background(), database, table, 0, fixedClaimSize(1)) + // The vote is checked before the claim's outcome: a participant that read + // the absent row as 0 would accept here, and only a later COMMIT would + // then fail, which must not be what reports it. round1Node2 := c.fanout.prepareCallsSince(2, mark2) require.NotEmpty(t, round1Node2) - require.False(t, round1Node2[0].resp.Success) + require.False(t, round1Node2[0].resp.Success, + "a claim against a table with a real schema but no claim row must never be accepted as base 0") require.True(t, round1Node2[0].resp.Rejected) - require.Contains(t, round1Node2[0].resp.Error, "no auto-increment claim row", - "rejection must name the absent-row cause, not some other failure") + require.Equal(t, uint64(5), round1Node2[0].resp.AutoIDStoredBase, + "the rejection must carry the base backfilled from the table's own ids") + + require.NoError(t, err, "a claim against a table with no claim row must backfill and be granted above the table's ids") + require.Equal(t, uint64(5), newBase, "the range must start above the ids the table already holds") } // --------------------------------------------------------------------- @@ -514,7 +570,7 @@ func TestClaimRange_SequentialSafety(t *testing.T) { c := setupClaimCluster(t, "testdb", "seqsafety") // Node 1 claims [1,100]. - newBase1, granted1, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "seqsafety", 0, 100) + newBase1, granted1, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "seqsafety", 0, fixedClaimSize(100)) require.NoError(t, err) require.Equal(t, uint64(0), newBase1) require.Equal(t, uint64(100), granted1) @@ -525,7 +581,7 @@ func TestClaimRange_SequentialSafety(t *testing.T) { // Node 2 proposes the SAME stale prevBase=0, unaware of node 1's claim. mark1 := c.fanout.prepareCallCount(1) mark3 := c.fanout.prepareCallCount(3) - newBase2, granted2, err := c.nodes[2].wc.ClaimRange(context.Background(), "testdb", "seqsafety", 0, 50) + newBase2, granted2, err := c.nodes[2].wc.ClaimRange(context.Background(), "testdb", "seqsafety", 0, fixedClaimSize(50)) require.NoError(t, err, "node 2's retry must still succeed after the stale round is rejected") require.Equal(t, uint64(100), newBase2, "node 2 must retry starting at node 1's committed base, not spin on 0") require.Equal(t, uint64(50), granted2) @@ -600,7 +656,7 @@ const maxLockWaitClientRetries = 10 // that signal; this helper stands in for that caller. func claimRangeRetryingOnLockWait(ctx context.Context, node *inprocNode, database, table string, prevBase, size uint64) (newBase, granted uint64, err error) { for attempt := 0; attempt < maxLockWaitClientRetries; attempt++ { - newBase, granted, err = node.wc.ClaimRange(ctx, database, table, prevBase, size) + newBase, granted, err = node.wc.ClaimRange(ctx, database, table, prevBase, fixedClaimSize(size)) if err == nil { return newBase, granted, nil } @@ -715,7 +771,7 @@ func TestClaimRange_ConcurrentBoundary(t *testing.T) { func TestClaimRange_RestartRecoversCommittedBase(t *testing.T) { c := setupClaimCluster(t, "testdb", "restart") - newBase, granted, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "restart", 0, 40) + newBase, granted, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "restart", 0, fixedClaimSize(40)) require.NoError(t, err) require.Equal(t, uint64(0), newBase) require.Equal(t, uint64(40), granted) @@ -733,7 +789,7 @@ func TestClaimRange_RestartRecoversCommittedBase(t *testing.T) { // rejected (including by node 3, over the rewired fanout) and retry above // 40 - never reissue [1,40]. mark3 := c.fanout.prepareCallCount(3) - newBase2, granted2, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "restart", 0, 25) + newBase2, granted2, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "restart", 0, fixedClaimSize(25)) require.NoError(t, err) require.Equal(t, uint64(40), newBase2, "subsequent claim must start above the base node 3 recovered on reopen") require.Equal(t, uint64(25), granted2) @@ -860,7 +916,7 @@ func TestClaimRange_KillNineRecoversCommittedBase(t *testing.T) { // A stale claim (prevBase=0) must be rejected and retried above the // recovered base, not reissue [1,20]. - newBase2, granted2, err := c.nodes[1].wc.ClaimRange(context.Background(), killNineDatabase, killNineTable, 0, 15) + newBase2, granted2, err := c.nodes[1].wc.ClaimRange(context.Background(), killNineDatabase, killNineTable, 0, fixedClaimSize(15)) require.NoError(t, err) require.Equal(t, newBase+granted, newBase2, "post-kill claim must start above the recovered base, not reissue the same range") require.Equal(t, uint64(15), granted2) @@ -889,7 +945,7 @@ func TestClaimRangeKillNineChild(t *testing.T) { waitForBase(t, c.nodes[id], killNineDatabase, killNineTable, 0) } - newBase, granted, err := c.nodes[1].wc.ClaimRange(context.Background(), killNineDatabase, killNineTable, 0, killNineClaimSize) + newBase, granted, err := c.nodes[1].wc.ClaimRange(context.Background(), killNineDatabase, killNineTable, 0, fixedClaimSize(killNineClaimSize)) require.NoError(t, err) for _, id := range []uint64{1, 2, 3} { waitForBase(t, c.nodes[id], killNineDatabase, killNineTable, newBase+granted) @@ -933,7 +989,7 @@ func metaPebbleDirFor(dbFilePath string) string { func TestClaimRange_SurvivesMetaStoreWipe(t *testing.T) { c := setupClaimCluster(t, "testdb", "metawipe") - newBase, granted, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "metawipe", 0, 10) + newBase, granted, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "metawipe", 0, fixedClaimSize(10)) require.NoError(t, err) require.Equal(t, uint64(0), newBase) require.Equal(t, uint64(10), granted) @@ -965,7 +1021,7 @@ func TestClaimRange_SurvivesMetaStoreWipe(t *testing.T) { // A stale proposal from node 2 must be rejected BY NODE 3 specifically, // carrying its real base. mark3 := c.fanout.prepareCallCount(3) - newBase2, granted2, err := c.nodes[2].wc.ClaimRange(context.Background(), "testdb", "metawipe", 0, 5) + newBase2, granted2, err := c.nodes[2].wc.ClaimRange(context.Background(), "testdb", "metawipe", 0, fixedClaimSize(5)) require.NoError(t, err) require.Equal(t, uint64(10), newBase2) require.Equal(t, uint64(5), granted2) @@ -1006,7 +1062,7 @@ func TestClaimRange_SurvivesMetaStoreWipe(t *testing.T) { func TestClaimRange_SurvivesFullClusterSnapshotRestore(t *testing.T) { c := setupClaimCluster(t, "testdb", "snaprestore") - newBase, granted, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "snaprestore", 0, 20) + newBase, granted, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "snaprestore", 0, fixedClaimSize(20)) require.NoError(t, err) require.Equal(t, uint64(0), newBase) require.Equal(t, uint64(20), granted) @@ -1039,8 +1095,13 @@ func TestClaimRange_SurvivesFullClusterSnapshotRestore(t *testing.T) { // Restore each node's files into its fresh, empty directory through the // SAME production restorer a real node uses to apply a snapshot - // (db/snapshot/restorer.go), including its system-database merge hook - - // not a fixture-only file copy. The manager is fully closed at this + // (db/snapshot/restorer.go) - not a fixture-only file copy. Each node gets + // back its OWN snapshot, its own claim history, so the restorer runs + // without the system-database merge hook: that hook belongs to catching + // up from a PEER (grpc/catch_up.go), and there it holds a node with no + // local system database until it merges bases from a majority + // (db.AutoIncHoldTable) - which in a cluster where every node was rebuilt + // that way nobody could ever supply. The manager is fully closed at this // point, not live, exactly like a node coming up against a freshly // provisioned data directory, so no ConnectionManager is needed. for _, id := range []uint64{1, 2, 3} { @@ -1055,7 +1116,6 @@ func TestClaimRange_SurvivesFullClusterSnapshotRestore(t *testing.T) { }) } r := snapshot.NewRestorer(dataDir, nil) - r.SetSystemDBMerge(db.RaiseAutoIncBasesFrom) require.NoError(t, r.RestoreFiles(snapDirByNode[id], files)) } @@ -1072,7 +1132,7 @@ func TestClaimRange_SurvivesFullClusterSnapshotRestore(t *testing.T) { // A stale claim, from node 1, must retry above the restored base rather // than reissue [1,20]. - newBase2, granted2, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "snaprestore", 0, 15) + newBase2, granted2, err := c.nodes[1].wc.ClaimRange(context.Background(), "testdb", "snaprestore", 0, fixedClaimSize(15)) require.NoError(t, err) require.Equal(t, uint64(20), newBase2, "post-restore claim must start above the restored base, not reissue [1,20]") require.Equal(t, uint64(15), granted2) @@ -1081,3 +1141,9 @@ func TestClaimRange_SurvivesFullClusterSnapshotRestore(t *testing.T) { waitForBase(t, c.nodes[id], "testdb", "snaprestore", 35) } } + +// fixedClaimSize is a RangeSizer asking for the same size above any base: +// these tests pin the claim protocol, not the allocator's sizing policy. +func fixedClaimSize(size uint64) id.RangeSizer { + return func(uint64) (uint64, error) { return size, nil } +} diff --git a/test/autoinc_lifecycle_cluster_test.go b/test/autoinc_lifecycle_cluster_test.go new file mode 100644 index 00000000..5e93de7a --- /dev/null +++ b/test/autoinc_lifecycle_cluster_test.go @@ -0,0 +1,344 @@ +package test + +// Cluster tests for the lifecycle of narrow AUTO_INCREMENT ranges across DDL, +// rebuilt nodes and the client shapes that reach the allocator only through +// bound values or a qualified table, over the real MySQL wire protocol +// against a 3-node ClusterHarness (test/crash_recovery_test.go). + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// openNodeDatabase connects to database on nodeID. +func openNodeDatabase(t *testing.T, harness *ClusterHarness, nodeID int, database string) *sql.DB { + t.Helper() + conn, err := sql.Open("mysql", fmt.Sprintf("root:@tcp(localhost:%d)/%s", harness.Nodes[nodeID-1].MySQLPort, database)) + if err != nil { + t.Fatalf("open %s on node %d: %v", database, nodeID, err) + } + t.Cleanup(func() { conn.Close() }) + return conn +} + +// waitForDatabase waits until database accepts a connection on every node. +func waitForDatabase(t *testing.T, harness *ClusterHarness, database string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for nodeID := 1; nodeID <= numNodes; nodeID++ { + for { + conn, err := sql.Open("mysql", fmt.Sprintf("root:@tcp(localhost:%d)/%s", harness.Nodes[nodeID-1].MySQLPort, database)) + if err == nil { + err = conn.Ping() + conn.Close() + } + if err == nil { + break + } + if time.Now().After(deadline) { + harness.dumpNodeLogs("wait_database_" + database) + t.Fatalf("database %s not on node %d: %v", database, nodeID, err) + } + time.Sleep(250 * time.Millisecond) + } + } +} + +// waitForTableIn waits until table exists in database on every node. +func waitForTableIn(t *testing.T, harness *ClusterHarness, database, table string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for nodeID := 1; nodeID <= numNodes; nodeID++ { + conn := openNodeDatabase(t, harness, nodeID, database) + for { + var n int + err := conn.QueryRow("SELECT COUNT(*) FROM " + table).Scan(&n) + if err == nil { + break + } + if time.Now().After(deadline) { + harness.dumpNodeLogs("wait_table_" + table) + t.Fatalf("%s.%s not on node %d: %v", database, table, nodeID, err) + } + time.Sleep(250 * time.Millisecond) + } + } +} + +// insertGenerated inserts one row with a generated id through conn, retrying +// the retryable lock-wait code a claim returns while the cluster settles. +func insertGenerated(conn *sql.DB, query string, args ...interface{}) (int64, error) { + var lastErr error + for attempt := 0; attempt < 40; attempt++ { + res, err := conn.Exec(query, args...) + if err == nil { + return res.LastInsertId() + } + lastErr = err + if !strings.Contains(err.Error(), "1205") { + return 0, err + } + time.Sleep(250 * time.Millisecond) + } + return 0, lastErr +} + +// TestNarrowAutoInc_DropDatabaseRecreateNeverReissues is reviewer A's F1: +// node 1 issues ids from a range, the database is dropped and recreated with +// the same table, and every node inserts again. Before, node 1 kept minting +// from its dead range while the others were granted the same ids afresh, and +// generated inserts failed with 1062 (or, unreplicated, overwrote). Now the +// claim row survives the drop and every node forgets its ranges of the +// dropped incarnation: no generated insert fails and no id repeats. +func TestNarrowAutoInc_DropDatabaseRecreateNeverReissues(t *testing.T) { + const database, table = "dropdb", "t" + harness := NewClusterHarness(t) + defer harness.Cleanup() + if err := harness.StartCluster(); err != nil { + t.Fatalf("StartCluster: %v", err) + } + + create := func() { + t.Helper() + if _, err := harness.ExecNode(1, "CREATE DATABASE "+database); err != nil { + t.Fatalf("CREATE DATABASE: %v", err) + } + waitForDatabase(t, harness, database, 30*time.Second) + if _, err := openNodeDatabase(t, harness, 1, database).Exec("CREATE TABLE " + table + " (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)"); err != nil { + t.Fatalf("CREATE TABLE: %v", err) + } + waitForTableIn(t, harness, database, table, 30*time.Second) + } + + create() + node1 := openNodeDatabase(t, harness, 1, database) + for i := 0; i < 3; i++ { + if _, err := insertGenerated(node1, "INSERT INTO "+table+" (v) VALUES ('before')"); err != nil { + t.Fatalf("insert before the drop: %v", err) + } + } + + if _, err := harness.ExecNode(1, "DROP DATABASE "+database); err != nil { + t.Fatalf("DROP DATABASE: %v", err) + } + time.Sleep(2 * time.Second) + create() + + // Enough rounds that a node granted the dropped incarnation's range + // afresh would reach the ids node 1 kept issuing from its old cursor. + seen := make(map[int64]int) + for round := 0; round < 10; round++ { + for nodeID := 1; nodeID <= numNodes; nodeID++ { + id, err := insertGenerated(openNodeDatabase(t, harness, nodeID, database), + "INSERT INTO "+table+" (v) VALUES (?)", fmt.Sprintf("n%d-%d", nodeID, round)) + if err != nil { + harness.dumpNodeLogs("dropdb_after") + t.Fatalf("node %d: generated insert after the recreate failed: %v", nodeID, err) + } + if other, dup := seen[id]; dup { + t.Fatalf("id %d issued by node %d and node %d after the recreate", id, other, nodeID) + } + seen[id] = nodeID + } + } + if err := waitForRowCountIn(harness, database, table, len(seen), 30*time.Second); err != nil { + harness.dumpNodeLogs("dropdb_rows") + t.Fatal(err) + } +} + +// waitForRowCountIn waits until database.table holds want rows with want +// distinct ids on every node. +func waitForRowCountIn(harness *ClusterHarness, database, table string, want int, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for nodeID := 1; nodeID <= numNodes; nodeID++ { + for { + conn, err := sql.Open("mysql", fmt.Sprintf("root:@tcp(localhost:%d)/%s", harness.Nodes[nodeID-1].MySQLPort, database)) + var rows, distinct int + if err == nil { + err = conn.QueryRow("SELECT COUNT(*), COUNT(DISTINCT id) FROM "+table).Scan(&rows, &distinct) + conn.Close() + } + if err == nil && rows == want && distinct == want { + break + } + if time.Now().After(deadline) { + return fmt.Errorf("node %d: %s.%s has %d rows, %d distinct ids, want %d (%v)", nodeID, database, table, rows, distinct, want, err) + } + time.Sleep(250 * time.Millisecond) + } + } + return nil +} + +// TestNarrowAutoInc_BoundNullAndQualifiedInsertsAreGenerated is reviewer A's +// F3 and F4 over the wire: a prepared INSERT binding NULL to the id, and an +// INSERT naming its table as db.t from a session in another database, text +// and prepared. Before, SQLite assigned such a row's id itself (MAX+1), and +// whenever that fell inside another node's unissued range the admission +// refused it with 1235: about half the bound NULLs and every qualified INSERT +// in reviewer A's runs. +// +// Each round node 1 first issues a generated id from its range and the row +// reaches the others; nodes 2 and 3 then write every shape. A SQLite rowid +// would land inside another node's unissued range and be refused, so every +// write must get a generated, distinct 32-bit id. +func TestNarrowAutoInc_BoundNullAndQualifiedInsertsAreGenerated(t *testing.T) { + const database, table = "qual", "t" + harness := NewClusterHarness(t) + defer harness.Cleanup() + if err := harness.StartCluster(); err != nil { + t.Fatalf("StartCluster: %v", err) + } + if _, err := harness.ExecNode(1, "CREATE DATABASE "+database); err != nil { + t.Fatalf("CREATE DATABASE: %v", err) + } + waitForDatabase(t, harness, database, 30*time.Second) + if _, err := openNodeDatabase(t, harness, 1, database).Exec("CREATE TABLE " + table + " (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)"); err != nil { + t.Fatalf("CREATE TABLE: %v", err) + } + waitForTableIn(t, harness, database, table, 30*time.Second) + + seen := make(map[int64]bool) + record := func(nodeID int, shape string, id int64, err error) { + t.Helper() + if err != nil { + harness.dumpNodeLogs("qual_" + shape) + t.Fatalf("node %d %s: %v", nodeID, shape, err) + } + if id <= 0 || id > narrowInt32Max || seen[id] { + t.Fatalf("node %d %s: id %d is not a new 32-bit id", nodeID, shape, id) + } + seen[id] = true + } + node1 := openNodeDatabase(t, harness, 1, database) + const rounds = 10 + for round := 0; round < rounds; round++ { + id, err := insertGenerated(node1, "INSERT INTO "+table+" (v) VALUES ('node1')") + record(1, "generated", id, err) + if err := waitForRowCountIn(harness, database, table, len(seen), 30*time.Second); err != nil { + t.Fatal(err) + } + // Node 2 writes the qualified shapes first and node 3 the bound NULL + // first, so that in the first round each shape is the first write + // on its node: a rowid it got from SQLite would be MAX+1, inside + // another node's unissued range. + boundNull := func(nodeID int) { + id, err := insertGenerated(openNodeDatabase(t, harness, nodeID, database), + "INSERT INTO "+table+" (id, v) VALUES (?, ?)", nil, "bound-null") + record(nodeID, "bound NULL", id, err) + } + qualified := func(nodeID int) { + inDefault := openNodeDatabase(t, harness, nodeID, "marmot") + id, err := insertGenerated(inDefault, "INSERT INTO "+database+"."+table+" (v) VALUES ('qualified')") + record(nodeID, "qualified text", id, err) + id, err = insertGenerated(inDefault, "INSERT INTO "+database+"."+table+" (v) VALUES (?)", "qualified-bound") + record(nodeID, "qualified bound", id, err) + } + qualified(2) + boundNull(2) + boundNull(3) + qualified(3) + } + if err := waitForRowCountIn(harness, database, table, len(seen), 30*time.Second); err != nil { + harness.dumpNodeLogs("qual_rows") + t.Fatal(err) + } +} + +// wipeNode stops nodeID and deletes everything in its data directory but its +// config, as a lost disk would. +func wipeNode(t *testing.T, harness *ClusterHarness, nodeID int) { + t.Helper() + if err := harness.StopNode(nodeID); err != nil { + t.Fatalf("StopNode(%d): %v", nodeID, err) + } + node := harness.Nodes[nodeID-1] + entries, err := os.ReadDir(node.DataDir) + if err != nil { + t.Fatalf("read node %d data dir: %v", nodeID, err) + } + for _, e := range entries { + if e.Name() == filepath.Base(node.ConfigPath) { + continue + } + if err := os.RemoveAll(filepath.Join(node.DataDir, e.Name())); err != nil { + t.Fatalf("wipe node %d: %v", nodeID, err) + } + } +} + +// nodeLogContains reports whether nodeID's current log holds text. +func nodeLogContains(t *testing.T, harness *ClusterHarness, nodeID int, text string) bool { + t.Helper() + log, err := os.ReadFile(harness.Nodes[nodeID-1].LogFile) + if err != nil { + t.Fatalf("read node %d log: %v", nodeID, err) + } + return strings.Contains(string(log), text) +} + +const autoIncReleasedLog = "AUTO_INCREMENT claim votes released" + +// TestNarrowAutoInc_RebuiltNodesHoldTheirVotes is reviewer B's C2 end to end: +// a node that loses its data directory restarts with its votes held, merges +// from the other members and releases, and inserts on every node then +// succeed with no repeated id. The new cluster itself released without help: +// every node of a new cluster starts held. +// +// The seedless seed node (C2 (a)) is covered in process +// (TestClaimRange_RebuiltNodeHoldsVotesUntilMerged) and its membership-of-one +// belt by grpc.TestAutoIncMergeSafe: in this harness a wiped node 1 comes back +// knowing only itself and stays that way, so the only observable outcome here +// would be that it never releases. +func TestNarrowAutoInc_RebuiltNodesHoldTheirVotes(t *testing.T) { + const table = "rebuilt" + harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + defer harness.Cleanup() + + seen := make(map[int64]bool) + insertOn := func(phase string, nodes ...int) { + t.Helper() + for _, nodeID := range nodes { + for i := 0; i < 5; i++ { + id, err := insertNarrow(harness, nodeID, "INSERT INTO "+table+" (v) VALUES (?)", phase) + if err != nil { + harness.dumpNodeLogs("rebuilt_" + phase) + t.Fatalf("%s: node %d insert: %v", phase, nodeID, err) + } + if seen[id] || id > narrowInt32Max { + t.Fatalf("%s: node %d issued %d again or beyond INT", phase, nodeID, id) + } + seen[id] = true + } + } + } + insertOn("before", 1, 2, 3) + + wipeNode(t, harness, 3) + if err := harness.StartNode(3); err != nil { + t.Fatalf("StartNode(3): %v", err) + } + if err := harness.WaitForAlive(3, 60*time.Second); err != nil { + harness.dumpNodeLogs("rebuilt_node3") + t.Fatalf("node 3 did not come back: %v", err) + } + deadline := time.Now().Add(60 * time.Second) + for !nodeLogContains(t, harness, 3, autoIncReleasedLog) { + if time.Now().After(deadline) { + harness.dumpNodeLogs("rebuilt_node3_release") + t.Fatal("rebuilt node 3 never released its claim votes") + } + time.Sleep(500 * time.Millisecond) + } + if err := harness.WaitForTableExists(table, []int{3}, 60*time.Second); err != nil { + harness.dumpNodeLogs("rebuilt_node3_table") + t.Fatalf("node 3 did not restore %s: %v", table, err) + } + insertOn("after-node3", 1, 2, 3) +} diff --git a/test/autoinc_membership_cluster_test.go b/test/autoinc_membership_cluster_test.go new file mode 100644 index 00000000..125136bd --- /dev/null +++ b/test/autoinc_membership_cluster_test.go @@ -0,0 +1,436 @@ +package test + +// Cluster tests for how a node that lost its claim state, or joined, comes +// back into the AUTO_INCREMENT claim protocol: the vote hold on every path +// that loses claim state, the release once the node can reach enough members, +// and the operator's sync command for a membership change. Every client call +// here carries its own timeout, so a hang fails the test instead of blocking +// it. + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "testing" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/maxpert/marmot/cfg" + "github.com/maxpert/marmot/protocol/mysqlcode" +) + +// clusterQueryTimeout bounds every single client query in these tests. +const clusterQueryTimeout = 20 * time.Second + +// openTimedNodeDatabase connects to database on nodeID with I/O timeouts on +// the connection itself, on top of each query's own context. +func openTimedNodeDatabase(t *testing.T, harness *ClusterHarness, nodeID int, database string) *sql.DB { + t.Helper() + dsn := fmt.Sprintf("root:@tcp(localhost:%d)/%s?timeout=5s&readTimeout=%s&writeTimeout=%s", + harness.Nodes[nodeID-1].MySQLPort, database, clusterQueryTimeout, clusterQueryTimeout) + conn, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("open %s on node %d: %v", database, nodeID, err) + } + t.Cleanup(func() { conn.Close() }) + return conn +} + +// execTimed runs one statement under clusterQueryTimeout. +func execTimed(conn *sql.DB, query string, args ...interface{}) (sql.Result, error) { + ctx, cancel := context.WithTimeout(context.Background(), clusterQueryTimeout) + defer cancel() + return conn.ExecContext(ctx, query, args...) +} + +// isLockTimeout reports whether err is the retryable 1205 a held or +// quorum-less claim returns. +func isLockTimeout(err error) bool { + var mysqlErr *mysql.MySQLError + return errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlcode.ErrCodeLockTimeout +} + +// insertNarrowWithin inserts one row with a generated id, retrying 1205 until +// budget is spent. +func insertNarrowWithin(conn *sql.DB, budget time.Duration, query string, args ...interface{}) (int64, error) { + deadline := time.Now().Add(budget) + for { + res, err := execTimed(conn, query, args...) + if err == nil { + return res.LastInsertId() + } + if !isLockTimeout(err) || time.Now().After(deadline) { + return 0, err + } + time.Sleep(250 * time.Millisecond) + } +} + +// autoIncSyncNode is one member's entry in the sync command's answer. +type autoIncSyncNode struct { + NodeID uint64 `json:"node_id"` + Answered bool `json:"answered"` + Complete bool `json:"complete"` + VotesHeld bool `json:"votes_held"` + Members []uint64 `json:"members"` + Reached []uint64 `json:"reached"` + ReachedEveryMember bool `json:"reached_every_member"` + Error string `json:"error"` +} + +// autoIncSyncAnswer is the sync command's answer. +type autoIncSyncAnswer struct { + Complete bool `json:"complete"` + Nodes []autoIncSyncNode `json:"nodes"` +} + +// node returns id's entry. +func (a autoIncSyncAnswer) node(id uint64) (autoIncSyncNode, bool) { + for _, n := range a.Nodes { + if n.NodeID == id { + return n, true + } + } + return autoIncSyncNode{}, false +} + +// runAutoIncSync runs POST /admin/cluster/autoinc/sync on nodeID with the +// harness's cluster secret. +func runAutoIncSync(harness *ClusterHarness, nodeID int) (autoIncSyncAnswer, error) { + url := fmt.Sprintf("http://localhost:%d/admin/cluster/autoinc/sync", harness.Nodes[nodeID-1].GRPCPort) + req, err := http.NewRequest(http.MethodPost, url, nil) + if err != nil { + return autoIncSyncAnswer{}, err + } + req.Header.Set("X-Marmot-Secret", "test-secret") + resp, err := (&http.Client{Timeout: 90 * time.Second}).Do(req) + if err != nil { + return autoIncSyncAnswer{}, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return autoIncSyncAnswer{}, err + } + if resp.StatusCode != http.StatusOK { + return autoIncSyncAnswer{}, fmt.Errorf("sync on node %d: %d %s", nodeID, resp.StatusCode, body) + } + var wrapped struct { + Data autoIncSyncAnswer `json:"data"` + } + if err := json.Unmarshal(body, &wrapped); err != nil { + return autoIncSyncAnswer{}, fmt.Errorf("sync on node %d: %w: %s", nodeID, err, body) + } + return wrapped.Data, nil +} + +// waitForRelease waits until nodeID's current log says its held votes were +// released. The log starts afresh with each start, so the line proves this +// run of the node was held. +func waitForRelease(t *testing.T, harness *ClusterHarness, nodeID int, timeout time.Duration, why string) { + t.Helper() + deadline := time.Now().Add(timeout) + for !nodeLogContains(t, harness, nodeID, autoIncReleasedLog) { + if time.Now().After(deadline) { + harness.dumpNodeLogs(fmt.Sprintf("release_node%d", nodeID)) + t.Fatalf("node %d: %s", nodeID, why) + } + time.Sleep(500 * time.Millisecond) + } +} + +// idLedger records every acknowledged generated id and fails on a repeat. +type idLedger map[int64]string + +func (l idLedger) record(t *testing.T, id int64, who string) { + t.Helper() + if id <= 0 || id > narrowInt32Max { + t.Fatalf("%s: id %d is not a 32-bit id", who, id) + } + if prev, dup := l[id]; dup { + t.Fatalf("DUPLICATE id %d: acknowledged to %s and to %s", id, prev, who) + } + l[id] = who +} + +// quorumProbeTable is a wide table every test here creates, so +// waitForQuorumWrites can tell when each node can commit writes again. +const quorumProbeTable = "quorum_probe" + +// createQuorumProbe creates quorumProbeTable on every node. +func createQuorumProbe(t *testing.T, harness *ClusterHarness) { + t.Helper() + if _, err := execTimed(openTimedNodeDatabase(t, harness, 1, "marmot"), + "CREATE TABLE "+quorumProbeTable+" (k INT PRIMARY KEY, v TEXT)"); err != nil { + t.Fatalf("CREATE TABLE %s: %v", quorumProbeTable, err) + } + if err := harness.WaitForTableExists(quorumProbeTable, []int{1, 2, 3}, 30*time.Second); err != nil { + t.Fatal(err) + } +} + +// waitForQuorumWrites waits until every node in nodes commits an ordinary +// replicated write. A node's connections to peers that restarted come back +// on gRPC's reconnect backoff, and until they do its writes cannot reach a +// quorum; that is not what these tests are about. +func waitForQuorumWrites(t *testing.T, harness *ClusterHarness, timeout time.Duration, nodes ...int) { + t.Helper() + deadline := time.Now().Add(timeout) + for _, nodeID := range nodes { + conn := openTimedNodeDatabase(t, harness, nodeID, "marmot") + for { + _, err := execTimed(conn, "REPLACE INTO "+quorumProbeTable+" (k, v) VALUES (?, 'probe')", nodeID) + if err == nil { + break + } + if time.Now().After(deadline) { + harness.dumpNodeLogs("quorum_writes") + t.Fatalf("node %d cannot commit a replicated write: %v", nodeID, err) + } + time.Sleep(time.Second) + } + } +} + +// insertOnEvery inserts k rows on each node and records their ids. +func insertOnEvery(t *testing.T, harness *ClusterHarness, ledger idLedger, table, phase string, k int, nodes ...int) { + t.Helper() + for _, nodeID := range nodes { + conn := openTimedNodeDatabase(t, harness, nodeID, "marmot") + for i := 0; i < k; i++ { + who := fmt.Sprintf("%s node %d #%d", phase, nodeID, i) + id, err := insertNarrowWithin(conn, 60*time.Second, "INSERT INTO "+table+" (v) VALUES (?)", who) + if err != nil { + harness.dumpNodeLogs(phase) + t.Fatalf("%s: %v", who, err) + } + ledger.record(t, id, who) + } + } +} + +// TestNarrowAutoInc_HeldNodeReleasesAfterItsSeedReturns is reviewer B's C2 +// (b) and (d) - a wiped node whose only seed is down, so its join fails and +// its catch-up strategy cannot use the seed - and reviewer A's F-L1: the node +// is held while it cannot reach enough members, and releases once its seed +// is back. Before the R3c-14 fix it never connected to the returned seed, +// which it had learned of as ALIVE from another member, and stayed held. +// +// Mutations: never hold (both hold sites) - "was not held while its seed was +// down" fires; drop the ALIVE callback on discovery (node_registry.go) - "did +// not release after its seed returned" fires. +func TestNarrowAutoInc_HeldNodeReleasesAfterItsSeedReturns(t *testing.T) { + const table = "seedback" + harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + defer harness.Cleanup() + createQuorumProbe(t, harness) + ledger := idLedger{} + insertOnEvery(t, harness, ledger, table, "before", 3, 1, 2, 3) + + wipeNode(t, harness, 3) + if err := harness.StopNode(1); err != nil { + t.Fatalf("StopNode(1): %v", err) + } + if err := harness.StartNode(3); err != nil { + t.Fatalf("StartNode(3): %v", err) + } + if err := harness.WaitForAlive(3, 90*time.Second); err != nil { + t.Fatalf("node 3 did not come back: %v", err) + } + if err := harness.WaitForTableExists(table, []int{3}, 60*time.Second); err != nil { + harness.dumpNodeLogs("seedback_table") + t.Fatalf("node 3 did not restore %s: %v", table, err) + } + + // With node 1 down node 3 hears one unheld member of three: it must stay + // held, and its narrow inserts wait. + answer, err := runAutoIncSync(harness, 3) + if err != nil { + t.Fatal(err) + } + self, ok := answer.node(3) + if !ok || !self.VotesHeld { + harness.dumpNodeLogs("seedback_held") + t.Fatalf("node 3 was not held while its seed was down: %+v", answer) + } + if answer.Complete { + t.Fatalf("the sync reported complete with node 1 down: %+v", answer) + } + if _, err := execTimed(openTimedNodeDatabase(t, harness, 3, "marmot"), "INSERT INTO "+table+" (v) VALUES ('held')"); !isLockTimeout(err) { + t.Fatalf("a narrow insert on held node 3 returned %v, want 1205", err) + } + + if err := harness.StartNode(1); err != nil { + t.Fatalf("StartNode(1): %v", err) + } + if err := harness.WaitForAlive(1, 90*time.Second); err != nil { + t.Fatalf("node 1 did not come back: %v", err) + } + waitForRelease(t, harness, 3, 90*time.Second, "did not release after its seed returned") + + waitForQuorumWrites(t, harness, 120*time.Second, 1, 2, 3) + insertOnEvery(t, harness, ledger, table, "after", 3, 1, 2, 3) + waitForCompleteSync(t, harness, 2, 3) +} + +// syncCompletePromotions is how many JOINING-to-ALIVE promotion windows a +// complete sync may take after a restart: a sync gathers only from the peers +// a node sees ALIVE, and a restarted node is ALIVE to its peers once it has +// been promoted and gossip has carried that. +const syncCompletePromotions = 6 + +// syncRetryInterval spaces the sync command's re-runs. +const syncRetryInterval = time.Second + +// syncCompleteDeadline bounds waitForCompleteSync: syncCompletePromotions +// promotion windows (check interval plus minimum healthy time) of the +// configuration the harness's nodes run with. +func syncCompleteDeadline() time.Duration { + p := cfg.Config.Cluster.Promotion + return syncCompletePromotions * time.Duration(p.CheckIntervalSeconds+p.MinHealthyDurationSec) * time.Second +} + +// waitForCompleteSync runs the sync command on nodeID until it reports +// complete, as the membership-change procedure does: re-run until the report +// is complete. Every report before that may miss only restarted, the member +// whose promotion to ALIVE has not reached every peer yet; any other +// unanswered, held or unreached member fails the test, and a complete report +// must show every member reached by every node. +func waitForCompleteSync(t *testing.T, harness *ClusterHarness, nodeID int, restarted uint64) { + t.Helper() + deadline := time.Now().Add(syncCompleteDeadline()) + for { + answer, err := runAutoIncSync(harness, nodeID) + if err != nil { + t.Fatal(err) + } + for _, n := range answer.Nodes { + if !n.Answered || n.VotesHeld { + t.Fatalf("node %d did not answer the sync or is held: %+v", n.NodeID, answer) + } + reached := make(map[uint64]bool, len(n.Reached)) + for _, id := range n.Reached { + reached[id] = true + } + for _, member := range n.Members { + if reached[member] { + continue + } + if answer.Complete || member != restarted { + t.Fatalf("node %d did not reach member %d in a sync that reported complete=%v: %+v", + n.NodeID, member, answer.Complete, answer) + } + } + } + if answer.Complete { + return + } + if time.Now().After(deadline) { + harness.dumpNodeLogs("sync_incomplete") + t.Fatalf("the sync with every member back is not complete after %s: %+v", syncCompleteDeadline(), answer) + } + time.Sleep(syncRetryInterval) + } +} + +// TestNarrowAutoInc_WipedSeedAndItsJoinerHold is reviewer B's C2 (a) and +// (c): the seed node, which has no seeds of its own, loses its data +// directory, and so does a node whose only seed it is, which therefore +// catches up from an empty seed. Both restart held and release only by +// merging claim bases from the member that kept them, and the sync command +// then finds every member unheld and reachable. +// +// It stops there: with two of three nodes rebuilt, a new write needs one of +// them, and a rebuilt node refuses writes until anti-entropy restores its +// schema version (the pre-existing catch-up gap, a separate step). +// +// Mutation: never hold (both hold sites). "node 1 was never held" fires. +func TestNarrowAutoInc_WipedSeedAndItsJoinerHold(t *testing.T) { + const table = "wipedseed" + harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + defer harness.Cleanup() + insertOnEvery(t, harness, idLedger{}, table, "before", 5, 1, 2, 3) + + wipeNode(t, harness, 1) + wipeNode(t, harness, 3) + for _, nodeID := range []int{1, 3} { + if err := harness.StartNode(nodeID); err != nil { + t.Fatalf("StartNode(%d): %v", nodeID, err) + } + if err := harness.WaitForAlive(nodeID, 90*time.Second); err != nil { + t.Fatalf("node %d did not come back: %v", nodeID, err) + } + } + waitForRelease(t, harness, 1, 90*time.Second, "node 1 was never held, or never released") + waitForRelease(t, harness, 3, 90*time.Second, "node 3 was never held, or never released") + + deadline := time.Now().Add(60 * time.Second) + for { + answer, err := runAutoIncSync(harness, 2) + if err != nil { + t.Fatal(err) + } + if answer.Complete { + break + } + if time.Now().After(deadline) { + harness.dumpNodeLogs("wipedseed_sync") + t.Fatalf("the sync never found every member unheld and reachable: %+v", answer) + } + time.Sleep(2 * time.Second) + } +} + +// TestNarrowAutoInc_SyncAfterAddingAMember is the membership-change procedure +// (R3c-8b) on a growing cluster: a fourth node joins, becomes ALIVE and +// releases, the operator runs the sync command, and it reports every member +// answered, unheld and reaching every member. Ids stay unique across all four. +// +// Mutation: syncAutoIncBasesEverywhere asks no other member. "the sync did +// not reach every member" fires. +func TestNarrowAutoInc_SyncAfterAddingAMember(t *testing.T) { + const table = "grow" + harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + defer harness.Cleanup() + ledger := idLedger{} + insertOnEvery(t, harness, ledger, table, "three", 3, 1, 2, 3) + + harness.Nodes = append(harness.Nodes, harness.createNode(4)) + if err := harness.StartNode(4); err != nil { + t.Fatalf("StartNode(4): %v", err) + } + if err := harness.WaitForAlive(4, 90*time.Second); err != nil { + t.Fatalf("node 4 did not come up: %v", err) + } + waitForRelease(t, harness, 4, 90*time.Second, "the new member never released its votes") + if err := harness.WaitForTableExists(table, []int{4}, 90*time.Second); err != nil { + harness.dumpNodeLogs("grow_table") + t.Fatalf("node 4 did not restore %s: %v", table, err) + } + + deadline := time.Now().Add(60 * time.Second) + for { + answer, err := runAutoIncSync(harness, 1) + if err != nil { + t.Fatal(err) + } + if answer.Complete && len(answer.Nodes) == 4 { + for _, n := range answer.Nodes { + if len(n.Members) != 4 || !n.ReachedEveryMember { + t.Fatalf("node %d: complete, yet reached %v of %v", n.NodeID, n.Reached, n.Members) + } + } + break + } + if time.Now().After(deadline) { + harness.dumpNodeLogs("grow_sync") + t.Fatalf("the sync did not reach every member: %+v", answer) + } + time.Sleep(2 * time.Second) + } + insertOnEvery(t, harness, ledger, table, "four", 3, 1, 2, 3, 4) +} diff --git a/test/autoinc_membership_inprocess_test.go b/test/autoinc_membership_inprocess_test.go new file mode 100644 index 00000000..46223603 --- /dev/null +++ b/test/autoinc_membership_inprocess_test.go @@ -0,0 +1,107 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestClaimRange_JoinersHoldVotesUntilMerged is the regression for the +// membership-growth overlap (step 3c, D8 F1). +// +// Claim C = [0,64) commits on {n1, n2} while n3 is unreachable, so n3 still +// holds base 0. Two nodes then join by catching up; claims never enter CDC, +// so each holds only the DDL-time seed, base 0. The membership is now five +// and a quorum is three: {n3, n4, n5} is a majority that shares no node with +// C's. Before joiners held their votes, n3 proposing base 0 was granted +// [0,64) a second time. With the hold, n4 and n5 decline until they have +// merged bases from a majority, so that majority cannot form, and n3 is +// taught C's end by n1 and n2 and granted a disjoint range. +// +// Mutation: ignore the vote hold in prepareAutoIncClaim. n3's first round is +// then granted at base 0 and "a claim after the membership grew overlapped +// C" fires. +func TestClaimRange_JoinersHoldVotesUntilMerged(t *testing.T) { + const database, table = "testdb", "grow" + c := setupClaimCluster(t, database, table) + + c.fanout.setUnreachable(3, true) + cBase, cSize, err := c.nodes[1].wc.ClaimRange(context.Background(), database, table, 0, fixedClaimSize(64)) + require.NoError(t, err) + require.Equal(t, uint64(0), cBase) + waitForQuorumBase(t, []*inprocNode{c.nodes[1], c.nodes[2]}, database, table, 64, 2) + c.fanout.setUnreachable(3, false) + base3, err := c.nodes[3].claimStore().ReadBase(database, table) + require.NoError(t, err) + require.Equal(t, uint64(0), base3, "n3 must have missed C for the sequence to mean anything") + + for _, id := range []uint64{4, 5} { + joiner := c.buildNode(id, t.TempDir()) + c.nodes[id] = joiner + require.NoError(t, joiner.dm.CreateDatabase(database)) + mdb, err := joiner.dm.GetDatabase(database) + require.NoError(t, err) + _, err = mdb.GetWriteDB().Exec(sprintfDDL(table)) + require.NoError(t, err) + require.NoError(t, mdb.ReloadSchema()) + // What replaying the table's CREATE seeds on a joiner. + require.NoError(t, joiner.claimStore().Seed(database, table, 0, id)) + } + c.provider.setNodes([]uint64{1, 2, 3, 4, 5}) + + mark4 := c.fanout.prepareCallCount(4) + dBase, dSize, err := c.nodes[3].wc.ClaimRange(context.Background(), database, table, 0, fixedClaimSize(64)) + require.NoError(t, err) + require.True(t, rangesDisjoint(cBase, cSize, dBase, dSize), + "a claim after the membership grew overlapped C: C=[%d,%d) D=[%d,%d)", cBase, cBase+cSize, dBase, dBase+dSize) + + round1Node4 := c.fanout.prepareCallsSince(4, mark4) + require.NotEmpty(t, round1Node4) + require.False(t, round1Node4[0].resp.Success, "a joiner voted before merging claim bases") + require.False(t, round1Node4[0].resp.Rejected, "a held joiner must decline, not cast a verdict") +} + +// TestClaimRange_RebuiltNodeHoldsVotesUntilMerged is reviewer B's C2 (a): a +// seed node configured with no seeds loses its data directory and restarts. +// Its hold no longer depends on seeds or on a catch-up decision: any start +// that initialises the system database holds. +// +// C = [0,64) commits on {n1, n2} while n3 is unreachable. n1 is rebuilt from +// an empty directory, and its user table comes back from n3 (which missed C) +// with ids up to 10, so its DDL-time seed is 10. Unheld, n1 would join n3 in +// granting a range starting at 10 - {n1, n3} is a majority of three - and +// overlap C. Held, it declines, and n3 is taught C's end by n2. +// +// Mutation: never place the hold in createAutoIncTables. n1 votes, the claim +// is granted at 10 and "a claim after n1 was rebuilt overlapped C" fires. +func TestClaimRange_RebuiltNodeHoldsVotesUntilMerged(t *testing.T) { + const database, table = "testdb", "rebuilt" + c := setupClaimCluster(t, database, table) + + c.fanout.setUnreachable(3, true) + cBase, cSize, err := c.nodes[1].wc.ClaimRange(context.Background(), database, table, 0, fixedClaimSize(64)) + require.NoError(t, err) + waitForQuorumBase(t, []*inprocNode{c.nodes[1], c.nodes[2]}, database, table, 64, 2) + c.fanout.setUnreachable(3, false) + + c.closeNode(1) + rebuilt := c.buildNode(1, t.TempDir()) + c.nodes[1] = rebuilt + require.NoError(t, rebuilt.dm.CreateDatabase(database)) + mdb, err := rebuilt.dm.GetDatabase(database) + require.NoError(t, err) + _, err = mdb.GetWriteDB().Exec(sprintfDDL(table)) + require.NoError(t, err) + require.NoError(t, mdb.ReloadSchema()) + // What the restored table's MAX(id) seeds on the rebuilt node. + require.NoError(t, rebuilt.claimStore().Seed(database, table, 10, 1)) + + dBase, dSize, err := c.nodes[3].wc.ClaimRange(context.Background(), database, table, 10, fixedClaimSize(64)) + require.NoError(t, err) + require.True(t, rangesDisjoint(cBase, cSize, dBase, dSize), + "a claim after n1 was rebuilt overlapped C: C=[%d,%d) D=[%d,%d)", cBase, cBase+cSize, dBase, dBase+dSize) +} diff --git a/test/autoinc_narrow_cluster_test.go b/test/autoinc_narrow_cluster_test.go new file mode 100644 index 00000000..017e0918 --- /dev/null +++ b/test/autoinc_narrow_cluster_test.go @@ -0,0 +1,457 @@ +package test + +// Cluster tests for narrow AUTO_INCREMENT columns on the insert path: every +// INSERT into an INT/TINYINT AUTO_INCREMENT table takes its id from a range +// claimed cluster-wide (id.RangeAllocator over coordinator.ClaimRange), +// driven over the real MySQL wire protocol against a 3-node ClusterHarness +// (test/crash_recovery_test.go). + +import ( + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/protocol/mysqlcode" +) + +const narrowInt32Max = 2147483647 + +// startNarrowCluster starts a 3-node cluster and creates table on it. +func startNarrowCluster(t *testing.T, table, ddl string) *ClusterHarness { + t.Helper() + harness := NewClusterHarness(t) + if err := harness.StartCluster(); err != nil { + harness.Cleanup() + t.Fatalf("StartCluster: %v", err) + } + if _, err := harness.ExecNode(1, ddl); err != nil { + harness.Cleanup() + t.Fatalf("CREATE TABLE %s: %v", table, err) + } + if err := harness.WaitForTableExists(table, []int{1, 2, 3}, 15*time.Second); err != nil { + harness.dumpNodeLogs("narrow_" + table) + harness.Cleanup() + t.Fatalf("DDL did not replicate: %v", err) + } + return harness +} + +// insertNarrow inserts one row on nodeID, retrying the retryable lock-wait +// timeout a claim returns while the cluster settles, and returns the id. +func insertNarrow(harness *ClusterHarness, nodeID int, query string, args ...interface{}) (int64, error) { + var lastErr error + for attempt := 0; attempt < 20; attempt++ { + res, err := harness.ExecNode(nodeID, query, args...) + if err == nil { + return res.LastInsertId() + } + lastErr = err + var mysqlErr *mysql.MySQLError + if !errors.As(err, &mysqlErr) || mysqlErr.Number != mysqlcode.ErrCodeLockTimeout { + return 0, err + } + time.Sleep(250 * time.Millisecond) + } + return 0, lastErr +} + +// narrowIDStats returns COUNT(*), COUNT(DISTINCT id) and MAX(id) on nodeID. +func narrowIDStats(harness *ClusterHarness, nodeID int, table, col string) (count, distinct, maxID int64, err error) { + rows, err := harness.QueryNode(nodeID, fmt.Sprintf("SELECT COUNT(*), COUNT(DISTINCT %s), MAX(%s) FROM %s", col, col, table)) + if err != nil { + return 0, 0, 0, err + } + defer rows.Close() + if !rows.Next() { + return 0, 0, 0, fmt.Errorf("no row from %s", table) + } + err = rows.Scan(&count, &distinct, &maxID) + return count, distinct, maxID, err +} + +// TestNarrowAutoInc_ConcurrentInsertsFitAndNeverCollide: three nodes insert +// concurrently into one INT AUTO_INCREMENT table. Every id must fit 32 bits +// and no id may be issued twice - the two properties the 53-bit ids of +// GitHub issue #170 broke and node-local rowids would break. +func TestNarrowAutoInc_ConcurrentInsertsFitAndNeverCollide(t *testing.T) { + const table = "narrow_concurrent" + harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (group_id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)") + defer harness.Cleanup() + + const perNode = 40 + var wg sync.WaitGroup + errs := make(chan error, 3*perNode) + for nodeID := 1; nodeID <= 3; nodeID++ { + wg.Add(1) + go func(nodeID int) { + defer wg.Done() + for i := 0; i < perNode; i++ { + id, err := insertNarrow(harness, nodeID, "INSERT INTO "+table+" (name) VALUES (?)", fmt.Sprintf("n%d-%d", nodeID, i)) + if err != nil { + errs <- fmt.Errorf("node %d insert %d: %w", nodeID, i, err) + return + } + if id <= 0 || id > narrowInt32Max { + errs <- fmt.Errorf("node %d issued id %d, outside [1, %d]", nodeID, id, narrowInt32Max) + } + } + }(nodeID) + } + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } + if t.Failed() { + harness.dumpNodeLogs("narrow_concurrent") + t.FailNow() + } + + if err := harness.WaitForRowCount(table, []int{1, 2, 3}, 3*perNode, 30*time.Second); err != nil { + harness.dumpNodeLogs("narrow_concurrent_rows") + t.Fatalf("rows did not replicate: %v", err) + } + for nodeID := 1; nodeID <= 3; nodeID++ { + count, distinct, maxID, err := narrowIDStats(harness, nodeID, table, "group_id") + if err != nil { + t.Fatalf("node %d: %v", nodeID, err) + } + if count != 3*perNode || distinct != count { + t.Fatalf("node %d: COUNT(*)=%d COUNT(DISTINCT group_id)=%d, want both %d", nodeID, count, distinct, 3*perNode) + } + if maxID > narrowInt32Max { + t.Fatalf("node %d: MAX(group_id)=%d does not fit INT", nodeID, maxID) + } + } +} + +// TestNarrowAutoInc_KillNineNeverReissues: a node killed with SIGKILL in the +// middle of a range loses its unissued tail as a gap; after restart it +// claims a fresh range and never issues an id it issued before. +func TestNarrowAutoInc_KillNineNeverReissues(t *testing.T) { + const table = "narrow_kill9" + harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + defer harness.Cleanup() + + issued := make(map[int64]bool) + var highest int64 + for i := 0; i < 10; i++ { + id, err := insertNarrow(harness, 2, "INSERT INTO "+table+" (v) VALUES ('before')") + if err != nil { + t.Fatalf("insert before kill: %v", err) + } + issued[id] = true + highest = max(highest, id) + } + + if err := harness.KillNode(2); err != nil { + t.Fatalf("KillNode: %v", err) + } + if err := harness.StartNode(2); err != nil { + t.Fatalf("StartNode: %v", err) + } + if err := harness.WaitForAlive(2, 60*time.Second); err != nil { + harness.dumpNodeLogs("narrow_kill9_restart") + t.Fatalf("node 2 did not come back: %v", err) + } + + for i := 0; i < 10; i++ { + id, err := insertNarrow(harness, 2, "INSERT INTO "+table+" (v) VALUES ('after')") + if err != nil { + harness.dumpNodeLogs("narrow_kill9_after") + t.Fatalf("insert after restart: %v", err) + } + if issued[id] || id <= highest { + t.Fatalf("restarted node issued %d; ids up to %d were issued before the kill", id, highest) + } + if id > narrowInt32Max { + t.Fatalf("id %d does not fit INT", id) + } + issued[id] = true + } +} + +// TestNarrowAutoInc_TinyIntExhaustionIsDupEntry: a TINYINT AUTO_INCREMENT +// column filled to 127 refuses the next INSERT with ER_DUP_ENTRY (1062), the +// error MySQL reports, never the retryable 1205. +func TestNarrowAutoInc_TinyIntExhaustionIsDupEntry(t *testing.T) { + const table = "narrow_tiny" + harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id TINYINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + defer harness.Cleanup() + + for i := 1; i <= 127; i++ { + if _, err := insertNarrow(harness, 1, "INSERT INTO "+table+" (v) VALUES ('x')"); err != nil { + harness.dumpNodeLogs("narrow_tiny_fill") + t.Fatalf("insert %d of 127: %v", i, err) + } + } + + _, err := harness.ExecNode(1, "INSERT INTO "+table+" (v) VALUES ('overflow')") + var mysqlErr *mysql.MySQLError + if !errors.As(err, &mysqlErr) || mysqlErr.Number != mysqlcode.ErrCodeDupEntry { + t.Fatalf("insert past 127 into TINYINT: err = %v, want MySQL error %d", err, mysqlcode.ErrCodeDupEntry) + } +} + +// TestNarrowAutoInc_LLDAPTransactionShape: LLDAP boots with one transaction +// inserting into two narrow tables that both need their first claim. It must +// complete, and each INSERT's LAST_INSERT_ID must be its own first id. +func TestNarrowAutoInc_LLDAPTransactionShape(t *testing.T) { + harness := startNarrowCluster(t, "users", "CREATE TABLE users (user_id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)") + defer harness.Cleanup() + if _, err := harness.ExecNode(1, "CREATE TABLE `groups` (group_id INT AUTO_INCREMENT PRIMARY KEY, display_name TEXT)"); err != nil { + t.Fatalf("CREATE TABLE groups: %v", err) + } + if err := harness.WaitForTableExists("groups", []int{1, 2, 3}, 15*time.Second); err != nil { + t.Fatalf("groups did not replicate: %v", err) + } + + conn, err := harness.ConnectToNode(2) + if err != nil { + t.Fatalf("ConnectToNode: %v", err) + } + tx, err := conn.Begin() + if err != nil { + t.Fatalf("BEGIN: %v", err) + } + usersRes, err := tx.Exec("INSERT INTO users (name) VALUES (?)", "admin") + if err != nil { + _ = tx.Rollback() + t.Fatalf("INSERT users: %v", err) + } + groupsRes, err := tx.Exec("INSERT INTO `groups` (display_name) VALUES (?), (?)", "lldap_admin", "lldap_password_manager") + if err != nil { + _ = tx.Rollback() + t.Fatalf("INSERT groups: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("COMMIT: %v", err) + } + + userID, _ := usersRes.LastInsertId() + groupID, _ := groupsRes.LastInsertId() + var storedUser, firstGroup, groupCount int64 + if err := conn.QueryRow("SELECT user_id FROM users WHERE name = 'admin'").Scan(&storedUser); err != nil { + t.Fatalf("read users: %v", err) + } + if err := conn.QueryRow("SELECT MIN(group_id), COUNT(*) FROM `groups`").Scan(&firstGroup, &groupCount); err != nil { + t.Fatalf("read groups: %v", err) + } + if userID != storedUser || groupID != firstGroup || groupCount != 2 { + t.Fatalf("LAST_INSERT_ID users=%d (stored %d), groups=%d (first stored %d, %d rows)", userID, storedUser, groupID, firstGroup, groupCount) + } + if storedUser > narrowInt32Max || firstGroup+1 > narrowInt32Max { + t.Fatalf("ids do not fit INT: user %d, groups from %d", storedUser, firstGroup) + } +} + +// TestNarrowAutoInc_UnmarkedTableKeepsWideIDs: a BIGINT AUTO_INCREMENT table +// carries no width marker and keeps the 53-bit ids it always had. +func TestNarrowAutoInc_UnmarkedTableKeepsWideIDs(t *testing.T) { + const table = "wide_bigint" + harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id BIGINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + defer harness.Cleanup() + + id, err := insertNarrow(harness, 1, "INSERT INTO "+table+" (v) VALUES ('x')") + if err != nil { + t.Fatalf("insert: %v", err) + } + if id <= 4294967295 || id >= 1<<53 { + t.Fatalf("BIGINT id %d is not a 53-bit generated id", id) + } +} + +// TestNarrowAutoInc_CopiedIDsRaiseTheBaseOnEveryNode: ids copied into an INT +// AUTO_INCREMENT table with INSERT ... SELECT - MySQL's usual way to +// re-declare a column's width, whose ids are known only once it has run - are +// admitted from the transaction's rows before it replicates, so the first +// generated id on EVERY node lands above the copied ones. +// +// Mutation: skip the admission of the transaction's rows +// (coordinator.admitNarrowIDs). The first generated id then reuses a copied +// one and the insert fails or "landed at or below the copied MAX" fires. +func TestNarrowAutoInc_CopiedIDsRaiseTheBaseOnEveryNode(t *testing.T) { + const source, table, copied = "copy_source", "copy_target", 300 + harness := startNarrowCluster(t, source, "CREATE TABLE "+source+" (id BIGINT PRIMARY KEY, v TEXT)") + defer harness.Cleanup() + + fillExplicitIDs(t, harness, source, copied) + if _, err := harness.ExecNode(1, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)"); err != nil { + t.Fatalf("CREATE TABLE %s: %v", table, err) + } + if err := harness.WaitForTableExists(table, []int{1, 2, 3}, 15*time.Second); err != nil { + t.Fatalf("%s did not replicate: %v", table, err) + } + if _, err := harness.ExecNode(1, "INSERT INTO "+table+" (id, v) SELECT id, v FROM "+source); err != nil { + t.Fatalf("INSERT ... SELECT: %v", err) + } + if err := harness.WaitForRowCount(table, []int{1, 2, 3}, copied, 30*time.Second); err != nil { + t.Fatalf("copied rows did not replicate: %v", err) + } + assertFirstGeneratedAbove(t, harness, table, copied) +} + +// TestNarrowAutoInc_SeedFromLegacyRowsOnEveryNode: rows that reached a narrow +// table without ever passing the allocator - written by a binary that +// predates it - are covered by the DDL-time seed. The next DDL on the table +// seeds every node's base from that node's own MAX(id), so the first +// generated id on EVERY node lands above them. The legacy rows are written +// straight into each stopped node's SQLite file, the only way such rows can +// arise now that every client write is admitted. +// +// Mutation: make autoIncSeedFloor ignore MAX(id) and seed only the declared +// floor. The first generated id then reuses a legacy one, and the insert +// fails or "landed at or below the copied MAX" fires. +func TestNarrowAutoInc_SeedFromLegacyRowsOnEveryNode(t *testing.T) { + const table, legacy = "legacy_rows", 300 + harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + defer harness.Cleanup() + + harness.StopCluster() + for nodeID := 1; nodeID <= 3; nodeID++ { + path := filepath.Join(harness.Nodes[nodeID-1].DataDir, "databases", db.DefaultDatabaseName+".db") + conn, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatalf("open node %d database: %v", nodeID, err) + } + for id := 1; id <= legacy; id++ { + if _, err := conn.Exec("INSERT INTO "+table+" (id, v) VALUES (?, 'legacy')", id); err != nil { + conn.Close() + t.Fatalf("write legacy row %d on node %d: %v", id, nodeID, err) + } + } + conn.Close() + } + for nodeID := 1; nodeID <= 3; nodeID++ { + if err := harness.StartNode(nodeID); err != nil { + t.Fatalf("StartNode %d: %v", nodeID, err) + } + } + for nodeID := 1; nodeID <= 3; nodeID++ { + if err := harness.WaitForAlive(nodeID, 60*time.Second); err != nil { + harness.dumpNodeLogs("narrow_legacy_restart") + t.Fatalf("node %d did not come back: %v", nodeID, err) + } + } + + if _, err := harness.ExecNode(1, "ALTER TABLE "+table+" ADD COLUMN note TEXT"); err != nil { + t.Fatalf("ALTER TABLE ADD COLUMN: %v", err) + } + for nodeID := 1; nodeID <= 3; nodeID++ { + waitForAutoIncBaseAtLeast(t, harness.Nodes[nodeID-1].DataDir, db.DefaultDatabaseName, table, legacy, 15*time.Second) + } + assertFirstGeneratedAbove(t, harness, table, legacy) +} + +// TestNarrowAutoInc_IDInAnotherNodesRangeIsRefused: node 1 claims a range +// and issues its first id. An explicit id inside node 1's unissued range, +// written through node 2 - as a literal, as a bound parameter, or copied +// with INSERT ... SELECT - is refused with 1235 before anything is written; +// were it accepted, node 1 would later issue the same id and the replicated +// row would replace the client's on every node. Node 1 then issues ids past +// it and the table never holds the client's row. +// +// Mutation: admit any id at or below the base. Node 2's writes succeed and +// "was admitted" fires. +func TestNarrowAutoInc_IDInAnotherNodesRangeIsRefused(t *testing.T) { + const table, source = "foreign_range", "foreign_source" + harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + defer harness.Cleanup() + if _, err := harness.ExecNode(1, "CREATE TABLE "+source+" (id BIGINT PRIMARY KEY, v TEXT)"); err != nil { + t.Fatalf("CREATE TABLE %s: %v", source, err) + } + if err := harness.WaitForTableExists(source, []int{1, 2, 3}, 15*time.Second); err != nil { + t.Fatalf("%s did not replicate: %v", source, err) + } + + first, err := insertNarrow(harness, 1, "INSERT INTO "+table+" (v) VALUES ('node1')") + if err != nil { + t.Fatalf("node 1 insert: %v", err) + } + if _, err := harness.ExecNode(2, "INSERT INTO "+source+" (id, v) VALUES (?, 'copied')", first+3); err != nil { + t.Fatalf("fill %s: %v", source, err) + } + if err := harness.WaitForRowCount(source, []int{2}, 1, 15*time.Second); err != nil { + t.Fatalf("%s row did not reach node 2: %v", source, err) + } + + for name, write := range map[string]func() error{ + "literal": func() error { + _, err := harness.ExecNode(2, fmt.Sprintf("INSERT INTO %s (id, v) VALUES (%d, 'client')", table, first+1)) + return err + }, + "bound parameter": func() error { + _, err := harness.ExecNode(2, "INSERT INTO "+table+" (id, v) VALUES (?, 'client')", first+2) + return err + }, + "insert select": func() error { + _, err := harness.ExecNode(2, "INSERT INTO "+table+" (id, v) SELECT id, v FROM "+source) + return err + }, + } { + err := write() + var mysqlErr *mysql.MySQLError + if !errors.As(err, &mysqlErr) || mysqlErr.Number != mysqlcode.ErrCodeNotSupportedYet { + t.Fatalf("%s: an id inside node 1's unissued range was admitted through node 2: err = %v", name, err) + } + } + + for i := 0; i < 5; i++ { + if _, err := insertNarrow(harness, 1, "INSERT INTO "+table+" (v) VALUES ('node1')"); err != nil { + t.Fatalf("node 1 insert %d: %v", i, err) + } + } + if err := harness.WaitForRowCount(table, []int{1, 2, 3}, 6, 15*time.Second); err != nil { + t.Fatalf("rows did not replicate: %v", err) + } + for nodeID := 1; nodeID <= 3; nodeID++ { + rows, err := harness.QueryNode(nodeID, "SELECT COUNT(*) FROM "+table+" WHERE v <> 'node1'") + if err != nil { + t.Fatalf("node %d: %v", nodeID, err) + } + var foreign int + rows.Next() + _ = rows.Scan(&foreign) + rows.Close() + if foreign != 0 { + t.Fatalf("node %d holds %d rows written through node 2 in node 1's range", nodeID, foreign) + } + } +} + +// fillExplicitIDs inserts rows 1..n into table through node 1. +func fillExplicitIDs(t *testing.T, harness *ClusterHarness, table string, n int) { + t.Helper() + for start := 1; start <= n; start += 100 { + var values []string + for id := start; id < start+100 && id <= n; id++ { + values = append(values, fmt.Sprintf("(%d, 'row_%d')", id, id)) + } + if _, err := harness.ExecNode(1, "INSERT INTO "+table+" (id, v) VALUES "+strings.Join(values, ", ")); err != nil { + t.Fatalf("fill %s: %v", table, err) + } + } +} + +// assertFirstGeneratedAbove inserts one generated row through every node and +// requires each id to lie above highest. +func assertFirstGeneratedAbove(t *testing.T, harness *ClusterHarness, table string, highest int64) { + t.Helper() + for nodeID := 1; nodeID <= 3; nodeID++ { + id, err := insertNarrow(harness, nodeID, "INSERT INTO "+table+" (v) VALUES ('generated')") + if err != nil { + harness.dumpNodeLogs("narrow_generated_above") + t.Fatalf("node %d: first generated insert: %v", nodeID, err) + } + if id <= highest { + t.Fatalf("node %d: generated id %d landed at or below the copied MAX %d", nodeID, id, highest) + } + } +} diff --git a/test/cluster_integration_test.go b/test/cluster_integration_test.go index 4987d69b..442bd157 100644 --- a/test/cluster_integration_test.go +++ b/test/cluster_integration_test.go @@ -452,6 +452,9 @@ func startNode(t *testing.T, node *testNode, seedNodes []string) { // Setup coordinators nodeProvider := marmotgrpc.NewGossipNodeProvider(gossip.GetNodeRegistry(), false) + // As marmot.go does: a claim participant counts the cluster exactly as a + // claimant's quorum does. + dbMgr.SetClusterMembership(nodeProvider.GetTotalMembershipSize) replicator := marmotgrpc.NewGRPCReplicator(client) localReplicator := db.NewLocalReplicator(node.nodeID, dbMgr, node.clock) diff --git a/test/crash_recovery_test.go b/test/crash_recovery_test.go index d8d0bd89..5e8b77bc 100644 --- a/test/crash_recovery_test.go +++ b/test/crash_recovery_test.go @@ -686,7 +686,7 @@ func (h *ClusterHarness) StartCluster() error { func (h *ClusterHarness) StopCluster() { h.t.Logf("Stopping cluster...") - for i := 1; i <= numNodes; i++ { + for i := 1; i <= len(h.Nodes); i++ { h.StopNode(i) } } diff --git a/test/information_schema_cluster_test.go b/test/information_schema_cluster_test.go new file mode 100644 index 00000000..3a565690 --- /dev/null +++ b/test/information_schema_cluster_test.go @@ -0,0 +1,181 @@ +package test + +// INFORMATION_SCHEMA over the real MySQL wire protocol, text and prepared, +// against a 3-node ClusterHarness. Every query carries its own timeout: the +// failure these tests exist for is a query that never answers. + +import ( + "context" + "database/sql" + "fmt" + "reflect" + "strconv" + "testing" + "time" +) + +// isAnswer is one INFORMATION_SCHEMA answer, normalised so the text +// protocol's strings and the binary protocol's typed values compare equal. +type isAnswer struct { + columns []string + rows [][]string +} + +// queryAnswer runs one query under clusterQueryTimeout. With args, the driver +// prepares it on the server (COM_STMT_PREPARE/EXECUTE); without, it is a text +// query. +func queryAnswer(conn *sql.DB, query string, args ...interface{}) (isAnswer, error) { + ctx, cancel := context.WithTimeout(context.Background(), clusterQueryTimeout) + defer cancel() + rows, err := conn.QueryContext(ctx, query, args...) + if err != nil { + return isAnswer{}, err + } + defer rows.Close() + columns, err := rows.Columns() + if err != nil { + return isAnswer{}, err + } + answer := isAnswer{columns: columns} + for rows.Next() { + values := make([]interface{}, len(columns)) + ptrs := make([]interface{}, len(columns)) + for i := range values { + ptrs[i] = &values[i] + } + if err := rows.Scan(ptrs...); err != nil { + return isAnswer{}, err + } + row := make([]string, len(values)) + for i, v := range values { + switch x := v.(type) { + case nil: + row[i] = "NULL" + case []byte: + row[i] = string(x) + case int64: + row[i] = strconv.FormatInt(x, 10) + default: + row[i] = fmt.Sprint(x) + } + } + answer.rows = append(answer.rows, row) + } + return answer, rows.Err() +} + +// TestInformationSchemaAnswersOnEveryPath is R3c-9 (reviewer A's F-P1 and +// F-P2) end to end: every INFORMATION_SCHEMA table Marmot serves answers, +// with and without filters, the same rows and columns to a prepared +// statement as to the text query, on every node; and DDL coordinated from +// the node afterwards completes. Before, a prepared COLUMNS or TABLES query +// never answered and took the database's only write connection with it, so +// the DDL hung; a prepared query that did answer came back as an empty OK. +// The existence probe migration tools send (table_schema = ? AND table_name +// = ?) is among them. +// +// Mutations: the metadata handlers read through the write handle with the +// nested table loop (the 6cf7e580 handlers) - the unfiltered COLUMNS query +// times out; a prepared statement answers a result set only for SELECT - the +// prepared answers come back with no columns. +func TestInformationSchemaAnswersOnEveryPath(t *testing.T) { + harness := NewClusterHarness(t) + defer harness.Cleanup() + if err := harness.StartCluster(); err != nil { + t.Fatalf("StartCluster: %v", err) + } + + if _, err := execTimed(openTimedNodeDatabase(t, harness, 1, "marmot"), "CREATE DATABASE isdb"); err != nil { + t.Fatalf("CREATE DATABASE: %v", err) + } + waitForDatabase(t, harness, "isdb", 30*time.Second) + isdb := openTimedNodeDatabase(t, harness, 1, "isdb") + for _, ddl := range []string{ + "CREATE TABLE users (id INT AUTO_INCREMENT PRIMARY KEY, name VARCHAR(64) NOT NULL, email VARCHAR(128))", + "CREATE INDEX idx_users_email ON users (email)", + "CREATE TABLE posts (id BIGINT PRIMARY KEY, body TEXT)", + } { + if _, err := execTimed(isdb, ddl); err != nil { + t.Fatalf("%s: %v", ddl, err) + } + } + waitForTableIn(t, harness, "isdb", "posts", 30*time.Second) + + cases := []struct { + name string + database string + text string + prepared string + args []interface{} + wantRows int + }{ + {"COLUMNS of one table", "marmot", + "SELECT * FROM information_schema.columns WHERE table_schema = 'isdb' AND table_name = 'users'", + "SELECT * FROM information_schema.columns WHERE table_schema = ? AND table_name = ?", + []interface{}{"isdb", "users"}, 3}, + {"COLUMNS of every table", "marmot", + "SELECT * FROM information_schema.columns WHERE table_schema = 'isdb'", + "SELECT * FROM information_schema.columns WHERE table_schema = ?", + []interface{}{"isdb"}, 5}, + {"TABLES of a schema", "marmot", + "SELECT * FROM information_schema.tables WHERE table_schema = 'isdb'", + "SELECT * FROM information_schema.tables WHERE table_schema = ?", + []interface{}{"isdb"}, 2}, + {"TABLES existence probe", "marmot", + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'isdb' AND table_name = 'users'", + "SELECT table_name FROM information_schema.tables WHERE table_schema = ? AND table_name = ?", + []interface{}{"isdb", "users"}, 1}, + {"TABLES existence probe, absent table", "marmot", + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'isdb' AND table_name = 'nosuch'", + "SELECT table_name FROM information_schema.tables WHERE table_schema = ? AND table_name = ?", + []interface{}{"isdb", "nosuch"}, 0}, + {"TABLES of the session database", "isdb", + "SELECT * FROM information_schema.tables WHERE table_type = 'BASE TABLE'", + "SELECT * FROM information_schema.tables WHERE table_type = ?", + []interface{}{"BASE TABLE"}, 2}, + {"SCHEMATA", "marmot", + "SELECT * FROM information_schema.schemata WHERE schema_name = 'isdb'", + "SELECT * FROM information_schema.schemata WHERE schema_name = ?", + []interface{}{"isdb"}, 1}, + {"STATISTICS", "marmot", + "SELECT * FROM information_schema.statistics WHERE table_schema = 'isdb' AND table_name = 'users'", + "SELECT * FROM information_schema.statistics WHERE table_schema = ? AND table_name = ?", + []interface{}{"isdb", "users"}, 1}, + } + + for _, nodeID := range []int{1, 2, 3} { + // Prepared first: before the fix the first prepared query wedged the + // node, so every later query on it would time out. + for _, tc := range cases { + conn := openTimedNodeDatabase(t, harness, nodeID, tc.database) + prepared, err := queryAnswer(conn, tc.prepared, tc.args...) + if err != nil { + harness.dumpNodeLogs("is_prepared") + t.Fatalf("node %d %s, prepared: %v", nodeID, tc.name, err) + } + text, err := queryAnswer(conn, tc.text) + if err != nil { + t.Fatalf("node %d %s, text: %v", nodeID, tc.name, err) + } + if len(text.rows) != tc.wantRows { + t.Fatalf("node %d %s: text answered %d rows, want %d: %v", nodeID, tc.name, len(text.rows), tc.wantRows, text.rows) + } + if len(prepared.columns) == 0 { + t.Fatalf("node %d %s: the prepared answer has no columns", nodeID, tc.name) + } + if !reflect.DeepEqual(prepared, text) { + t.Fatalf("node %d %s: prepared and text answers differ\nprepared: %v %v\ntext: %v %v", + nodeID, tc.name, prepared.columns, prepared.rows, text.columns, text.rows) + } + } + + // DDL coordinated from the node that just answered them, on the + // database they read, completes. An index on posts leaves every answer + // above unchanged for the next node. + ddl := fmt.Sprintf("CREATE INDEX after_is_%d ON posts (body)", nodeID) + if _, err := execTimed(openTimedNodeDatabase(t, harness, nodeID, "isdb"), ddl); err != nil { + harness.dumpNodeLogs("is_ddl") + t.Fatalf("node %d: %s after INFORMATION_SCHEMA queries: %v", nodeID, ddl, err) + } + } +} diff --git a/test/schema_version_kill_cluster_test.go b/test/schema_version_kill_cluster_test.go new file mode 100644 index 00000000..ca5a5432 --- /dev/null +++ b/test/schema_version_kill_cluster_test.go @@ -0,0 +1,36 @@ +package test + +import ( + "testing" + "time" +) + +// TestSchemaVersionSurvivesKillNine: a node applies DDL, is killed with +// SIGKILL and restarts with its tables intact. It must also keep its schema +// version: a node reporting a lower version than the schema it runs refuses +// every peer transaction that requires the newer one, and nothing but a +// snapshot restore heals it. With node 3 stopped, node 2's write needs node +// 1's vote, so it commits only if node 1 kept its version. +// +// Mutation: write the schema version with pebble.NoSync +// (PebbleMetaStore.UpdateSchemaVersion). "node 2 cannot commit a replicated +// write" fires: node 1 restarts at version 0. +func TestSchemaVersionSurvivesKillNine(t *testing.T) { + harness := startNarrowCluster(t, "sv", "CREATE TABLE sv (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + defer harness.Cleanup() + createQuorumProbe(t, harness) + + if err := harness.StopNode(1); err != nil { + t.Fatalf("StopNode(1): %v", err) + } + if err := harness.StartNode(1); err != nil { + t.Fatalf("StartNode(1): %v", err) + } + if err := harness.WaitForAlive(1, 90*time.Second); err != nil { + t.Fatalf("node 1 did not come back: %v", err) + } + if err := harness.StopNode(3); err != nil { + t.Fatalf("StopNode(3): %v", err) + } + waitForQuorumWrites(t, harness, 60*time.Second, 2) +} diff --git a/test/verification_test.go b/test/verification_test.go index 60aae32f..4eb3376d 100644 --- a/test/verification_test.go +++ b/test/verification_test.go @@ -58,6 +58,12 @@ func (tdm *TestDatabaseManager) GetDatabase(name string) (*db.ReplicatedDatabase return tdm.db, nil } +// ClusterMembership reports a single-node cluster: this fixture serves one +// node and claims no AUTO_INCREMENT ranges across any other. +func (tdm *TestDatabaseManager) ClusterMembership() (int, error) { + return 1, nil +} + func (tdm *TestDatabaseManager) ListDatabases() []string { return []string{"marmot"} } From d0d47f03878f8fb7c4dc9164e3ccd11490dbd7de Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Mon, 28 Sep 2026 13:34:51 -0500 Subject: [PATCH 06/17] feat(replication): anti-entropy pulls every peer's commit log from persisted cursors Delta sync could miss ACKed rows on a node that lagged or restarted: it compared watermarks, not the transactions each peer actually committed, and replay did not advance the schema version. Anti-entropy now pulls, per (peer, database), from each alive peer's local commit log (ListCommittedLog / FetchTransactions), with a durable cursor keyed (seq, txnID) that only advances over a contiguous prefix of applied entries. An exact in-flight tracker bounds what a peer lists. Replay is idempotent: an __marmot_applied_txn marker is written in the same SQLite transaction as the effect on every commit path, and the schema version lives in __marmot_schema_version in the DDL's own transaction. Startup catch-up uses the same pull, and a node turns ALIVE only when every database is caught up. A transaction a peer's log holds is decided COMMITTED: a locally prepared copy is committed through the local path, a begun-only record is discarded and replayed, and a late PREPARE for it is refused. A commit whose captured rows or intents no longer match its durable prepare record is refused, and the stale-transaction GC aborts only under the per-transaction commit guard. The log is garbage-collected by the minimum position every member has consumed, bounded by a maximum retention; a pair whose cursor falls behind the truncation point restores from a snapshot and re-applies the node's own log. The database set is reconciled each round with a legacy marker for pre-upgrade registry rows. DDL and CREATE/DROP DATABASE are refused while any member does not serve the log pull protocol; upgrade every node, then run DDL. Delta sync and the snapshot schema-version trailer are removed. Cluster tests gain a log-based stall watchdog, poll readiness instead of sleeping, and each finishes within a minute. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- admin/autoinc.go | 17 + admin/autoinc_test.go | 34 +- admin/cluster.go | 73 +- admin/handlers.go | 15 +- admin/replication.go | 72 +- admin/replication_state_test.go | 104 ++ admin/routes.go | 18 +- admin/schema.go | 10 +- cfg/config.go | 8 + coordinator/commit_lost_pinned_state_test.go | 4 +- .../database_registry_generation_test.go | 130 ++ coordinator/errors.go | 43 + coordinator/handler.go | 74 +- coordinator/handler_draining_test.go | 10 +- coordinator/noop_dml_test.go | 5 +- coordinator/rolling_upgrade_ddl_gate_test.go | 135 ++ db/applied_txn_marker.go | 8 +- db/autoinc_claim.go | 37 +- db/autoinc_seed_test.go | 10 +- db/cdc_segment_log.go | 12 + db/database_manager.go | 795 ++++++++---- db/database_manager_test.go | 526 ++++++-- db/db_integration.go | 100 +- db/ddl_prepare_validate.go | 9 + db/ddl_prepare_validate_test.go | 9 + db/detach_drain_test.go | 35 +- db/gc_log_test.go | 363 ++++++ db/gc_membership_test.go | 151 +++ db/log_position.go | 355 ++++++ db/log_position_test.go | 425 +++++++ db/meta_schema.go | 28 +- db/meta_store.go | 153 ++- db/meta_store_memory.go | 201 ++- db/meta_store_memory_test.go | 147 ++- db/meta_store_pebble.go | 1125 ++++++++++------- db/meta_store_pebble_test.go | 146 ++- db/prepared_intent_test.go | 28 +- db/replay_apply.go | 598 +++++++++ db/replay_apply_test.go | 496 ++++++++ db/replication_engine.go | 265 ++-- db/replication_engine_test.go | 97 ++ db/schema_version_manager.go | 177 +-- db/schema_version_restore.go | 50 - db/schema_version_restore_test.go | 77 -- db/schema_version_table.go | 154 +++ db/schema_version_table_test.go | 215 ++++ db/snapshot/grpc_adapter.go | 6 + db/snapshot/restorer.go | 49 +- db/snapshot/restorer_test.go | 32 + db/snapshot/snapshot.go | 4 + db/stale_gc_commit_race_test.go | 129 ++ db/test_helpers.go | 7 + db/transaction.go | 706 ++++++++--- db/transaction_nondml_atomic_test.go | 246 ++++ db/transaction_rowcount_contract_test.go | 188 +++ db/transaction_schema_reload_test.go | 41 +- db/transaction_schema_version_test.go | 3 + docs/src/pages/operations.mdx | 52 + grpc/anti_entropy.go | 959 ++++++-------- grpc/anti_entropy_regression_test.go | 502 ++++++++ grpc/anti_entropy_test.go | 557 +++----- grpc/auth_test.go | 72 ++ grpc/autoinc_claim_handler_test.go | 2 +- grpc/autoinc_claim_schema_gate_test.go | 16 +- grpc/catch_up.go | 275 ++-- grpc/catch_up_detach_test.go | 20 +- grpc/catch_up_test.go | 374 +----- grpc/cdc_apply_helpers.go | 152 ++- grpc/cdc_apply_helpers_test.go | 135 +- grpc/delta_sync.go | 534 -------- grpc/delta_sync_apply_test.go | 249 ---- grpc/delta_sync_test.go | 118 -- grpc/log_protocol_version_test.go | 170 +++ grpc/log_pull_harness_test.go | 157 +++ grpc/log_puller.go | 712 +++++++++++ grpc/log_puller_gc_race_test.go | 169 +++ grpc/log_puller_test.go | 562 ++++++++ grpc/log_server.go | 318 +++++ grpc/log_server_test.go | 247 ++++ grpc/marmot.pb.go | 946 +++++++++++--- grpc/marmot.proto | 83 +- grpc/marmot_grpc.pb.go | 133 +- grpc/node_registry.go | 140 +- grpc/node_registry_test.go | 80 +- grpc/replication_focused_test.go | 86 +- grpc/replication_handler.go | 327 +---- grpc/replication_handler_test.go | 86 +- grpc/replicator_adapter.go | 1 + grpc/replicator_adapter_test.go | 4 +- grpc/rolling_upgrade_prepare_test.go | 115 ++ grpc/server.go | 335 ++--- grpc/server_promotion_test.go | 55 +- grpc/shutdown_integration_test.go | 4 +- grpc/snapshot_export.go | 11 +- grpc/snapshot_schema_version.go | 177 --- grpc/snapshot_schema_version_test.go | 193 --- grpc/test_helpers.go | 52 +- grpc/vector_statement.go | 1 + marmot.go | 62 +- protocol/transaction.go | 12 + replica/database_discovery_test.go | 24 + replica/stream_client.go | 33 +- telemetry/metrics.go | 48 +- test/autoinc_backstop_cluster_test.go | 12 +- test/autoinc_claim_inprocess_test.go | 2 +- test/autoinc_membership_cluster_test.go | 19 +- test/autoinc_narrow_cluster_test.go | 8 +- test/cluster_integration_test.go | 21 +- test/crash_recovery_test.go | 576 +++++++-- test/ddl_replication_test.go | 124 +- test/gap_detection_test.go | 95 +- test/information_schema_cluster_test.go | 5 +- test/node_outage_convergence_cluster_test.go | 883 +++++++++++++ test/rolling_upgrade_cluster_test.go | 216 ++++ test/stallwatch/watchdog.go | 65 + test/stallwatch/watchdog_test.go | 66 + test/verification_test.go | 7 + test/write_ledger_convergence_cluster_test.go | 244 ++++ 118 files changed, 14856 insertions(+), 5802 deletions(-) create mode 100644 admin/replication_state_test.go create mode 100644 coordinator/database_registry_generation_test.go create mode 100644 coordinator/rolling_upgrade_ddl_gate_test.go create mode 100644 db/gc_log_test.go create mode 100644 db/gc_membership_test.go create mode 100644 db/log_position.go create mode 100644 db/log_position_test.go create mode 100644 db/replay_apply.go create mode 100644 db/replay_apply_test.go delete mode 100644 db/schema_version_restore.go delete mode 100644 db/schema_version_restore_test.go create mode 100644 db/schema_version_table.go create mode 100644 db/schema_version_table_test.go create mode 100644 db/stale_gc_commit_race_test.go create mode 100644 db/transaction_nondml_atomic_test.go create mode 100644 db/transaction_rowcount_contract_test.go create mode 100644 grpc/anti_entropy_regression_test.go delete mode 100644 grpc/delta_sync.go delete mode 100644 grpc/delta_sync_apply_test.go delete mode 100644 grpc/delta_sync_test.go create mode 100644 grpc/log_protocol_version_test.go create mode 100644 grpc/log_pull_harness_test.go create mode 100644 grpc/log_puller.go create mode 100644 grpc/log_puller_gc_race_test.go create mode 100644 grpc/log_puller_test.go create mode 100644 grpc/log_server.go create mode 100644 grpc/log_server_test.go create mode 100644 grpc/rolling_upgrade_prepare_test.go delete mode 100644 grpc/snapshot_schema_version.go delete mode 100644 grpc/snapshot_schema_version_test.go create mode 100644 test/node_outage_convergence_cluster_test.go create mode 100644 test/rolling_upgrade_cluster_test.go create mode 100644 test/stallwatch/watchdog.go create mode 100644 test/stallwatch/watchdog_test.go create mode 100644 test/write_ledger_convergence_cluster_test.go diff --git a/admin/autoinc.go b/admin/autoinc.go index 5b03473a..0b656f19 100644 --- a/admin/autoinc.go +++ b/admin/autoinc.go @@ -3,9 +3,26 @@ package admin import ( "net/http" + "github.com/maxpert/marmot/db" marmotgrpc "github.com/maxpert/marmot/grpc" ) +// handleAutoIncVotes handles GET /admin/cluster/autoinc/votes: whether this +// node's AUTO_INCREMENT claim votes are held (db.AutoIncHoldTable). While +// they are, the node declines every claim at PREPARE, so narrow +// AUTO_INCREMENT inserts that need a new range can return 1205 until enough +// members answered its merge. It is the "unheld" check of the +// membership-change procedure (handleAutoIncSync), and a readiness signal +// after a node's first start. +func (h *AdminHandlers) handleAutoIncVotes(w http.ResponseWriter, r *http.Request) { + held, err := db.NewAutoIncClaimStore(h.dbManager.GetSystemDatabase()).VotesHeld() + if err != nil { + writeErrorResponse(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSONResponse(w, map[string]interface{}{"votes_held": held}, false, "") +} + // handleAutoIncReleaseVotes handles POST // /admin/cluster/autoinc/release-votes?accept_risk=duplicate-ids. // diff --git a/admin/autoinc_test.go b/admin/autoinc_test.go index 059464d8..c784a42a 100644 --- a/admin/autoinc_test.go +++ b/admin/autoinc_test.go @@ -8,7 +8,9 @@ import ( "testing" "github.com/maxpert/marmot/cfg" + "github.com/maxpert/marmot/db" marmotgrpc "github.com/maxpert/marmot/grpc" + "github.com/maxpert/marmot/hlc" ) // TestAutoIncReleaseVotesRequiresTheRiskToken pins that the forced release is @@ -32,7 +34,7 @@ func TestAutoIncReleaseVotesRequiresTheRiskToken(t *testing.T) { } } -// TestAutoIncEndpointsRequireTheClusterSecret pins R3c-15: the release and +// TestAutoIncEndpointsRequireTheClusterSecret pins that the release and // sync endpoints are refused on a cluster with no secret, and served only // with the right one on a cluster that has one - unlike the other admin // endpoints, which AuthMiddleware serves to anyone when no secret is set. @@ -93,3 +95,33 @@ func TestAutoIncSyncBodyIsCompleteOnlyWhenEveryMemberIs(t *testing.T) { t.Fatal("no outcome at all reported complete") } } + +// TestAutoIncVotesReportsTheHold pins the read-only readiness signal: a node +// whose system database was just created holds its claim votes, and reports +// them released once the hold row is gone. +func TestAutoIncVotesReportsTheHold(t *testing.T) { + dm, err := db.NewDatabaseManager(t.TempDir(), 1, hlc.NewClock(1)) + if err != nil { + t.Fatal(err) + } + defer dm.Close() + h := &AdminHandlers{dbManager: dm} + + votesHeld := func() string { + rec := httptest.NewRecorder() + h.handleAutoIncVotes(rec, httptest.NewRequest(http.MethodGet, "/cluster/autoinc/votes", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + return rec.Body.String() + } + if body := votesHeld(); !strings.Contains(body, `"votes_held":true`) { + t.Fatalf("a fresh node must report its votes held: %s", body) + } + if _, err := dm.GetSystemDatabase().GetWriteDB().Exec("DELETE FROM " + db.AutoIncHoldTable); err != nil { + t.Fatal(err) + } + if body := votesHeld(); !strings.Contains(body, `"votes_held":false`) { + t.Fatalf("a released node must report its votes unheld: %s", body) + } +} diff --git a/admin/cluster.go b/admin/cluster.go index 2cc3668a..64fdff1b 100644 --- a/admin/cluster.go +++ b/admin/cluster.go @@ -82,7 +82,15 @@ func (h *AdminHandlers) handleClusterHealth(w http.ResponseWriter, r *http.Reque }, false, "") } -// handleClusterReplication handles GET /admin/cluster/replication +// handleClusterReplication handles GET /admin/cluster/replication. It +// reports the log-pull replication state per peer and per database, +// instead of the removed MetaStore.ReplicationState table: for each peer, +// its last-consumed position in this node's log for the database +// (MetaStore.ConsumedPositions), and, separately, this node's own pull +// cursor into that peer's log (MetaStore.GetPullCursor) - the two directions +// of replication progress between this node and the peer. The per-database +// caught-up/stuck status (h.antiEntropyStatus) is not per peer: it is this +// node's own view of the whole database. func (h *AdminHandlers) handleClusterReplication(w http.ResponseWriter, r *http.Request) { registry := h.resolveRegistryOrError(w) if registry == nil { @@ -107,33 +115,26 @@ func (h *AdminHandlers) handleClusterReplication(w http.ResponseWriter, r *http. if err != nil { continue } - metaStore := mdb.GetMetaStore() - state, _ := metaStore.GetReplicationState(member.NodeID, dbName) - localMaxTxn, _ := metaStore.GetMaxCommittedTxnID() - - var lastAppliedTxnID uint64 - var lastSyncTime string - var syncStatus string - if state != nil { - lastAppliedTxnID = state.LastAppliedTxnID - lastSyncTime = formatTimestamp(state.LastSyncTime) - syncStatus = state.SyncStatus.String() - } else { - syncStatus = "UNKNOWN" - } - lag := uint64(0) - if localMaxTxn > lastAppliedTxnID { - lag = localMaxTxn - lastAppliedTxnID + consumed, err := metaStore.ConsumedPositions() + if err != nil { + continue + } + cursor, err := metaStore.GetPullCursor(member.NodeID) + if err != nil { + continue } + consumedPos, hasConsumed := consumed[member.NodeID] + dbStates = append(dbStates, map[string]interface{}{ - "database": dbName, - "last_applied_txn_id": lastAppliedTxnID, - "last_sync_time": lastSyncTime, - "sync_status": syncStatus, - "lag_txns": lag, + "database": dbName, + "peer_consumed_seq": consumedPos.Seq, + "peer_consumed_txn_id": consumedPos.TxnID, + "peer_has_consumed_at_all": hasConsumed, + "local_pull_cursor_seq": cursor.Seq, + "local_pull_cursor_txn_id": cursor.TxnID, }) } @@ -146,12 +147,32 @@ func (h *AdminHandlers) handleClusterReplication(w http.ResponseWriter, r *http. } writeJSONResponse(w, map[string]interface{}{ - "peers": result, - "cluster_min_watermark": registry.GetClusterMinWatermark(), - "local_watermark": registry.GetLocalWatermark(), + "peers": result, + "anti_entropy": h.antiEntropyStatus(databases), }, false, "") } +// antiEntropyStatus reports, per database, anti-entropy's status as of its +// last round: whether every current member's +// log was reachable and fully pulled (CaughtUp), and how many transactions +// are STUCK on a deterministic replay failure (LogPuller.StuckTxns). Empty +// until anti-entropy is wired in (SetAntiEntropy). +func (h *AdminHandlers) antiEntropyStatus(databases []string) []map[string]interface{} { + result := make([]map[string]interface{}, 0, len(databases)) + if h.antiEntropy == nil { + return result + } + for _, dbName := range databases { + result = append(result, map[string]interface{}{ + "database": dbName, + "caught_up": h.antiEntropy.CaughtUp(dbName), + "promotion_ready": h.antiEntropy.PromotionReady(dbName), + "stuck_txns": h.antiEntropy.StuckTxnCount(dbName), + }) + } + return result +} + // handleClusterRemove handles POST /admin/cluster/remove/{node_id} func (h *AdminHandlers) handleClusterRemove(w http.ResponseWriter, r *http.Request) { nodeID, err := parsePeerNodeID(chi.URLParam(r, "nodeID")) diff --git a/admin/handlers.go b/admin/handlers.go index 2d80c631..e32663b1 100644 --- a/admin/handlers.go +++ b/admin/handlers.go @@ -19,8 +19,9 @@ var uiHTML embed.FS // AdminHandlers handles admin API endpoints for MetaStore operations type AdminHandlers struct { - server *grpc.Server - dbManager *db.DatabaseManager + server *grpc.Server + dbManager *db.DatabaseManager + antiEntropy *grpc.AntiEntropyService // set once anti-entropy is constructed (SetAntiEntropy); nil until then } // NewAdminHandlers creates a new AdminHandlers instance @@ -31,6 +32,16 @@ func NewAdminHandlers(server *grpc.Server, dbManager *db.DatabaseManager) *Admin } } +// SetAntiEntropy wires the running AntiEntropyService into the admin +// surface, so cluster/replication status can report per-database caught-up +// state and stuck transactions. marmot.go calls +// this once anti-entropy is constructed, which happens after +// NewAdminHandlers (anti-entropy needs the DatabaseManager and replication +// handler that are wired after admin routes are registered). +func (h *AdminHandlers) SetAntiEntropy(ae *grpc.AntiEntropyService) { + h.antiEntropy = ae +} + // ServeUI serves the admin UI HTML file func (h *AdminHandlers) ServeUI(w http.ResponseWriter, r *http.Request) { // Serve the embedded HTML file diff --git a/admin/replication.go b/admin/replication.go index f2a4fdd5..6ab1842a 100644 --- a/admin/replication.go +++ b/admin/replication.go @@ -6,53 +6,69 @@ import ( "github.com/maxpert/marmot/db" ) -// handleReplicationByPeer returns replication state for specific peer +// handleReplicationByPeer reports database's log-pull replication state as seen +// from peerNodeID's side: peerNodeID's +// last-consumed position in this node's log (MetaStore.ConsumedPositions), +// and this node's own pull cursor into peerNodeID's log +// (MetaStore.GetPullCursor). It replaces the removed +// MetaStore.ReplicationState table. func (h *AdminHandlers) handleReplicationByPeer(w http.ResponseWriter, r *http.Request, metaStore db.MetaStore, peerNodeID uint64, database string) { - rec, err := metaStore.GetReplicationState(peerNodeID, database) + consumed, err := metaStore.ConsumedPositions() if err != nil { writeErrorResponse(w, http.StatusInternalServerError, err.Error()) return } - - if rec == nil { - writeErrorResponse(w, http.StatusNotFound, "replication state not found") + cursor, err := metaStore.GetPullCursor(peerNodeID) + if err != nil { + writeErrorResponse(w, http.StatusInternalServerError, err.Error()) return } + consumedPos, hasConsumed := consumed[peerNodeID] response := map[string]interface{}{ - "peer_node_id": rec.PeerNodeID, - "database_name": rec.DatabaseName, - "last_applied_txn_id": rec.LastAppliedTxnID, - "last_applied_ts_wall": rec.LastAppliedTSWall, - "last_applied_ts_logical": rec.LastAppliedTSLogical, - "last_sync_time": formatTimestamp(rec.LastSyncTime), - "sync_status": rec.SyncStatus, + "peer_node_id": peerNodeID, + "database_name": database, + "peer_consumed_seq": consumedPos.Seq, + "peer_consumed_txn_id": consumedPos.TxnID, + "peer_has_consumed_at_all": hasConsumed, + "local_pull_cursor_seq": cursor.Seq, + "local_pull_cursor_txn_id": cursor.TxnID, } - writeJSONResponse(w, response, false, "") } -// handleReplicationAll returns all replication states -func (h *AdminHandlers) handleReplicationAll(w http.ResponseWriter, r *http.Request, metaStore db.MetaStore) { - states, err := metaStore.GetAllReplicationStates() +// handleReplicationAll reports database's log-pull replication state for every +// peer that has ever pulled from this node (MetaStore.ConsumedPositions), +// plus this node's own GC truncation point (MetaStore.TruncatedThrough): the +// highest log position this node's GC has deleted through, which bounds how +// far behind a peer can safely fall before it needs a snapshot restore +// instead of a log pull. +func (h *AdminHandlers) handleReplicationAll(w http.ResponseWriter, r *http.Request, metaStore db.MetaStore, database string) { + consumed, err := metaStore.ConsumedPositions() + if err != nil { + writeErrorResponse(w, http.StatusInternalServerError, err.Error()) + return + } + truncated, err := metaStore.TruncatedThrough() if err != nil { writeErrorResponse(w, http.StatusInternalServerError, err.Error()) return } - var response []map[string]interface{} - for _, rec := range states { - item := map[string]interface{}{ - "peer_node_id": rec.PeerNodeID, - "database_name": rec.DatabaseName, - "last_applied_txn_id": rec.LastAppliedTxnID, - "last_applied_ts_wall": rec.LastAppliedTSWall, - "last_applied_ts_logical": rec.LastAppliedTSLogical, - "last_sync_time": formatTimestamp(rec.LastSyncTime), - "sync_status": rec.SyncStatus, - } - response = append(response, item) + peers := make([]map[string]interface{}, 0, len(consumed)) + for peerNodeID, pos := range consumed { + peers = append(peers, map[string]interface{}{ + "peer_node_id": peerNodeID, + "peer_consumed_seq": pos.Seq, + "peer_consumed_txn_id": pos.TxnID, + }) } + response := map[string]interface{}{ + "database_name": database, + "truncated_through_seq": truncated.Seq, + "truncated_through_txn_id": truncated.TxnID, + "peers": peers, + } writeJSONResponse(w, response, false, "") } diff --git a/admin/replication_state_test.go b/admin/replication_state_test.go new file mode 100644 index 00000000..03ce5866 --- /dev/null +++ b/admin/replication_state_test.go @@ -0,0 +1,104 @@ +package admin + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/maxpert/marmot/db" +) + +// newTestMetaStore creates a real db.MetaStore backed by a temp directory, +// for exercising the admin replication handlers directly. +func newTestMetaStore(t *testing.T) db.MetaStore { + t.Helper() + ms, err := db.NewMetaStore(t.TempDir() + "/replication_state") + if err != nil { + t.Fatalf("NewMetaStore failed: %v", err) + } + t.Cleanup(func() { _ = ms.Close() }) + return ms +} + +// TestHandleReplicationByPeerReportsLogPullState: the admin per-peer +// replication endpoint must report the log-pull state - the peer's last-consumed position in this +// node's log, and this node's own pull cursor into the peer's log - not the +// removed MetaStore.ReplicationState table. +func TestHandleReplicationByPeerReportsLogPullState(t *testing.T) { + metaStore := newTestMetaStore(t) + if err := metaStore.SetConsumedPosition(2, db.LogPosition{Seq: 7, TxnID: 700}); err != nil { + t.Fatalf("SetConsumedPosition failed: %v", err) + } + if err := metaStore.SetPullCursor(2, db.LogPosition{Seq: 3, TxnID: 300}); err != nil { + t.Fatalf("SetPullCursor failed: %v", err) + } + + h := &AdminHandlers{} + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/replication/state/2", nil) + h.handleReplicationByPeer(rec, req, metaStore, 2, "app") + + if rec.Code != http.StatusOK { + t.Fatalf("status %d, want 200; body: %s", rec.Code, rec.Body.String()) + } + + var envelope map[string]interface{} + if err := json.Unmarshal(rec.Body.Bytes(), &envelope); err != nil { + t.Fatalf("failed to decode response: %v; body: %s", err, rec.Body.String()) + } + body, ok := envelope["data"].(map[string]interface{}) + if !ok { + t.Fatalf("expected a data object, got: %s", rec.Body.String()) + } + if got, want := body["peer_consumed_seq"], float64(7); got != want { + t.Errorf("peer_consumed_seq = %v, want %v", got, want) + } + if got, want := body["peer_consumed_txn_id"], float64(700); got != want { + t.Errorf("peer_consumed_txn_id = %v, want %v", got, want) + } + if got, want := body["local_pull_cursor_seq"], float64(3); got != want { + t.Errorf("local_pull_cursor_seq = %v, want %v", got, want) + } + if got, want := body["local_pull_cursor_txn_id"], float64(300); got != want { + t.Errorf("local_pull_cursor_txn_id = %v, want %v", got, want) + } +} + +// TestHandleReplicationAllReportsConsumedPositionsAndTruncation: every peer's last-consumed position, and the database's own GC +// truncation point, in place of the removed replication-state table. +func TestHandleReplicationAllReportsConsumedPositionsAndTruncation(t *testing.T) { + metaStore := newTestMetaStore(t) + if err := metaStore.SetConsumedPosition(5, db.LogPosition{Seq: 11, TxnID: 1100}); err != nil { + t.Fatalf("SetConsumedPosition failed: %v", err) + } + + h := &AdminHandlers{} + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/replication/states", nil) + h.handleReplicationAll(rec, req, metaStore, "app") + + if rec.Code != http.StatusOK { + t.Fatalf("status %d, want 200; body: %s", rec.Code, rec.Body.String()) + } + + var envelope map[string]interface{} + if err := json.Unmarshal(rec.Body.Bytes(), &envelope); err != nil { + t.Fatalf("failed to decode response: %v; body: %s", err, rec.Body.String()) + } + body, ok := envelope["data"].(map[string]interface{}) + if !ok { + t.Fatalf("expected a data object, got: %s", rec.Body.String()) + } + peers, ok := body["peers"].([]interface{}) + if !ok || len(peers) != 1 { + t.Fatalf("expected exactly one peer in the response, got %v", body["peers"]) + } + peer := peers[0].(map[string]interface{}) + if got, want := peer["peer_node_id"], float64(5); got != want { + t.Errorf("peer_node_id = %v, want %v", got, want) + } + if got, want := peer["peer_consumed_seq"], float64(11); got != want { + t.Errorf("peer_consumed_seq = %v, want %v", got, want) + } +} diff --git a/admin/routes.go b/admin/routes.go index 2f2a0825..81dac50a 100644 --- a/admin/routes.go +++ b/admin/routes.go @@ -24,6 +24,7 @@ func RegisterRoutes(mux *http.ServeMux, handlers *AdminHandlers) { r.Use(chiAuthMiddleware) r.Get("/members", handlers.handleClusterMembers) r.Get("/health", handlers.handleClusterHealth) + r.Get("/autoinc/votes", handlers.handleAutoIncVotes) r.Get("/replication", handlers.handleClusterReplication) r.Post("/remove/{nodeID}", handlers.handleClusterRemove) r.Post("/allow/{nodeID}", handlers.handleClusterAllow) @@ -198,7 +199,7 @@ func (h *AdminHandlers) replicationStates(w http.ResponseWriter, r *http.Request writeErrorResponse(w, http.StatusNotFound, err.Error()) return } - h.handleReplicationAll(w, r, metaStore) + h.handleReplicationAll(w, r, metaStore, database) } func (h *AdminHandlers) replicationStatePeer(w http.ResponseWriter, r *http.Request) { @@ -219,20 +220,9 @@ func (h *AdminHandlers) replicationStatePeer(w http.ResponseWriter, r *http.Requ // Schema handlers func (h *AdminHandlers) schemaVersion(w http.ResponseWriter, r *http.Request) { database := chi.URLParam(r, "database") - metaStore, err := h.getMetaStore(database) - if err != nil { - writeErrorResponse(w, http.StatusNotFound, err.Error()) - return - } - h.handleSchemaDatabase(w, r, metaStore, database) + h.handleSchemaDatabase(w, r, database) } func (h *AdminHandlers) schemaVersions(w http.ResponseWriter, r *http.Request) { - database := chi.URLParam(r, "database") - metaStore, err := h.getMetaStore(database) - if err != nil { - writeErrorResponse(w, http.StatusNotFound, err.Error()) - return - } - h.handleSchemaAll(w, r, metaStore) + h.handleSchemaAll(w, r) } diff --git a/admin/schema.go b/admin/schema.go index 539cdd68..b8086be5 100644 --- a/admin/schema.go +++ b/admin/schema.go @@ -7,10 +7,10 @@ import ( ) // handleSchemaDatabase returns schema version for current database -func (h *AdminHandlers) handleSchemaDatabase(w http.ResponseWriter, r *http.Request, metaStore db.MetaStore, database string) { - version, err := metaStore.GetSchemaVersion(database) +func (h *AdminHandlers) handleSchemaDatabase(w http.ResponseWriter, r *http.Request, database string) { + version, err := db.NewSchemaVersionManager(h.dbManager).GetSchemaVersion(database) if err != nil { - writeErrorResponse(w, http.StatusInternalServerError, err.Error()) + writeErrorResponse(w, http.StatusNotFound, err.Error()) return } @@ -23,8 +23,8 @@ func (h *AdminHandlers) handleSchemaDatabase(w http.ResponseWriter, r *http.Requ } // handleSchemaAll returns schema versions for all databases -func (h *AdminHandlers) handleSchemaAll(w http.ResponseWriter, r *http.Request, metaStore db.MetaStore) { - versions, err := metaStore.GetAllSchemaVersions() +func (h *AdminHandlers) handleSchemaAll(w http.ResponseWriter, r *http.Request) { + versions, err := db.NewSchemaVersionManager(h.dbManager).GetAllSchemaVersions() if err != nil { writeErrorResponse(w, http.StatusInternalServerError, err.Error()) return diff --git a/cfg/config.go b/cfg/config.go index 98f69fb3..b67c809c 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -64,6 +64,13 @@ type ReplicationConfiguration struct { GCMinRetentionHours int `toml:"gc_min_retention_hours"` GCMaxRetentionHours int `toml:"gc_max_retention_hours"` StreamChunkSizeKB int `toml:"stream_chunk_size_kb"` // Size in KB for streaming chunks (default: 1024 = 1MB) + // SnapshotRestoreTimeoutS bounds anti-entropy's whole snapshot restore + // sequence (transfer, re-attach, and the local-log re-apply), + // independent of anti_entropy_interval_seconds: a restore is a one-shot + // transfer of a database's full current size, which can take far longer + // than one anti-entropy round, while the round interval is tuned for + // pulling a bounded page of log entries. + SnapshotRestoreTimeoutS int `toml:"snapshot_restore_timeout_seconds"` } // MySQLConfiguration for MySQL wire protocol server @@ -305,6 +312,7 @@ var Config = &Configuration{ GCMinRetentionHours: 2, // 2 hours - MUST be >= 2x delta threshold (safety margin) GCMaxRetentionHours: 24, // 24 hours - 24x delta threshold (like Cassandra's 10-day gc_grace) StreamChunkSizeKB: 1024, // 1MB - use streaming for large payloads >= this size + SnapshotRestoreTimeoutS: 1800, // 30 minutes - bounds a full database transfer, independent of the AE round interval }, Transaction: TransactionConfiguration{ diff --git a/coordinator/commit_lost_pinned_state_test.go b/coordinator/commit_lost_pinned_state_test.go index 80ab71ff..171ad0a4 100644 --- a/coordinator/commit_lost_pinned_state_test.go +++ b/coordinator/commit_lost_pinned_state_test.go @@ -59,9 +59,9 @@ func setupLostPinnedState(t *testing.T) *lostPinnedStateSetup { const dbName = "lostpinned" require.NoError(t, dbMgr.CreateDatabase(dbName)) - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) + _, err = dbMgr.GetDatabase(db.SystemDatabaseName) require.NoError(t, err) - schemaVersionMgr := db.NewSchemaVersionManager(systemDB.GetMetaStore()) + schemaVersionMgr := db.NewSchemaVersionManager(dbMgr) nodeProvider := coordinator.NewMockNodeProvider([]uint64{1}) // A claim participant counts the cluster as the claimant does. diff --git a/coordinator/database_registry_generation_test.go b/coordinator/database_registry_generation_test.go new file mode 100644 index 00000000..6100c9ab --- /dev/null +++ b/coordinator/database_registry_generation_test.go @@ -0,0 +1,130 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package coordinator_test + +import ( + "testing" + "time" + + "github.com/maxpert/marmot/coordinator" + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// setupDatabaseOpHandler builds a single-node CoordinatorHandler over a real +// DatabaseManager, so CREATE/DROP DATABASE runs the real stamping +// (handleMutation), PREPARE gate (ReplicationEngine.prepareDatabaseOperation) +// and COMMIT merge (DatabaseManager.ApplyDatabaseOp) - not a mock of any of +// them. The single node's own write goes through db.LocalReplicator, which +// calls the real ReplicationEngine in-process. +func setupDatabaseOpHandler(t testing.TB) (*coordinator.CoordinatorHandler, *db.DatabaseManager, *protocol.ConnectionSession) { + t.Helper() + + tmpDir := t.TempDir() + clock := hlc.NewClock(1) + + dbMgr, err := db.NewDatabaseManager(tmpDir, 1, clock) + require.NoError(t, err) + t.Cleanup(func() { dbMgr.Close() }) + require.NoError(t, dbMgr.MergeAutoIncBasesAndReleaseVotes(nil)) + + _, err = dbMgr.GetDatabase(db.SystemDatabaseName) + require.NoError(t, err) + schemaVersionMgr := db.NewSchemaVersionManager(dbMgr) + + nodeProvider := coordinator.NewMockNodeProvider([]uint64{1}) + dbMgr.SetClusterMembership(nodeProvider.GetTotalMembershipSize) + + writeCoord := coordinator.NewWriteCoordinator( + 1, + nodeProvider, + &countingReplicator{}, + db.NewLocalReplicator(1, dbMgr, clock), + 10*time.Second, + clock, + ) + readCoord := coordinator.NewReadCoordinator(1, nodeProvider, db.NewLocalReader(dbMgr), 10*time.Second) + + handler := coordinator.NewCoordinatorHandler( + 1, writeCoord, readCoord, clock, dbMgr, + coordinator.NewDDLLockManager(30*time.Second), + schemaVersionMgr, + noopNodeRegistry{}, + ) + + session := &protocol.ConnectionSession{ConnID: 1, TranspilationEnabled: true} + return handler, dbMgr, session +} + +// TestCoordinatorStampsDatabaseGeneration_CreateAbsentIsGenerationOne pins +// that CREATE DATABASE on a name never seen stamps and commits generation 1, +// live. +func TestCoordinatorStampsDatabaseGeneration_CreateAbsentIsGenerationOne(t *testing.T) { + handler, dbMgr, session := setupDatabaseOpHandler(t) + + _, err := handler.HandleQuery(session, "CREATE DATABASE gentest1", nil) + require.NoError(t, err) + + key, err := dbMgr.RegistryKey("gentest1") + require.NoError(t, err) + require.Equal(t, db.DatabaseRegistryKey{Generation: 1, Dropped: false}, key) +} + +// TestCoordinatorStampsDatabaseGeneration_CreateLiveIsNoOpGeneration pins that +// a second CREATE DATABASE on an already-live name stamps and commits the +// same generation (a no-op), never advancing it. +func TestCoordinatorStampsDatabaseGeneration_CreateLiveIsNoOpGeneration(t *testing.T) { + handler, dbMgr, session := setupDatabaseOpHandler(t) + + _, err := handler.HandleQuery(session, "CREATE DATABASE gentest2", nil) + require.NoError(t, err) + + _, err = handler.HandleQuery(session, "CREATE DATABASE gentest2", nil) + require.NoError(t, err) + + key, err := dbMgr.RegistryKey("gentest2") + require.NoError(t, err) + require.Equal(t, db.DatabaseRegistryKey{Generation: 1, Dropped: false}, key) +} + +// TestCoordinatorStampsDatabaseGeneration_DropIsCurrentGeneration pins that +// DROP DATABASE stamps and commits the database's current generation, +// tombstoned. +func TestCoordinatorStampsDatabaseGeneration_DropIsCurrentGeneration(t *testing.T) { + handler, dbMgr, session := setupDatabaseOpHandler(t) + + _, err := handler.HandleQuery(session, "CREATE DATABASE gentest3", nil) + require.NoError(t, err) + + _, err = handler.HandleQuery(session, "DROP DATABASE gentest3", nil) + require.NoError(t, err) + + key, err := dbMgr.RegistryKey("gentest3") + require.NoError(t, err) + require.Equal(t, db.DatabaseRegistryKey{Generation: 1, Dropped: true}, key) + require.False(t, dbMgr.DatabaseExists("gentest3")) +} + +// TestCoordinatorStampsDatabaseGeneration_CreateTombstonedIsNextGeneration +// pins that CREATE DATABASE on a tombstoned name stamps and commits one +// generation above the tombstone, live - so the re-created database's +// generation always outranks the drop it followed. +func TestCoordinatorStampsDatabaseGeneration_CreateTombstonedIsNextGeneration(t *testing.T) { + handler, dbMgr, session := setupDatabaseOpHandler(t) + + _, err := handler.HandleQuery(session, "CREATE DATABASE gentest4", nil) + require.NoError(t, err) + _, err = handler.HandleQuery(session, "DROP DATABASE gentest4", nil) + require.NoError(t, err) + + _, err = handler.HandleQuery(session, "CREATE DATABASE gentest4", nil) + require.NoError(t, err) + + key, err := dbMgr.RegistryKey("gentest4") + require.NoError(t, err) + require.Equal(t, db.DatabaseRegistryKey{Generation: 2, Dropped: false}, key) + require.True(t, dbMgr.DatabaseExists("gentest4")) +} diff --git a/coordinator/errors.go b/coordinator/errors.go index 084b5ff0..abb2f613 100644 --- a/coordinator/errors.go +++ b/coordinator/errors.go @@ -1,8 +1,10 @@ package coordinator import ( + "errors" "fmt" + "github.com/maxpert/marmot/protocol/mysqlcode" "github.com/maxpert/marmot/protocol/query/transform" ) @@ -138,3 +140,44 @@ func (e *PartialCommitError) Error() string { return fmt.Sprintf("partial commit: got %d remote commit acks, needed %d (some nodes may have committed)", e.RemoteAcks, e.RemoteQuorumNeeded) } + +// ErrLegacyMembersPresent is what LegacyMembersDDLRefusal matches with +// errors.Is: DDL or CREATE/DROP DATABASE refused because a cluster member does +// not serve the commit-log pull protocol yet. +var ErrLegacyMembersPresent = errors.New("DDL refused while a cluster member does not serve the commit-log pull protocol") + +// LegacyMembersDDLRefusal refuses DDL and CREATE/DROP DATABASE cluster-wide +// while LegacyIDs names members whose log protocol version is below the one +// this binary serves (grpc.LegacyLogProtocolMembers). Such a member counts a +// database's DDL history differently, so a schema version bumped by DDL run +// now would never match between the two kinds of node, and every write +// coordinated across them would then be refused at the schema-version gates. +// The refusal is the same on the coordinator (coordinator.handleMutation) and +// on a participant's PREPARE (grpc.ReplicationHandler), so an old coordinator's +// DDL cannot reach a quorum either. +// +// It carries 1213 ER_LOCK_DEADLOCK / SQLSTATE 40001 (through its unwrapped +// *transform.CodedError), the code this codebase pairs with "nothing was +// written, safe to retry the whole transaction" (see mapSQLiteError's +// ErrConstraintCommitHook case in protocol/error_mapper.go): MySQL clients and +// ORMs already treat 1213 as retryable, while 1105 carries no such +// convention. +type LegacyMembersDDLRefusal struct { + LegacyIDs []uint64 +} + +// NewLegacyMembersDDLRefusal returns the refusal naming legacyIDs. +func NewLegacyMembersDDLRefusal(legacyIDs []uint64) *LegacyMembersDDLRefusal { + return &LegacyMembersDDLRefusal{LegacyIDs: legacyIDs} +} + +func (e *LegacyMembersDDLRefusal) Error() string { + return fmt.Sprintf("DDL and CREATE/DROP DATABASE are refused cluster-wide: node(s) %v do not serve the "+ + "commit-log pull protocol yet (an older Marmot release); upgrade every node, then retry", e.LegacyIDs) +} + +// Unwrap exposes ErrLegacyMembersPresent for errors.Is and the coded MySQL +// error for protocol.ConvertToMySQLError. +func (e *LegacyMembersDDLRefusal) Unwrap() []error { + return []error{ErrLegacyMembersPresent, transform.NewCodedError(mysqlcode.ErrCodeDeadlock, "%s", e.Error())} +} diff --git a/coordinator/handler.go b/coordinator/handler.go index aeafd9a9..ca62c240 100644 --- a/coordinator/handler.go +++ b/coordinator/handler.go @@ -61,6 +61,15 @@ type DatabaseManager interface { GetTranspilerSchema(database, table string) (*transform.SchemaInfo, error) // GetVectorIndexManager returns the vector index manager (may be nil). GetVectorIndexManager() VectorIndexManagerProvider + // RegistryKeyGeneration reports database name's current registry + // generation and whether it is presently live (as opposed to tombstoned + // by a DROP, or never created). handleMutation uses it to stamp a + // CREATE/DROP DATABASE statement's Statement.DatabaseGeneration, which + // fences a stale coordinator at PREPARE (see + // db.DatabaseManager.ApplyDatabaseOp). It + // is db.DatabaseManager.RegistryKey with the same information carried as + // two primitives, to avoid an import cycle with package db. + RegistryKeyGeneration(name string) (generation uint64, live bool, err error) } // ReplicatedDatabaseProvider provides access to replicated database operations @@ -101,7 +110,6 @@ type PendingExecution interface { // SchemaVersionManager interface to avoid import cycles type SchemaVersionManager interface { GetSchemaVersion(database string) (uint64, error) - IncrementSchemaVersion(database string, ddlSQL string, txnID uint64) (uint64, error) GetAllSchemaVersions() (map[string]uint64, error) } @@ -112,6 +120,11 @@ type NodeRegistry interface { GetAll() []any // Returns slice of node states (avoids import cycle) IsLeaving(nodeID uint64) bool GetLocalNodeID() uint64 + // LegacyLogProtocolMembers returns the ids of current members (self + // included) running an older release that does not serve the commit-log + // pull protocol, sorted ascending; empty means every member serves it. + // See grpc.NodeRegistry's method of the same name. + LegacyLogProtocolMembers() []uint64 } // NodeState represents cluster node state @@ -637,6 +650,16 @@ func (h *CoordinatorHandler) handleMutation(stmt protocol.Statement, params []in stmt.Type == protocol.StatementCreateDatabase || stmt.Type == protocol.StatementDropDatabase + // Rolling-upgrade gate: while any member does not serve the commit-log + // pull protocol yet, DDL and CREATE/DROP DATABASE are refused cluster-wide, + // retryable, until every member is upgraded (LegacyMembersDDLRefusal). + if isDDL && h.nodeRegistry != nil { + if legacy := h.nodeRegistry.LegacyLogProtocolMembers(); len(legacy) > 0 { + telemetry.QueriesTotal.With("ddl", "failed").Inc() + return nil, NewLegacyMembersDDLRefusal(legacy) + } + } + // Rewrite DDL for idempotency (safe to replay) if isDDL { originalSQL := stmt.SQL @@ -667,6 +690,28 @@ func (h *CoordinatorHandler) handleMutation(stmt protocol.Statement, params []in }() } + // CREATE/DROP DATABASE statements carry a fencing generation, stamped from + // this coordinator's own registry view under the cluster-wide DDL lock + // above (so the read is not racing another coordinator's create/drop of + // the same name). A participant's PREPARE gate refuses a stamp below its + // own local key - see db.DatabaseManager.ApplyDatabaseOp. A failed read leaves DatabaseGeneration at + // its zero value, which participants treat as "unstamped" and compute + // locally instead of gating on: a safe, graceful degradation. + if stmt.Type == protocol.StatementCreateDatabase || stmt.Type == protocol.StatementDropDatabase { + if h.dbManager != nil { + generation, live, genErr := h.dbManager.RegistryKeyGeneration(stmt.Database) + if genErr != nil { + log.Warn().Err(genErr).Str("database", stmt.Database).Msg("Failed to read database registry; leaving the operation unstamped") + } else if stmt.Type == protocol.StatementDropDatabase { + stmt.DatabaseGeneration = generation + } else if live { + stmt.DatabaseGeneration = generation // already live: CREATE is a no-op at the current generation + } else { + stmt.DatabaseGeneration = generation + 1 + } + } + } + // Get current schema version for this database var schemaVersion uint64 if h.schemaVersionMgr != nil { @@ -801,27 +846,16 @@ func (h *CoordinatorHandler) handleMutation(stmt protocol.Statement, params []in } } - // If DDL succeeded, increment schema version - if isDDL && h.schemaVersionMgr != nil { - newVersion, err := h.schemaVersionMgr.IncrementSchemaVersion(stmt.Database, stmt.SQL, uint64(txnID)) + // The DDL's schema version bump is already durable, atomic with the DDL + // itself, in the database's own SQLite file: TransactionManager + // bumped and cached it as part of committing txn above. Refresh gossip with + // the current versions so peers learn it without waiting for their own poll. + if isDDL && h.schemaVersionMgr != nil && h.nodeRegistry != nil { + allVersions, err := h.schemaVersionMgr.GetAllSchemaVersions() if err != nil { - log.Error().Err(err).Str("database", stmt.Database).Msg("Failed to increment schema version") + log.Error().Err(err).Msg("Failed to get schema versions for gossip") } else { - log.Info(). - Str("database", stmt.Database). - Uint64("new_version", newVersion). - Uint64("txn_id", uint64(txnID)). - Msg("Schema version incremented after DDL") - - // Update gossip with new schema versions - if h.nodeRegistry != nil { - allVersions, err := h.schemaVersionMgr.GetAllSchemaVersions() - if err != nil { - log.Error().Err(err).Msg("Failed to get schema versions for gossip") - } else { - h.nodeRegistry.UpdateSchemaVersions(allVersions) - } - } + h.nodeRegistry.UpdateSchemaVersions(allVersions) } } diff --git a/coordinator/handler_draining_test.go b/coordinator/handler_draining_test.go index dd749252..ed620b49 100644 --- a/coordinator/handler_draining_test.go +++ b/coordinator/handler_draining_test.go @@ -7,10 +7,13 @@ import ( "github.com/maxpert/marmot/protocol" ) -// stubNodeRegistry satisfies coordinator.NodeRegistry with configurable leaving state. +// stubNodeRegistry satisfies coordinator.NodeRegistry with configurable +// leaving state and legacy members (ones not serving the commit-log pull +// protocol yet). type stubNodeRegistry struct { - localNodeID uint64 - leaving bool + localNodeID uint64 + leaving bool + legacyMembers []uint64 } func (s *stubNodeRegistry) UpdateSchemaVersions(_ map[string]uint64) {} @@ -18,6 +21,7 @@ func (s *stubNodeRegistry) CountAlive() int { return 1 func (s *stubNodeRegistry) GetAll() []any { return []any{} } func (s *stubNodeRegistry) IsLeaving(nodeID uint64) bool { return s.leaving && nodeID == s.localNodeID } func (s *stubNodeRegistry) GetLocalNodeID() uint64 { return s.localNodeID } +func (s *stubNodeRegistry) LegacyLogProtocolMembers() []uint64 { return s.legacyMembers } // newDrainingHandler builds a CoordinatorHandler with no real coordinators, // suitable for testing the draining/mutation-rejection path only. diff --git a/coordinator/noop_dml_test.go b/coordinator/noop_dml_test.go index ed977dfa..d8665a14 100644 --- a/coordinator/noop_dml_test.go +++ b/coordinator/noop_dml_test.go @@ -45,6 +45,7 @@ func (noopNodeRegistry) CountAlive() int { return 1 } func (noopNodeRegistry) GetAll() []any { return nil } func (noopNodeRegistry) IsLeaving(uint64) bool { return false } func (noopNodeRegistry) GetLocalNodeID() uint64 { return 1 } +func (noopNodeRegistry) LegacyLogProtocolMembers() []uint64 { return nil } // noopDMLSetup builds a single-node handler over a real DatabaseManager so the // CDC preupdate hook runs for real. @@ -75,9 +76,9 @@ func setupNoopDML(t testing.TB) *noopDMLSetup { const dbName = "noopdml" require.NoError(t, dbMgr.CreateDatabase(dbName)) - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) + _, err = dbMgr.GetDatabase(db.SystemDatabaseName) require.NoError(t, err) - schemaVersionMgr := db.NewSchemaVersionManager(systemDB.GetMetaStore()) + schemaVersionMgr := db.NewSchemaVersionManager(dbMgr) replicator := &countingReplicator{} nodeProvider := coordinator.NewMockNodeProvider([]uint64{1}) diff --git a/coordinator/rolling_upgrade_ddl_gate_test.go b/coordinator/rolling_upgrade_ddl_gate_test.go new file mode 100644 index 00000000..8440dedb --- /dev/null +++ b/coordinator/rolling_upgrade_ddl_gate_test.go @@ -0,0 +1,135 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package coordinator_test + +import ( + "testing" + "time" + + "github.com/maxpert/marmot/coordinator" + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/maxpert/marmot/protocol/mysqlcode" + "github.com/stretchr/testify/require" +) + +// legacyAwareNodeRegistry satisfies coordinator.NodeRegistry with a +// configurable set of members that do not serve the commit-log pull protocol +// yet, for the rolling-upgrade DDL gate. +type legacyAwareNodeRegistry struct { + legacy []uint64 +} + +func (legacyAwareNodeRegistry) UpdateSchemaVersions(map[string]uint64) {} +func (legacyAwareNodeRegistry) CountAlive() int { return 1 } +func (legacyAwareNodeRegistry) GetAll() []any { return nil } +func (legacyAwareNodeRegistry) IsLeaving(uint64) bool { return false } +func (legacyAwareNodeRegistry) GetLocalNodeID() uint64 { return 1 } +func (r legacyAwareNodeRegistry) LegacyLogProtocolMembers() []uint64 { return r.legacy } + +// setupUpgradeGateCoordinator builds a single-node handler over a real +// DatabaseManager, identical to setupNoopDML's construction, except the node +// registry's legacy members are configurable so both the refusal and +// acceptance paths of the rolling-upgrade DDL gate can be exercised end to +// end through CoordinatorHandler.HandleQuery. +func setupUpgradeGateCoordinator(t testing.TB, legacyMembers []uint64) (*coordinator.CoordinatorHandler, *protocol.ConnectionSession) { + t.Helper() + + tmpDir := t.TempDir() + clock := hlc.NewClock(1) + + dbMgr, err := db.NewDatabaseManager(tmpDir, 1, clock) + require.NoError(t, err) + t.Cleanup(func() { dbMgr.Close() }) + require.NoError(t, dbMgr.MergeAutoIncBasesAndReleaseVotes(nil)) + + const dbName = "upgradegate" + require.NoError(t, dbMgr.CreateDatabase(dbName)) + + _, err = dbMgr.GetDatabase(db.SystemDatabaseName) + require.NoError(t, err) + schemaVersionMgr := db.NewSchemaVersionManager(dbMgr) + + replicator := &countingReplicator{} + nodeProvider := coordinator.NewMockNodeProvider([]uint64{1}) + dbMgr.SetClusterMembership(nodeProvider.GetTotalMembershipSize) + + writeCoord := coordinator.NewWriteCoordinator( + 1, + nodeProvider, + replicator, + db.NewLocalReplicator(1, dbMgr, clock), + 10*time.Second, + clock, + ) + readCoord := coordinator.NewReadCoordinator( + 1, + nodeProvider, + db.NewLocalReader(dbMgr), + 10*time.Second, + ) + + handler := coordinator.NewCoordinatorHandler( + 1, + writeCoord, + readCoord, + clock, + dbMgr, + coordinator.NewDDLLockManager(30*time.Second), + schemaVersionMgr, + legacyAwareNodeRegistry{legacy: legacyMembers}, + ) + + session := &protocol.ConnectionSession{ + ConnID: 1, + CurrentDatabase: dbName, + TranspilationEnabled: true, + } + + return handler, session +} + +// TestCoordinator_RefusesDDLWhileALegacyMemberIsPresent pins that a +// coordinator refuses DDL cluster-wide, with the retryable 1213, while any +// member does not serve the commit-log pull protocol yet, naming those +// members: such a node counts a database's DDL history differently, and DDL +// run now would leave the schema versions of the two kinds of node apart for +// good. +func TestCoordinator_RefusesDDLWhileALegacyMemberIsPresent(t *testing.T) { + handler, session := setupUpgradeGateCoordinator(t, []uint64{7, 3}) + + _, err := handler.HandleQuery(session, "CREATE TABLE t (id INTEGER PRIMARY KEY)", nil) + requireLegacyMembersRefusal(t, err, []uint64{7, 3}) +} + +// requireLegacyMembersRefusal asserts err is the rolling-upgrade DDL refusal +// naming legacy, carrying the retryable 1213. +func requireLegacyMembersRefusal(t *testing.T, err error, legacy []uint64) { + t.Helper() + require.ErrorIs(t, err, coordinator.ErrLegacyMembersPresent) + var refusal *coordinator.LegacyMembersDDLRefusal + require.ErrorAs(t, err, &refusal) + require.Equal(t, legacy, refusal.LegacyIDs, "the refusal must name the members still to upgrade") + require.Equal(t, mysqlcode.ErrCodeDeadlock, protocol.ConvertToMySQLError(err).Code) +} + +// TestCoordinator_RefusesCreateDatabaseWhileALegacyMemberIsPresent covers +// the other isDDL branch: CREATE/DROP DATABASE shares the same gate as DDL. +func TestCoordinator_RefusesCreateDatabaseWhileALegacyMemberIsPresent(t *testing.T) { + handler, session := setupUpgradeGateCoordinator(t, []uint64{2}) + + _, err := handler.HandleQuery(session, "CREATE DATABASE newdb", nil) + requireLegacyMembersRefusal(t, err, []uint64{2}) +} + +// TestCoordinator_AllowsDDLWhenEveryMemberServesLogPull proves the gate does +// not misfire on a fully upgraded cluster: DDL proceeds normally when +// LegacyLogProtocolMembers reports no member. +func TestCoordinator_AllowsDDLWhenEveryMemberServesLogPull(t *testing.T) { + handler, session := setupUpgradeGateCoordinator(t, nil) + + _, err := handler.HandleQuery(session, "CREATE TABLE t (id INTEGER PRIMARY KEY)", nil) + require.NoError(t, err, "DDL must proceed once every member reports LogPullProtocolVersion") +} diff --git a/db/applied_txn_marker.go b/db/applied_txn_marker.go index e0c201bf..5aeb352f 100644 --- a/db/applied_txn_marker.go +++ b/db/applied_txn_marker.go @@ -64,14 +64,14 @@ func repairAppliedTxnMetadata(db *sql.DB, metaStore MetaStore, dbName string) er nodeID := uint64(nodeIDRaw) rec, err := metaStore.GetTransaction(txnID) if err != nil { - return err + return fmt.Errorf("read txn %d: %w", txnID, err) } if rec != nil && rec.Status == TxnStatusCommitted { continue } entries, err := metaStore.GetIntentEntries(txnID) if err != nil { - return err + return fmt.Errorf("read captured rows of txn %d: %w", txnID, err) } if len(entries) == 0 { continue @@ -79,11 +79,11 @@ func repairAppliedTxnMetadata(db *sql.DB, metaStore MetaStore, dbName string) er commitTS := hlc.Timestamp{WallTime: wall, Logical: logical, NodeID: nodeID} if rec != nil { if err := metaStore.CommitTransaction(txnID, commitTS, nil, dbName, "", 0, uint32(len(entries))); err != nil { - return err + return fmt.Errorf("commit txn %d: %w", txnID, err) } continue } - if err := metaStore.StoreReplayedTransaction(txnID, nodeID, commitTS, dbName, uint32(len(entries))); err != nil { + if err := metaStore.StoreReplayedTransaction(txnID, nodeID, commitTS, dbName, uint32(len(entries)), 0); err != nil { return err } } diff --git a/db/autoinc_claim.go b/db/autoinc_claim.go index abef149d..f96a4353 100644 --- a/db/autoinc_claim.go +++ b/db/autoinc_claim.go @@ -608,9 +608,38 @@ func applyClaimTx(tx *sql.Tx, database string, intent *WriteIntentRecord) error if err != nil { return fmt.Errorf("write auto-increment base for %s.%s: %w", database, claim.Table, err) } - if n != 1 { - return fmt.Errorf("%w: the claim row for %s.%s is absent or its committed or seed floor has passed the claim's newBase %d", - ErrAutoIncClaimNotApplicable, database, claim.Table, claim.NewBase) + if n == 1 { + return nil } - return nil + applied, err := claimAlreadyApplied(tx, database, claim, intent) + if err != nil { + return err + } + if applied { + return nil + } + return fmt.Errorf("%w: the claim row for %s.%s is absent or its committed or seed floor has passed the claim's newBase %d", + ErrAutoIncClaimNotApplicable, database, claim.Table, claim.NewBase) +} + +// claimAlreadyApplied reports whether the claim row still holds exactly what +// applying this very claim wrote: committed at its end, with its owner and +// grant time. The claim applies in the system database before the +// transaction's user-database commit, so a commit that failed after it (the +// user database's writer busy, a crash) leaves the claim applied and the +// transaction PENDING; committing that transaction again must not be refused +// for the floor its own claim raised. No other claim for the table can have +// been applied since: this transaction's PENDING claim intent still holds the +// claim key's row lock, which every other claim's PREPARE needs first. +func claimAlreadyApplied(tx *sql.Tx, database string, claim protocol.AutoIncClaim, intent *WriteIntentRecord) (bool, error) { + var committed, owner, grantedAt int64 + err := tx.QueryRow("SELECT committed, owner, granted_at FROM "+AutoIncClaimTable+" WHERE db = ? AND tbl = ?", + database, claim.Table).Scan(&committed, &owner, &grantedAt) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, fmt.Errorf("read auto-increment base for %s.%s: %w", database, claim.Table, err) + } + return committed == int64(claim.NewBase+claim.Size) && owner == int64(intent.NodeID) && grantedAt == intent.TSWall, nil } diff --git a/db/autoinc_seed_test.go b/db/autoinc_seed_test.go index 70ddb261..2bde3b10 100644 --- a/db/autoinc_seed_test.go +++ b/db/autoinc_seed_test.go @@ -18,6 +18,12 @@ func newSeedTestDB(t *testing.T) *sql.DB { require.NoError(t, err) db.SetMaxOpenConns(1) t.Cleanup(func() { db.Close() }) + // applyNonDMLIntents writes the applied-txn marker, and bumps + // __marmot_schema_version for real DDL, unconditionally as part of its one + // SQLite tx; a raw *sql.DB like this one otherwise lacks both tables. + require.NoError(t, ensureAppliedTxnTable(db)) + _, err = ensureSchemaVersionTable(db, "testdb", false, nil) + require.NoError(t, err) return db } @@ -247,7 +253,7 @@ func TestApplyNonDMLIntents_SeedsAutoIncBaseAfterCreateTable(t *testing.T) { SQLStatement: `CREATE TABLE orders (id INTEGER /*M:16a*/ PRIMARY KEY, total INTEGER)`, }, } - require.NoError(t, tm.applyNonDMLIntents(1, intents)) + require.NoError(t, tm.applyNonDMLIntents(1, hlc.Timestamp{}, intents)) base, owner, ok := readClaimRow(t, store, "testdb", "orders") require.True(t, ok, "expected applyNonDMLIntents to seed a claim row for the tagged table") @@ -273,7 +279,7 @@ func TestApplyNonDMLIntents_UnmarkedDDLSeedsNothing(t *testing.T) { SQLStatement: `CREATE TABLE plain (id INTEGER PRIMARY KEY, name TEXT)`, }, } - require.NoError(t, tm.applyNonDMLIntents(1, intents)) + require.NoError(t, tm.applyNonDMLIntents(1, hlc.Timestamp{}, intents)) _, _, ok := readClaimRow(t, store, "testdb", "plain") require.False(t, ok) diff --git a/db/cdc_segment_log.go b/db/cdc_segment_log.go index fc5e2980..6ae19b6b 100644 --- a/db/cdc_segment_log.go +++ b/db/cdc_segment_log.go @@ -403,6 +403,18 @@ func (l *cdcSegmentLog) sealTxn(txnID uint64, immutable *TxnImmutableRecord, wai return manifest, nil } +// pendingRowCount reports how many rows txnID has appended and not yet +// sealed, and whether it has any pending manifest at all. +func (l *cdcSegmentLog) pendingRowCount(txnID uint64) (uint64, bool) { + l.mu.Lock() + defer l.mu.Unlock() + m := l.pending[txnID] + if m == nil { + return 0, false + } + return m.RowCount, true +} + func (l *cdcSegmentLog) getPendingManifest(txnID uint64) *cdcSegmentTxnManifest { l.mu.Lock() defer l.mu.Unlock() diff --git a/db/database_manager.go b/db/database_manager.go index 8db0b21c..4b84263d 100644 --- a/db/database_manager.go +++ b/db/database_manager.go @@ -39,25 +39,25 @@ type DatabaseProvider interface { // // Locking: mu guards the maps and the wiring fields and is only ever held for // map access and wiring, never while a database's GC is stopped, because a GC -// pass takes mu (GetMinAppliedTxnID, and anti-entropy's refresh through -// ListDatabases). lifecycleMu serialises the operations that open, close, +// pass takes mu (anti-entropy's refresh through ListDatabases). lifecycleMu +// serialises the operations that open, close, // register or delete a database (Create, Drop, Detach, Attach, Import, Close), // so one of them can close a database outside mu while the others wait; no GC // pass ever takes lifecycleMu. type DatabaseManager struct { - lifecycleMu sync.Mutex - mu sync.RWMutex - databases map[string]*ReplicatedDatabase - detached map[string]*detachedDatabase // user databases out of service for a snapshot restore - systemDB *ReplicatedDatabase - dataDir string - nodeID uint64 - clock *hlc.Clock - refreshReplicationStates RefreshReplicationStatesFunc // Callback to refresh peer states before GC - cdcHub CDCHub // CDC notification hub, can be nil - vecIndexMgr *VectorIndexManager // Optional vector index manager - autoIncClaimStore *AutoIncClaimStore // AUTO_INCREMENT claim store, backed by systemDB - membershipView atomic.Pointer[func() int] // this node's view of total cluster membership + lifecycleMu sync.Mutex + mu sync.RWMutex + databases map[string]*ReplicatedDatabase + detached map[string]*detachedDatabase // user databases out of service for a snapshot restore + systemDB *ReplicatedDatabase + dataDir string + nodeID uint64 + clock *hlc.Clock + cdcHub CDCHub // CDC notification hub, can be nil + vecIndexMgr *VectorIndexManager // Optional vector index manager + autoIncClaimStore *AutoIncClaimStore // AUTO_INCREMENT claim store, backed by systemDB + membershipView atomic.Pointer[func() int] // this node's view of total cluster membership + gcMembershipFunc atomic.Pointer[func() []uint64] // current member node ids (self included, not REMOVED); source for each database's GC safe position } // SetClusterMembership installs view as the source of this node's view of @@ -77,6 +77,16 @@ func (dm *DatabaseManager) ClusterMembership() (int, error) { return (*view)(), nil } +// SetGCMembershipFunc installs view as the source of this node's current +// cluster membership (self included, every node whose registry status is +// not REMOVED) for every database's GC safe deletion position +// (wireGCCoordination). It keeps this package free of a grpc import: the +// caller (marmot.go) closes over grpc's NodeRegistry itself. Until this is +// called, GC treats nothing as safe except entries past max retention. +func (dm *DatabaseManager) SetGCMembershipFunc(view func() []uint64) { + dm.gcMembershipFunc.Store(&view) +} + // DatabaseMetadata represents database registry information type DatabaseMetadata struct { Name string @@ -149,17 +159,28 @@ func (dm *DatabaseManager) initSystemDatabase() error { // Add system database to the databases map so it can be retrieved via GetDatabase() dm.databases[SystemDatabaseName] = systemDB - // Create database registry table + // Create database registry table. generation and dropped implement the + // registry key (DatabaseRegistryKey): a tombstoned DROP keeps its + // row (dropped=1) instead of deleting it, so a later CREATE can fence a + // stale peer with a strictly higher generation. See DatabaseRegistryKey. + // legacy marks a row that carries no real generation history; see + // migrateDatabaseRegistrySchema and ApplyDatabaseOp's doc comment. _, err = systemDB.GetDB().Exec(` CREATE TABLE IF NOT EXISTS __marmot_databases ( name TEXT PRIMARY KEY, created_at INTEGER NOT NULL, - path TEXT NOT NULL + path TEXT NOT NULL, + generation INTEGER NOT NULL DEFAULT 1, + dropped INTEGER NOT NULL DEFAULT 0, + legacy INTEGER NOT NULL DEFAULT 0 ) `) if err != nil { return fmt.Errorf("failed to create database registry table: %w", err) } + if err := migrateDatabaseRegistrySchema(systemDB.GetDB()); err != nil { + return fmt.Errorf("failed to migrate database registry schema: %w", err) + } // Create the AUTO_INCREMENT claim table (db/autoinc_claim.go) and its vote // hold (db/autoinc_vote_hold.go). They live here, in the system database, @@ -176,9 +197,67 @@ func (dm *DatabaseManager) initSystemDatabase() error { return nil } -// loadDatabases loads all databases from the registry +// migrateDatabaseRegistrySchema adds the generation, dropped and legacy +// columns to __marmot_databases when an existing system database predates +// them. It is idempotent: safe to call on +// every startup, including against a brand-new table that already has the +// columns. A row that predates the migration has no generation history of +// its own, so it becomes (generation 1, live) - the same key CREATE DATABASE +// stamps for a name's first-ever generation - and legacy=1: before this +// migration existed, DROP DATABASE deleted its registry row outright, so every row an upgrade finds +// still there was live, and its (1, live) key is a migration default, not a +// stamp from a real CREATE. ApplyDatabaseOp treats a legacy live row +// specially until a real CREATE or DROP stamps the name (createDatabaseAt +// GenerationLocked and dropDatabaseAtGenerationLocked both clear legacy). +func migrateDatabaseRegistrySchema(sqlDB *sql.DB) error { + rows, err := sqlDB.Query("PRAGMA table_info(__marmot_databases)") + if err != nil { + return fmt.Errorf("failed to inspect database registry schema: %w", err) + } + have := make(map[string]bool, 5) + for rows.Next() { + var cid int + var name, colType string + var notNull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &colType, ¬Null, &dflt, &pk); err != nil { + rows.Close() + return fmt.Errorf("failed to read database registry schema: %w", err) + } + have[name] = true + } + if err := rows.Err(); err != nil { + rows.Close() + return fmt.Errorf("failed to read database registry schema: %w", err) + } + rows.Close() + + if !have["generation"] { + if _, err := sqlDB.Exec("ALTER TABLE __marmot_databases ADD COLUMN generation INTEGER NOT NULL DEFAULT 1"); err != nil { + return fmt.Errorf("failed to add generation column to database registry: %w", err) + } + } + if !have["dropped"] { + if _, err := sqlDB.Exec("ALTER TABLE __marmot_databases ADD COLUMN dropped INTEGER NOT NULL DEFAULT 0"); err != nil { + return fmt.Errorf("failed to add dropped column to database registry: %w", err) + } + } + if !have["legacy"] { + // DEFAULT 1: every row already in the table when this ALTER first + // runs predates generation stamps and has no real generation history + // (see this function's doc comment). A row inserted after the column exists + // always states its own legacy value explicitly. + if _, err := sqlDB.Exec("ALTER TABLE __marmot_databases ADD COLUMN legacy INTEGER NOT NULL DEFAULT 1"); err != nil { + return fmt.Errorf("failed to add legacy column to database registry: %w", err) + } + } + return nil +} + +// loadDatabases loads every live database from the registry. Tombstoned rows +// (dropped=1) are excluded: a dropped database is never opened. func (dm *DatabaseManager) loadDatabases() error { - rows, err := dm.systemDB.GetDB().Query("SELECT name, created_at, path FROM __marmot_databases") + rows, err := dm.systemDB.GetDB().Query("SELECT name, created_at, path FROM __marmot_databases WHERE dropped = 0") if err != nil { return fmt.Errorf("failed to query database registry: %w", err) } @@ -206,6 +285,18 @@ func (dm *DatabaseManager) loadDatabases() error { return rows.Err() } +// schemaVersionOptions returns the NewReplicatedDatabase options that name a +// user database and wire its __marmot_schema_version migration read +// from the retiring pebble-stored counter in the system database's MetaStore. +// dm.systemDB must already be assigned; every call site below runs after +// initSystemDatabase. +func (dm *DatabaseManager) schemaVersionOptions(name string) []ReplicatedDatabaseOption { + return []ReplicatedDatabaseOption{ + WithDatabaseName(name), + WithLegacySchemaVersionSource(dm.systemDB.GetMetaStore().GetSchemaVersion), + } +} + // openDatabase opens a database and adds it to the registry // Creates a MetaStore for the database (stored in dbname_meta.pebble/) func (dm *DatabaseManager) openDatabase(name, path string) error { @@ -215,7 +306,7 @@ func (dm *DatabaseManager) openDatabase(name, path string) error { return fmt.Errorf("failed to create meta store for %s: %w", name, err) } - db, err := NewReplicatedDatabase(path, dm.nodeID, dm.clock, metaStore) + db, err := NewReplicatedDatabase(path, dm.nodeID, dm.clock, metaStore, dm.schemaVersionOptions(name)...) if err != nil { metaStore.Close() return fmt.Errorf("failed to open database %s: %w", name, err) @@ -233,19 +324,84 @@ func (dm *DatabaseManager) openDatabase(name, path string) error { // // NOTE: This function must NOT acquire dm.mu as it is called from contexts // that already hold the write lock (CreateDatabase, AttachDatabase, etc.) -// or during single-threaded initialization. Reading refreshReplicationStates -// is safe because the caller either has exclusive access via write lock -// or we're in initialization before any concurrent access is possible. +// or during single-threaded initialization. +// +// GC with partial membership: gcMembershipFunc's +// view of current membership can be only partly populated right after a +// restart, before gossip has re-learned every peer. GC can run against that +// partial view: it then treats a not-yet-gossiped member as absent from +// dm.gcMembershipFunc's list, so its consumed position is deleted +// (SetGCSafePositionFunc's DeleteConsumedPosition loop below) and it no +// longer holds back the safe deletion point. This is never a correctness +// problem - GC's own safe-position computation still only deletes what every +// member it does know about has consumed - but it means that member may find +// its cursor behind this node's truncation point once gossip reports it +// again, forcing a snapshot restore (AntiEntropyService's snapshot fallback) +// where a further log pull round would otherwise +// have sufficed. The cost is one unnecessary snapshot for that member, never +// a lost transaction. func (dm *DatabaseManager) wireGCCoordination(mdb *ReplicatedDatabase, dbName string) { txnMgr := mdb.GetTransactionManager() txnMgr.SetDatabaseName(dbName) - txnMgr.SetMinAppliedTxnIDFunc(dm.GetMinAppliedTxnID) - // Wire refresh function if available (set via SetRefreshReplicationStatesFunc) - // No lock needed: caller either holds write lock or we're in init - if dm.refreshReplicationStates != nil { - txnMgr.SetRefreshReplicationStatesFunc(dm.refreshReplicationStates) - } + // GC's safe deletion position is the min, across every current member + // other than self, of this database's log R[self,m,d] + // (db/meta_store.go's ConsumedPositions). A + // member with no recorded position counts as the zero position, so GC + // deletes nothing for that database until every member has reported at + // least once (bounded by gcMaxRetention regardless). A member no longer + // current has its consumed position deleted so it stops pinning GC. + // + // A member down for longer than gc_max_retention_hours finds its + // consumed position has been superseded by GC's unconditional + // max-retention deletion; it restores by snapshot, which anti-entropy + // does automatically (AntiEntropyService's snapshot fallback). + metaStore := mdb.GetMetaStore() + selfID := dm.nodeID + txnMgr.SetGCSafePositionFunc(func() (LogPosition, bool) { + viewPtr := dm.gcMembershipFunc.Load() + if viewPtr == nil { + return LogPosition{}, false + } + members := (*viewPtr)() + + positions, err := metaStore.ConsumedPositions() + if err != nil { + log.Warn().Err(err).Str("database", dbName).Msg("GC: failed to read consumed positions") + return LogPosition{}, false + } + + memberSet := make(map[uint64]bool, len(members)) + for _, m := range members { + memberSet[m] = true + } + for nodeID := range positions { + if memberSet[nodeID] { + continue + } + if err := metaStore.DeleteConsumedPosition(nodeID); err != nil { + log.Warn().Err(err).Str("database", dbName).Uint64("node_id", nodeID). + Msg("GC: failed to delete consumed position of a departed member") + } + } + + var safe LogPosition + haveOther := false + for _, m := range members { + if m == selfID { + continue + } + pos := positions[m] // zero LogPosition when this member has never reported + if !haveOther || pos.Less(safe) { + safe = pos + } + haveOther = true + } + if !haveOther { + return LogPosition{}, false + } + return safe, true + }) // Wire CDC notifier if available if dm.cdcHub != nil { @@ -280,24 +436,6 @@ func (dm *DatabaseManager) SetAutoIncIncarnationListener(l AutoIncIncarnationLis dm.autoIncClaimStore.SetIncarnationListener(l) } -// SetRefreshReplicationStatesFunc sets the callback for refreshing peer replication states -// This is called by GC before Phase 2 to ensure fresh watermarks before deletion decisions -// The callback is wired to all existing and future TransactionManagers -func (dm *DatabaseManager) SetRefreshReplicationStatesFunc(fn RefreshReplicationStatesFunc) { - dm.mu.Lock() - dm.refreshReplicationStates = fn - dm.mu.Unlock() - - // Wire to all existing databases - dm.mu.RLock() - defer dm.mu.RUnlock() - - for _, mdb := range dm.databases { - txnMgr := mdb.GetTransactionManager() - txnMgr.SetRefreshReplicationStatesFunc(fn) - } -} - // SetCDCHub sets the CDC notification hub and wires it to all existing databases func (dm *DatabaseManager) SetCDCHub(hub CDCHub) { dm.mu.Lock() @@ -362,7 +500,10 @@ func (dm *DatabaseManager) ensureDefaultDatabase() error { return nil } -// CreateDatabase creates a new database with its own MetaStore +// CreateDatabase creates a new database with its own MetaStore. Returns nil +// if the database already exists (idempotent, IF NOT EXISTS semantics). A +// name previously dropped (tombstoned in the registry) is re-created one +// generation above its tombstone; see DatabaseRegistryKey. func (dm *DatabaseManager) CreateDatabase(name string) error { if name == SystemDatabaseName { return fmt.Errorf("cannot create system database") @@ -385,6 +526,39 @@ func (dm *DatabaseManager) CreateDatabase(name string) error { return nil } + local, err := dm.registryKeyRow(name) + if err != nil { + return err + } + return dm.createDatabaseAtGenerationLocked(name, local.Generation+1) +} + +// createDatabaseAtGenerationLocked creates name's file and wires it into +// service, stamping the registry row (name, generation, live). Caller must +// hold lifecycleMu. +// +// If a live local incarnation of name already exists, it is retired first - +// closed, its files removed, its AUTO_INCREMENT incarnation ended - exactly +// as DropDatabase would. This only happens through ApplyDatabaseOp +// reconciling a peer's higher live key: DatabaseRegistryKey's total order +// means a live key above the local one can only follow a DROP this node has +// not learned of yet, so the local incarnation is stale and must not survive +// under the new generation. +func (dm *DatabaseManager) createDatabaseAtGenerationLocked(name string, generation uint64) error { + dm.mu.RLock() + existingDB, exists := dm.databases[name] + detached := dm.detached[name] + dm.mu.RUnlock() + + if detached != nil { + return fmt.Errorf("database %s is out of service for a snapshot restore: %w", name, ErrDatabaseDetached) + } + if exists { + if err := dm.retireLiveIncarnationLocked(name, existingDB); err != nil { + return err + } + } + // Create database file dbPath := filepath.Join("databases", name+".db") fullPath := filepath.Join(dm.dataDir, dbPath) @@ -395,21 +569,32 @@ func (dm *DatabaseManager) CreateDatabase(name string) error { return fmt.Errorf("failed to create meta store: %w", err) } - db, err := NewReplicatedDatabase(fullPath, dm.nodeID, dm.clock, metaStore) + newDB, err := NewReplicatedDatabase(fullPath, dm.nodeID, dm.clock, metaStore, dm.schemaVersionOptions(name)...) if err != nil { metaStore.Close() cleanupMetaStoreFiles(fullPath) return fmt.Errorf("failed to create database file: %w", err) } - // Register in system database + // Register in system database. ON CONFLICT covers re-creating a + // tombstoned name: the row already exists, dropped=1, and this brings it + // back live at the given generation. legacy=0: this is a real CREATE with + // a generation stamp, so any legacy divergence for name is resolved + // (see ApplyDatabaseOp's doc comment). createdAt := time.Now().UnixNano() _, err = dm.systemDB.GetDB().Exec( - "INSERT INTO __marmot_databases (name, created_at, path) VALUES (?, ?, ?)", - name, createdAt, dbPath, + `INSERT INTO __marmot_databases (name, created_at, path, generation, dropped, legacy) + VALUES (?, ?, ?, ?, 0, 0) + ON CONFLICT(name) DO UPDATE SET + created_at = excluded.created_at, + path = excluded.path, + generation = excluded.generation, + dropped = 0, + legacy = 0`, + name, createdAt, dbPath, generation, ) if err != nil { - db.Close() + newDB.Close() os.Remove(fullPath) cleanupMetaStoreFiles(fullPath) return fmt.Errorf("failed to register database in system: %w", err) @@ -418,10 +603,39 @@ func (dm *DatabaseManager) CreateDatabase(name string) error { // Wire up GC coordination only once the database is registered: until // then its GC cannot reach mu, so the failure path above can close it. dm.mu.Lock() - dm.wireGCCoordination(db, name) - dm.databases[name] = db + dm.wireGCCoordination(newDB, name) + dm.databases[name] = newDB + dm.mu.Unlock() + log.Info().Str("name", name).Str("path", dbPath).Uint64("generation", generation).Msg("Database created") + return nil +} + +// retireLiveIncarnationLocked closes and removes a live local database +// incarnation that a newer registry generation is about to replace, and ends +// its AUTO_INCREMENT incarnation. Caller must hold lifecycleMu. +func (dm *DatabaseManager) retireLiveIncarnationLocked(name string, liveDB *ReplicatedDatabase) error { + var dbPath string + if err := dm.systemDB.GetDB().QueryRow( + "SELECT path FROM __marmot_databases WHERE name = ?", name, + ).Scan(&dbPath); err != nil { + return fmt.Errorf("failed to get database path: %w", err) + } + fullPath := filepath.Join(dm.dataDir, dbPath) + + dm.mu.Lock() + delete(dm.databases, name) dm.mu.Unlock() - log.Info().Str("name", name).Str("path", dbPath).Msg("Database created") + if err := liveDB.Close(); err != nil { + log.Error().Err(err).Str("name", name).Msg("Failed to close a stale database incarnation being replaced by a newer registry generation") + } + removeDatabaseFiles(fullPath) + + dm.mu.RLock() + claimStore := dm.autoIncClaimStore + dm.mu.RUnlock() + if claimStore != nil { + claimStore.databaseIncarnationEnded(name) + } return nil } @@ -445,7 +659,7 @@ func (dm *DatabaseManager) DropDatabase(name string) error { defer dm.lifecycleMu.Unlock() dm.mu.RLock() - db, exists := dm.databases[name] + _, exists := dm.databases[name] detached := dm.detached[name] dm.mu.RUnlock() @@ -455,20 +669,51 @@ func (dm *DatabaseManager) DropDatabase(name string) error { return nil } - // Get path before deletion + local, err := dm.registryKeyRow(name) + if err != nil { + return err + } + return dm.dropDatabaseAtGenerationLocked(name, local.Generation) +} + +// dropDatabaseAtGenerationLocked tombstones name's registry row at +// (generation, dropped) - keeping the row rather than deleting it, so a later +// CREATE can fence a stale peer with a strictly higher generation - and, if +// this node has a local incarnation, retires it: closes and removes its +// files, keeps its AUTO_INCREMENT claim rows, ends its incarnation. +// +// A name this node never created locally is tombstoned all the same: the +// registry row is what ApplyDatabaseOp reconciles against, not local file +// presence, so a node that missed both CREATE and DROP must still end up with +// the tombstone. Caller must hold lifecycleMu. +func (dm *DatabaseManager) dropDatabaseAtGenerationLocked(name string, generation uint64) error { + dm.mu.RLock() + liveDB, exists := dm.databases[name] + detached := dm.detached[name] + dm.mu.RUnlock() + var dbPath string err := dm.systemDB.GetDB().QueryRow( "SELECT path FROM __marmot_databases WHERE name = ?", name, ).Scan(&dbPath) - if err != nil { + if err != nil && !errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("failed to get database path: %w", err) } + if dbPath == "" { + dbPath = filepath.Join("databases", name+".db") + } fullPath := filepath.Join(dm.dataDir, dbPath) - // Remove from registry - _, err = dm.systemDB.GetDB().Exec("DELETE FROM __marmot_databases WHERE name = ?", name) + // legacy=0: this is a real DROP with a generation stamp, so any legacy + // divergence for name is resolved (see ApplyDatabaseOp's doc comment). + _, err = dm.systemDB.GetDB().Exec( + `INSERT INTO __marmot_databases (name, created_at, path, generation, dropped, legacy) + VALUES (?, ?, ?, ?, 1, 0) + ON CONFLICT(name) DO UPDATE SET generation = excluded.generation, dropped = 1, legacy = 0`, + name, time.Now().UnixNano(), dbPath, generation, + ) if err != nil { - return fmt.Errorf("failed to remove database from registry: %w", err) + return fmt.Errorf("failed to tombstone database in registry: %w", err) } // This database's AUTO_INCREMENT claim rows (db/autoinc_claim.go) stay, @@ -502,11 +747,17 @@ func (dm *DatabaseManager) DropDatabase(name string) error { return nil } + if !exists { + log.Info().Str("name", name).Uint64("generation", generation). + Msg("Database tombstoned in registry; no local incarnation to retire") + return nil + } + // Out of the map under mu, closed outside it (see DatabaseManager). dm.mu.Lock() delete(dm.databases, name) dm.mu.Unlock() - if err := db.Close(); err != nil { + if err := liveDB.Close(); err != nil { log.Error().Err(err).Str("name", name).Msg("Failed to close database") } removeDatabaseFiles(fullPath) @@ -526,6 +777,224 @@ func removeDatabaseFiles(fullPath string) { cleanupMetaStoreFiles(fullPath) } +// ErrStaleDatabaseOpCoordinator is returned by ReplicationEngine's PREPARE +// gate (prepareDatabaseOperation) when a CREATE/DROP DATABASE statement's +// stamped DatabaseRegistryKey is below this participant's local key for the +// database: the coordinator's view of the database's history is stale. See +// ApplyDatabaseOp's proof for why refusing it here keeps every peer's +// registry from ever regressing. +var ErrStaleDatabaseOpCoordinator = errors.New("stale coordinator: database operation key is below the local registry key") + +// DatabaseRegistryKey orders one database name's registry history so that a +// higher key always wins a merge. It is ordered +// lexicographically by (Generation, Dropped), with Dropped=true ranking above +// Dropped=false at the same generation, so a DROP always outranks the CREATE +// it follows. A name this node has never seen has key (0, dropped): lower +// than every generation a real CREATE ever stamps, since CREATE always +// stamps generation >= 1. +type DatabaseRegistryKey struct { + Generation uint64 + Dropped bool +} + +// Less reports whether k sorts strictly before other in the registry's total +// order. +func (k DatabaseRegistryKey) Less(other DatabaseRegistryKey) bool { + return k.Compare(other) < 0 +} + +// Compare returns -1, 0, or 1 as k sorts before, equal to, or after other, +// matching the conventions of cmp.Compare. +func (k DatabaseRegistryKey) Compare(other DatabaseRegistryKey) int { + if k.Generation != other.Generation { + if k.Generation < other.Generation { + return -1 + } + return 1 + } + if k.Dropped == other.Dropped { + return 0 + } + if other.Dropped { + return -1 + } + return 1 +} + +// DatabaseRegistryEntry is one database name's current registry key, as +// listed by RegistryEntries. Legacy is true when Key carries no real +// generation history - it is a migration default for a row from before +// generation stamps, not a stamp from an actual CREATE or DROP (see +// ApplyDatabaseOp's doc comment). +type DatabaseRegistryEntry struct { + Name string + Key DatabaseRegistryKey + Legacy bool +} + +// registryKeyRow reads name's raw registry key. It does not take dm.mu: +// dm.systemDB is fixed for the DatabaseManager's lifetime and its *sql.DB is +// safe for concurrent use, so callers that already hold dm.lifecycleMu (the +// Create/Drop/Apply family) can call it directly, while the exported readers +// below take dm.mu themselves for the same lock discipline as this file's +// other exported accessors. +func (dm *DatabaseManager) registryKeyRow(name string) (DatabaseRegistryKey, error) { + var generation uint64 + var dropped bool + err := dm.systemDB.GetDB().QueryRow( + "SELECT generation, dropped FROM __marmot_databases WHERE name = ?", name, + ).Scan(&generation, &dropped) + if errors.Is(err, sql.ErrNoRows) { + return DatabaseRegistryKey{Generation: 0, Dropped: true}, nil + } + if err != nil { + return DatabaseRegistryKey{}, fmt.Errorf("failed to read database registry key for %s: %w", name, err) + } + return DatabaseRegistryKey{Generation: generation, Dropped: dropped}, nil +} + +// RegistryKey returns name's current DatabaseRegistryKey: (0, dropped) when +// the name has never been created on this node. +func (dm *DatabaseManager) RegistryKey(name string) (DatabaseRegistryKey, error) { + dm.mu.RLock() + defer dm.mu.RUnlock() + return dm.registryKeyRow(name) +} + +// RegistryKeyGeneration reports name's current registry generation and +// whether it is presently live. It exists for coordinator.DatabaseManager, +// which cannot import package db's DatabaseRegistryKey without an import +// cycle: it is RegistryKey with the same information carried as two +// primitives instead. The coordinator stamps Statement.DatabaseGeneration +// from it: CREATE stamps generation when +// live (a no-op) or generation+1 otherwise; DROP stamps generation. +func (dm *DatabaseManager) RegistryKeyGeneration(name string) (generation uint64, live bool, err error) { + key, err := dm.RegistryKey(name) + if err != nil { + return 0, false, err + } + return key.Generation, !key.Dropped, nil +} + +// RegistryEntries lists every database name this node's registry has ever +// known, live or tombstoned, excluding the system database. Anti-entropy +// reconciles a peer's listing with this one by merging each entry through +// ApplyDatabaseOp. +func (dm *DatabaseManager) RegistryEntries() ([]DatabaseRegistryEntry, error) { + dm.mu.RLock() + defer dm.mu.RUnlock() + + rows, err := dm.systemDB.GetDB().Query( + "SELECT name, generation, dropped, legacy FROM __marmot_databases WHERE name != ?", SystemDatabaseName, + ) + if err != nil { + return nil, fmt.Errorf("failed to list database registry: %w", err) + } + defer rows.Close() + + var entries []DatabaseRegistryEntry + for rows.Next() { + var e DatabaseRegistryEntry + if err := rows.Scan(&e.Name, &e.Key.Generation, &e.Key.Dropped, &e.Legacy); err != nil { + return nil, fmt.Errorf("failed to read database registry row: %w", err) + } + entries = append(entries, e) + } + return entries, rows.Err() +} + +// ApplyDatabaseOp merges a peer's or a coordinator's DatabaseRegistryKey for +// name into this node's registry, adopting it if and only if it exceeds the +// local key: key.Dropped tombstones name at key.Generation through the same +// path DropDatabase uses; otherwise it creates name live at key.Generation +// through the same path CreateDatabase uses. changed reports whether key was +// adopted. +// +// It is the one merge function both the 2PC COMMIT (ReplicationEngine.Commit) +// and anti-entropy's database-set reconciliation use, and it is what the +// PREPARE gate (prepareDatabaseOperation) fences for by refusing a stamp +// below the local key. +// +// Proof that this converges every node's registry for name, and that a +// dropped database stays dropped: +// 1. Local keys only ever increase - this function is a no-op unless +// key > local - and the merge is a max over a total order +// (DatabaseRegistryKey.Compare), so every node's registry for name +// converges to the same key. +// 2. The last op on name that committed was applied by at least Q nodes, +// 2PC's write quorum. +// 3. Any later op's quorum contains at least one of them. That node's +// PREPARE gate refuses a stamp below its local key, and at most N-Q < Q +// nodes are stale (behind the last committed op), so a stale coordinator +// can never gather a quorum to commit an op with a lower key. +// 4. Hence each committed op's key exceeds every earlier committed op's key +// on name. The maximum over all committed ops is the last one to commit, +// so an older peer, whose key is lower, can never undo a drop by +// resurrecting an earlier live key. +// 5. A CREATE that follows a DROP stamps generation+1, live - strictly above +// the tombstone's (generation, dropped) key - so re-creating a dropped +// database always wins the merge over the tombstone. +// +// Legacy rows and the pre-upgrade divergence: before generation stamps, +// DROP DATABASE deleted the registry row outright, so +// a node that was down for a pre-upgrade DROP still has name's row after +// upgrading, migrated to (1, live) with Legacy=true +// (migrateDatabaseRegistrySchema). That row is a real disagreement about +// whether name exists, not a case this function's merge proof covers - no +// quorum ever stamped (1, live) for it post-upgrade. Reconciling it as an +// ordinary key would let the stale node's peers adopt it and create name +// empty cluster-wide, spreading a divergence that used to stay on one node. +// The caller (reconcileRegistryWithPeer, grpc/anti_entropy.go) is therefore +// responsible for never calling this function with a legacy live entry when +// the local registry has no row for name at all (DatabaseRegistryKey{0, +// dropped}, i.e. this peer never had name): that is the one case where +// applying the key would CREATE the database. Reconciliation does not +// spread this divergence, and it does not repair it either - the node that missed the +// DROP keeps the database until an operator drops it directly. A legacy row +// stops being legacy the moment any post-upgrade CREATE or DROP - local or +// merged - stamps name with a real generation +// (createDatabaseAtGenerationLocked, dropDatabaseAtGenerationLocked), at +// which point ordinary merge semantics apply to it from then on. +func (dm *DatabaseManager) ApplyDatabaseOp(name string, key DatabaseRegistryKey) (bool, error) { + if name == SystemDatabaseName { + return false, fmt.Errorf("cannot apply a database operation to the system database") + } + if key.Dropped && name == DefaultDatabaseName { + return false, fmt.Errorf("cannot drop default database") + } + + if local, err := dm.RegistryKey(name); err != nil { + return false, err + } else if key.Compare(local) <= 0 { + return false, nil + } + + dm.lifecycleMu.Lock() + defer dm.lifecycleMu.Unlock() + + // Re-check under lifecycleMu: a concurrent ApplyDatabaseOp, CreateDatabase + // or DropDatabase may have advanced the local key past key since the + // unlocked read above. + local, err := dm.registryKeyRow(name) + if err != nil { + return false, err + } + if key.Compare(local) <= 0 { + return false, nil + } + + if key.Dropped { + if err := dm.dropDatabaseAtGenerationLocked(name, key.Generation); err != nil { + return false, err + } + return true, nil + } + if err := dm.createDatabaseAtGenerationLocked(name, key.Generation); err != nil { + return false, err + } + return true, nil +} + // GetDatabase returns a database by name func (dm *DatabaseManager) GetDatabase(name string) (*ReplicatedDatabase, error) { dm.mu.RLock() @@ -833,7 +1302,7 @@ func (dm *DatabaseManager) openDetached(name string, metaStore MetaStore) (*Repl ).Scan(&dbPath); err != nil { return nil, fmt.Errorf("failed to get database path: %w", err) } - return NewReplicatedDatabase(filepath.Join(dm.dataDir, dbPath), dm.nodeID, dm.clock, metaStore) + return NewReplicatedDatabase(filepath.Join(dm.dataDir, dbPath), dm.nodeID, dm.clock, metaStore, dm.schemaVersionOptions(name)...) } // DatabasesAwaitingRestore lists the databases whose reattach after a @@ -979,6 +1448,17 @@ func (dm *DatabaseManager) ImportExistingDatabases(importDir string) (int, error // Extract database name (remove .db suffix) dbName := strings.TrimSuffix(name, ".db") + // A name this cluster has dropped must never be silently resurrected + // by an import: the registry, not file presence, is the source of + // truth for whether a database exists (see DatabaseRegistryKey). + if key, err := dm.registryKeyRow(dbName); err != nil { + log.Warn().Err(err).Str("name", dbName).Msg("Failed to read database registry, skipping import") + continue + } else if key.Dropped && key.Generation > 0 { + log.Debug().Str("name", dbName).Msg("Database is tombstoned in the registry, skipping import") + continue + } + // Check if database already exists dm.mu.RLock() existingDB, exists := dm.databases[dbName] @@ -1026,7 +1506,7 @@ func (dm *DatabaseManager) ImportExistingDatabases(importDir string) (int, error } // Open and register the database - db, err := NewReplicatedDatabase(dstPath, dm.nodeID, dm.clock, metaStore) + db, err := NewReplicatedDatabase(dstPath, dm.nodeID, dm.clock, metaStore, dm.schemaVersionOptions(dbName)...) if err != nil { log.Warn().Err(err).Str("name", dbName).Msg("Failed to open imported database") metaStore.Close() @@ -1274,27 +1754,21 @@ func (dm *DatabaseManager) checkpointDatabase(db *ReplicatedDatabase) error { // 1. Acquires write lock to block all writes // 2. Checkpoints all databases (TRUNCATE mode) // 3. Copies all database files to the target directory -// 4. Reads schema versions for the copied databases -// 5. Releases write lock +// 4. Releases write lock // // The caller should stream from the target directory and clean it up when done. // This ensures snapshot consistency since files are copied atomically under lock. // -// Schema versions are read from the system MetaStore in the same locked section -// that produces the copied files, so the returned versions describe exactly the -// bytes being handed back - not a value read by some earlier, separate call that -// a concurrent DDL commit could have moved past. A caller that instead reads -// schema versions via a different, earlier RPC (e.g. GetSnapshotInfo) can -// observe a version that no longer matches the file bytes actually streamed -// later; see SnapshotVersionsForRestore in the grpc package for how that is -// resolved on the receiving side. -func (dm *DatabaseManager) TakeSnapshotToDir(targetDir string) ([]SnapshotInfo, uint64, map[string]uint64, error) { +// Schema versions no longer travel out of band with a snapshot: each +// copied SQLite file carries its own __marmot_schema_version table, so the +// receiver's restored file is authoritative on its own. +func (dm *DatabaseManager) TakeSnapshotToDir(targetDir string) ([]SnapshotInfo, uint64, error) { // Create target directory structure if err := os.MkdirAll(targetDir, 0755); err != nil { - return nil, 0, nil, fmt.Errorf("failed to create snapshot directory: %w", err) + return nil, 0, fmt.Errorf("failed to create snapshot directory: %w", err) } if err := os.MkdirAll(filepath.Join(targetDir, "databases"), 0755); err != nil { - return nil, 0, nil, fmt.Errorf("failed to create databases directory: %w", err) + return nil, 0, fmt.Errorf("failed to create databases directory: %w", err) } // Acquire write lock to block all concurrent writes @@ -1302,7 +1776,7 @@ func (dm *DatabaseManager) TakeSnapshotToDir(targetDir string) ([]SnapshotInfo, defer dm.mu.Unlock() if err := dm.errIfDetachedLocked(); err != nil { - return nil, 0, nil, err + return nil, 0, err } var snapshots []SnapshotInfo @@ -1312,21 +1786,21 @@ func (dm *DatabaseManager) TakeSnapshotToDir(targetDir string) ([]SnapshotInfo, systemTargetPath := filepath.Join(targetDir, SystemDatabaseName+".db") if err := dm.checkpointDatabase(dm.systemDB); err != nil { - return nil, 0, nil, fmt.Errorf("failed to checkpoint system database: %w", err) + return nil, 0, fmt.Errorf("failed to checkpoint system database: %w", err) } if err := copyFile(systemDBPath, systemTargetPath); err != nil { - return nil, 0, nil, fmt.Errorf("failed to copy system database: %w", err) + return nil, 0, fmt.Errorf("failed to copy system database: %w", err) } info, err := os.Stat(systemTargetPath) if err != nil { - return nil, 0, nil, fmt.Errorf("failed to stat system database copy: %w", err) + return nil, 0, fmt.Errorf("failed to stat system database copy: %w", err) } systemSHA256, err := calculateFileSHA256(systemTargetPath) if err != nil { - return nil, 0, nil, fmt.Errorf("failed to hash system database: %w", err) + return nil, 0, fmt.Errorf("failed to hash system database: %w", err) } snapshots = append(snapshots, SnapshotInfo{ @@ -1380,43 +1854,13 @@ func (dm *DatabaseManager) TakeSnapshotToDir(targetDir string) ([]SnapshotInfo, // Get max committed transaction ID (without lock since we already hold it) maxTxnID := dm.getMaxCommittedTxnIDLocked() - // Read schema versions in this same locked section, immediately after the - // files above were checkpointed and copied, so the versions describe - // exactly the bytes being handed back to the caller. - schemaVersions := dm.schemaVersionsLocked() - log.Info(). Int("databases", len(snapshots)). Uint64("max_txn_id", maxTxnID). Str("target_dir", targetDir). Msg("Snapshot copied to directory") - return snapshots, maxTxnID, schemaVersions, nil -} - -// schemaVersionsLocked reads all schema versions from the system MetaStore. -// Caller must hold dm.mu. Returns an empty map when the versions cannot be -// read - the snapshot itself is still usable, the receiver simply has nothing -// to restore and falls back to whatever it already knew. -func (dm *DatabaseManager) schemaVersionsLocked() map[string]uint64 { - metaStore := dm.systemDB.GetMetaStore() - if metaStore == nil { - return nil - } - - stored, err := metaStore.GetAllSchemaVersions() - if err != nil { - log.Warn().Err(err).Msg("Failed to read schema versions for snapshot") - return nil - } - - versions := make(map[string]uint64, len(stored)) - for database, version := range stored { - if version > 0 { - versions[database] = uint64(version) - } - } - return versions + return snapshots, maxTxnID, nil } // TakeSnapshotForDatabase creates a snapshot for a single database. @@ -1537,129 +1981,6 @@ func (dm *DatabaseManager) GetDataDir() string { return dm.dataDir } -// ReplicationState tracks replication progress with a peer node per database -type ReplicationState struct { - PeerNodeID uint64 - DatabaseName string - LastAppliedTxnID uint64 - LastAppliedTSWall int64 - LastAppliedTSLog int32 - LastSyncTime int64 - SyncStatus string // SYNCED, CATCHING_UP, FAILED -} - -// GetReplicationState gets the replication state for a specific peer and database -func (dm *DatabaseManager) GetReplicationState(peerNodeID uint64, database string) (*ReplicationState, error) { - dm.mu.RLock() - defer dm.mu.RUnlock() - - db, ok := dm.databases[database] - if !ok { - return nil, fmt.Errorf("database %s not found", database) - } - - metaStore := db.GetMetaStore() - if metaStore == nil { - return nil, fmt.Errorf("database %s has no meta store", database) - } - - rec, err := metaStore.GetReplicationState(peerNodeID, database) - if err != nil { - return nil, err - } - if rec == nil { - // No replication state yet for this peer/database combination - return nil, nil - } - - return &ReplicationState{ - PeerNodeID: rec.PeerNodeID, - DatabaseName: rec.DatabaseName, - LastAppliedTxnID: rec.LastAppliedTxnID, - LastAppliedTSWall: rec.LastAppliedTSWall, - LastAppliedTSLog: rec.LastAppliedTSLogical, - LastSyncTime: rec.LastSyncTime, - SyncStatus: rec.SyncStatus.String(), - }, nil -} - -// UpdateReplicationState updates or inserts replication state for a peer and database -func (dm *DatabaseManager) UpdateReplicationState(state *ReplicationState) error { - dm.mu.RLock() - defer dm.mu.RUnlock() - - db, ok := dm.databases[state.DatabaseName] - if !ok { - return fmt.Errorf("database %s not found", state.DatabaseName) - } - - metaStore := db.GetMetaStore() - if metaStore == nil { - return fmt.Errorf("database %s has no meta store", state.DatabaseName) - } - - return metaStore.UpdateReplicationState( - state.PeerNodeID, - state.DatabaseName, - state.LastAppliedTxnID, - hlc.Timestamp{WallTime: state.LastAppliedTSWall, Logical: state.LastAppliedTSLog}, - ) -} - -// GetAllReplicationStates returns replication state for all known peers across all databases -func (dm *DatabaseManager) GetAllReplicationStates() ([]ReplicationState, error) { - dm.mu.RLock() - defer dm.mu.RUnlock() - - var allStates []ReplicationState - - // Query each database's MetaStore for its replication state - for _, mdb := range dm.databases { - metaStore := mdb.GetMetaStore() - if metaStore == nil { - continue // System DB has no MetaStore - } - - states, err := metaStore.GetAllReplicationStates() - if err != nil { - continue - } - - for _, rec := range states { - allStates = append(allStates, ReplicationState{ - PeerNodeID: rec.PeerNodeID, - DatabaseName: rec.DatabaseName, - LastAppliedTxnID: rec.LastAppliedTxnID, - LastAppliedTSWall: rec.LastAppliedTSWall, - LastAppliedTSLog: rec.LastAppliedTSLogical, - LastSyncTime: rec.LastSyncTime, - SyncStatus: rec.SyncStatus.String(), - }) - } - } - - return allStates, nil -} - -// GetMinAppliedTxnID returns the minimum last_applied_txn_id across all peers for a specific database -// This is used to determine the GC safe point - we can only GC transactions that all peers have applied -func (dm *DatabaseManager) GetMinAppliedTxnID(database string) (uint64, error) { - dm.mu.RLock() - defer dm.mu.RUnlock() - - db, ok := dm.databases[database] - if !ok { - return 0, fmt.Errorf("database %s not found", database) - } - - metaStore := db.GetMetaStore() - if metaStore == nil { - return 0, nil // System DB has no replication state - } - - return metaStore.GetMinAppliedTxnID(database) -} - // GetMaxTxnID returns the maximum COMMITTED transaction ID in a database // This is used to calculate replication lag and peer selection for anti-entropy // Only committed transactions are considered to ensure consistency with snapshots diff --git a/db/database_manager_test.go b/db/database_manager_test.go index 6b2def64..23ce6ba9 100644 --- a/db/database_manager_test.go +++ b/db/database_manager_test.go @@ -1,6 +1,7 @@ package db import ( + "database/sql" "os" "path/filepath" "strings" @@ -167,17 +168,29 @@ func TestDropDatabase(t *testing.T) { t.Error("Database file was not deleted") } - // Verify not in registry + // The row stays as a tombstone (generation, dropped) rather than being + // deleted - see DatabaseRegistryKey - so a later CREATE can fence a stale + // peer with a strictly higher generation. var count int + var dropped bool err = dm.systemDB.GetDB().QueryRow( - "SELECT COUNT(*) FROM __marmot_databases WHERE name = ?", "testdb", - ).Scan(&count) + "SELECT COUNT(*), dropped FROM __marmot_databases WHERE name = ? GROUP BY dropped", "testdb", + ).Scan(&count, &dropped) if err != nil { - t.Errorf("Failed to query registry: %v", err) + t.Fatalf("Failed to query registry: %v", err) + } + if count != 1 { + t.Errorf("Expected exactly 1 registry row for 'testdb' after drop, got %d", count) + } + if !dropped { + t.Error("Expected 'testdb' row to be tombstoned (dropped=1) after drop") } - if count != 0 { - t.Error("Database still in registry after drop") + // ListDatabases must exclude the tombstone. + for _, name := range dm.ListDatabases() { + if name == "testdb" { + t.Error("ListDatabases still lists a tombstoned database") + } } } @@ -658,70 +671,6 @@ func TestTakeSnapshotForDatabaseMaxTxnID(t *testing.T) { t.Logf("✓ TakeSnapshotForDatabase returns correct max_txn_id") } -// TestGetReplicationStateNilHandling verifies that GetReplicationState handles nil correctly -func TestGetReplicationStateNilHandling(t *testing.T) { - dm, _ := setupTestDatabaseManager(t) - defer dm.Close() - - // Get replication state for a non-existent peer - // This should return nil, nil (not found) rather than panic - state, err := dm.GetReplicationState(999, DefaultDatabaseName) - if err != nil { - t.Fatalf("GetReplicationState returned unexpected error: %v", err) - } - - // State should be nil when no replication state exists - if state != nil { - t.Errorf("Expected nil state for non-existent peer, got %+v", state) - } - - t.Log("✓ GetReplicationState handles nil correctly for non-existent peers") -} - -// TestGetReplicationStateAfterUpdate verifies that replication state can be set and retrieved -func TestGetReplicationStateAfterUpdate(t *testing.T) { - dm, _ := setupTestDatabaseManager(t) - defer dm.Close() - - // Update replication state for a peer - state := &ReplicationState{ - PeerNodeID: 42, - DatabaseName: DefaultDatabaseName, - LastAppliedTxnID: 100, - LastAppliedTSWall: 1234567890, - LastAppliedTSLog: 5, - SyncStatus: "SYNCED", - } - - err := dm.UpdateReplicationState(state) - if err != nil { - t.Fatalf("UpdateReplicationState failed: %v", err) - } - - // Get replication state for the peer - retrieved, err := dm.GetReplicationState(42, DefaultDatabaseName) - if err != nil { - t.Fatalf("GetReplicationState returned error: %v", err) - } - - if retrieved == nil { - t.Fatal("GetReplicationState returned nil for existing peer") - } - - // Verify values - if retrieved.PeerNodeID != 42 { - t.Errorf("Expected PeerNodeID 42, got %d", retrieved.PeerNodeID) - } - if retrieved.LastAppliedTxnID != 100 { - t.Errorf("Expected LastAppliedTxnID 100, got %d", retrieved.LastAppliedTxnID) - } - if retrieved.SyncStatus != "SYNCED" { - t.Errorf("Expected SyncStatus SYNCED, got %s", retrieved.SyncStatus) - } - - t.Log("✓ GetReplicationState retrieves correct values after update") -} - // TestReplica_CreatesOwnSystemDatabase tests that replica creates its own __marmot_system.db func TestReplica_CreatesOwnSystemDatabase(t *testing.T) { tmpDir := t.TempDir() @@ -851,15 +800,19 @@ func TestReplica_DropDatabaseDDL_UpdatesLocalSystemDB(t *testing.T) { t.Fatalf("Failed to drop database: %v", err) } + // The registry row is kept as a tombstone, not deleted (DatabaseRegistryKey). + var dropped bool err = dm.GetSystemDatabase().GetDB().QueryRow( - "SELECT COUNT(*) FROM __marmot_databases WHERE name = ?", "db1", - ).Scan(&count) + "SELECT COUNT(*), dropped FROM __marmot_databases WHERE name = ? GROUP BY dropped", "db1", + ).Scan(&count, &dropped) if err != nil { t.Fatalf("Failed to query system DB after drop: %v", err) } - - if count != 0 { - t.Errorf("Expected 0 rows for 'db1' after drop, got %d", count) + if count != 1 { + t.Errorf("Expected 1 tombstone row for 'db1' after drop, got %d", count) + } + if !dropped { + t.Error("Expected 'db1' row to be tombstoned (dropped=1) after drop") } if dm.DatabaseExists("db1") { @@ -963,7 +916,7 @@ func TestReplica_DoesNotReceiveSystemDBFromPrimary(t *testing.T) { } snapshotDir := filepath.Join(primaryDir, "snapshot") - snapshots, _, _, err := primaryDM.TakeSnapshotToDir(snapshotDir) + snapshots, _, err := primaryDM.TakeSnapshotToDir(snapshotDir) if err != nil { t.Fatalf("Failed to take snapshot: %v", err) } @@ -1046,60 +999,6 @@ func TestReplica_DoesNotReceiveSystemDBFromPrimary(t *testing.T) { t.Log("✓ Replica does not receive system DB from primary during snapshot") } -// TakeSnapshotToDir must return the schema versions it read in the same -// locked section that produced the copied files, so a caller streaming those -// exact bytes later can advertise a version that actually matches them - -// instead of a value read by an earlier, separate call that a concurrent DDL -// commit could move past. -func TestTakeSnapshotToDir_ReturnsSchemaVersionsCapturedWithFiles(t *testing.T) { - dm, tmpDir := setupTestDatabaseManager(t) - defer dm.Close() - - if err := dm.CreateDatabase("appdb"); err != nil { - t.Fatalf("Failed to create appdb: %v", err) - } - - systemMeta := dm.GetSystemDatabase().GetMetaStore() - if err := systemMeta.UpdateSchemaVersion("appdb", 5, "CREATE TABLE t(id INT)", 1); err != nil { - t.Fatalf("Failed to seed schema version: %v", err) - } - - snapshotDir := filepath.Join(tmpDir, "snapshot") - _, _, schemaVersions, err := dm.TakeSnapshotToDir(snapshotDir) - if err != nil { - t.Fatalf("TakeSnapshotToDir failed: %v", err) - } - - if got := schemaVersions["appdb"]; got != 5 { - t.Errorf("Expected schema version 5 for appdb, got %d", got) - } - - // A version bumped AFTER the snapshot was taken must not appear in the - // already-returned map - it describes the files copied above, not - // whatever the live database does next. - if err := systemMeta.UpdateSchemaVersion("appdb", 6, "ALTER TABLE t ADD COLUMN x", 2); err != nil { - t.Fatalf("Failed to bump schema version after snapshot: %v", err) - } - if got := schemaVersions["appdb"]; got != 5 { - t.Errorf("Snapshot's schema versions map must not reflect a later change, got %d", got) - } -} - -func TestTakeSnapshotToDir_NoSchemaVersionsIsEmptyMap(t *testing.T) { - dm, tmpDir := setupTestDatabaseManager(t) - defer dm.Close() - - snapshotDir := filepath.Join(tmpDir, "snapshot") - _, _, schemaVersions, err := dm.TakeSnapshotToDir(snapshotDir) - if err != nil { - t.Fatalf("TakeSnapshotToDir failed: %v", err) - } - - if len(schemaVersions) != 0 { - t.Errorf("Expected no schema versions for a database manager with no DDL applied, got %v", schemaVersions) - } -} - // TestReplica_SnapshotExcludesSystemDB tests that snapshot preparation excludes system DB for replicas func TestReplica_SnapshotExcludesSystemDB(t *testing.T) { dm, tmpDir := setupTestDatabaseManager(t) @@ -1212,3 +1111,370 @@ func TestReplica_SnapshotFileListExcludesSystemDB(t *testing.T) { t.Log("✓ Replica filters system DB from snapshot file list") } + +// TestMigrateDatabaseRegistrySchema_ExistingRowsBecomeLiveGenerationOne pins +// the idempotent migration: a system database +// created before the generation/dropped columns existed gets them added, and +// its existing rows become (generation 1, live) - the same key a first-ever +// CREATE stamps. +func TestMigrateDatabaseRegistrySchema_ExistingRowsBecomeLiveGenerationOne(t *testing.T) { + sqlDB, err := sql.Open(SQLiteDriverName, ":memory:") + if err != nil { + t.Fatalf("Failed to open sqlite: %v", err) + } + defer sqlDB.Close() + + // Pre-migration shape: no generation/dropped columns. + if _, err := sqlDB.Exec(`CREATE TABLE __marmot_databases ( + name TEXT PRIMARY KEY, + created_at INTEGER NOT NULL, + path TEXT NOT NULL + )`); err != nil { + t.Fatalf("Failed to create pre-migration table: %v", err) + } + if _, err := sqlDB.Exec( + "INSERT INTO __marmot_databases (name, created_at, path) VALUES (?, ?, ?)", + "legacy", 123, "databases/legacy.db", + ); err != nil { + t.Fatalf("Failed to seed pre-migration row: %v", err) + } + + if err := migrateDatabaseRegistrySchema(sqlDB); err != nil { + t.Fatalf("migrateDatabaseRegistrySchema failed: %v", err) + } + // Idempotent: a second call over the now-migrated schema must not error. + if err := migrateDatabaseRegistrySchema(sqlDB); err != nil { + t.Fatalf("migrateDatabaseRegistrySchema failed on re-run: %v", err) + } + + var generation int64 + var dropped bool + if err := sqlDB.QueryRow( + "SELECT generation, dropped FROM __marmot_databases WHERE name = ?", "legacy", + ).Scan(&generation, &dropped); err != nil { + t.Fatalf("Failed to read migrated row: %v", err) + } + if generation != 1 { + t.Errorf("Expected migrated row's generation to be 1, got %d", generation) + } + if dropped { + t.Error("Expected migrated row to be live (dropped=0)") + } +} + +// TestCreateDatabase_RecreateAfterDrop_YieldsNextGenerationLive pins that +// CREATE DATABASE on a tombstoned name stamps one generation above the +// tombstone, live. +func TestCreateDatabase_RecreateAfterDrop_YieldsNextGenerationLive(t *testing.T) { + dm, _ := setupTestDatabaseManager(t) + defer dm.Close() + + if err := dm.CreateDatabase("recreatedb"); err != nil { + t.Fatalf("Failed to create database: %v", err) + } + if err := dm.DropDatabase("recreatedb"); err != nil { + t.Fatalf("Failed to drop database: %v", err) + } + if err := dm.CreateDatabase("recreatedb"); err != nil { + t.Fatalf("Failed to re-create database: %v", err) + } + + key, err := dm.RegistryKey("recreatedb") + if err != nil { + t.Fatalf("RegistryKey failed: %v", err) + } + if key != (DatabaseRegistryKey{Generation: 2, Dropped: false}) { + t.Errorf("Expected (2, live) after re-create, got %+v", key) + } + if !dm.DatabaseExists("recreatedb") { + t.Error("Expected re-created database to exist") + } +} + +// TestRegistryEntries_IncludesTombstonesExcludesSystem pins RegistryEntries' +// contract: every name ever seen, live or tombstoned, and never the system +// database. +func TestRegistryEntries_IncludesTombstonesExcludesSystem(t *testing.T) { + dm, _ := setupTestDatabaseManager(t) + defer dm.Close() + + if err := dm.CreateDatabase("liveentry"); err != nil { + t.Fatalf("Failed to create database: %v", err) + } + if err := dm.CreateDatabase("droppedentry"); err != nil { + t.Fatalf("Failed to create database: %v", err) + } + if err := dm.DropDatabase("droppedentry"); err != nil { + t.Fatalf("Failed to drop database: %v", err) + } + + entries, err := dm.RegistryEntries() + if err != nil { + t.Fatalf("RegistryEntries failed: %v", err) + } + + byName := make(map[string]DatabaseRegistryKey, len(entries)) + for _, e := range entries { + if e.Name == SystemDatabaseName { + t.Error("RegistryEntries must exclude the system database") + } + byName[e.Name] = e.Key + } + + if got, ok := byName["liveentry"]; !ok || got != (DatabaseRegistryKey{Generation: 1, Dropped: false}) { + t.Errorf("Expected liveentry (1, live), got %+v (present=%v)", got, ok) + } + if got, ok := byName["droppedentry"]; !ok || got != (DatabaseRegistryKey{Generation: 1, Dropped: true}) { + t.Errorf("Expected droppedentry (1, dropped), got %+v (present=%v)", got, ok) + } +} + +// TestApplyDatabaseOp_HigherKeyApplies verifies ApplyDatabaseOp adopts a key +// strictly above the local one and reports changed=true. +func TestApplyDatabaseOp_HigherKeyApplies(t *testing.T) { + dm, _ := setupTestDatabaseManager(t) + defer dm.Close() + + changed, err := dm.ApplyDatabaseOp("applyhigher", DatabaseRegistryKey{Generation: 1, Dropped: false}) + if err != nil { + t.Fatalf("ApplyDatabaseOp failed: %v", err) + } + if !changed { + t.Error("Expected changed=true for a key above local (never seen)") + } + if !dm.DatabaseExists("applyhigher") { + t.Error("Expected database to be created") + } +} + +// TestApplyDatabaseOp_EqualOrLowerIsNoOp verifies ApplyDatabaseOp is a +// no-op, reporting changed=false, for a key at or below the local one. +func TestApplyDatabaseOp_EqualOrLowerIsNoOp(t *testing.T) { + dm, _ := setupTestDatabaseManager(t) + defer dm.Close() + + if err := dm.CreateDatabase("applyeq"); err != nil { + t.Fatalf("Failed to create database: %v", err) + } + + // Equal. + changed, err := dm.ApplyDatabaseOp("applyeq", DatabaseRegistryKey{Generation: 1, Dropped: false}) + if err != nil { + t.Fatalf("ApplyDatabaseOp failed: %v", err) + } + if changed { + t.Error("Expected changed=false for a key equal to local") + } + + // Below: generation 0 is only ever below a real (>=1) live generation. + changed, err = dm.ApplyDatabaseOp("applyeq", DatabaseRegistryKey{Generation: 0, Dropped: true}) + if err != nil { + t.Fatalf("ApplyDatabaseOp failed: %v", err) + } + if changed { + t.Error("Expected changed=false for a key below local") + } + if !dm.DatabaseExists("applyeq") { + t.Error("A no-op ApplyDatabaseOp must not drop the database") + } +} + +// TestApplyDatabaseOp_DroppedBeatsLiveAtSameGeneration verifies (g, dropped) +// outranks (g, live): a DROP at the same generation as the local live key is +// applied. +func TestApplyDatabaseOp_DroppedBeatsLiveAtSameGeneration(t *testing.T) { + dm, _ := setupTestDatabaseManager(t) + defer dm.Close() + + if err := dm.CreateDatabase("dropbeatslive"); err != nil { + t.Fatalf("Failed to create database: %v", err) + } + + changed, err := dm.ApplyDatabaseOp("dropbeatslive", DatabaseRegistryKey{Generation: 1, Dropped: true}) + if err != nil { + t.Fatalf("ApplyDatabaseOp failed: %v", err) + } + if !changed { + t.Error("Expected (1, dropped) to outrank and replace local (1, live)") + } + if dm.DatabaseExists("dropbeatslive") { + t.Error("Expected database to be dropped") + } +} + +// TestApplyDatabaseOp_LiveAtNextGenerationBeatsDropped verifies (g+1, live) +// outranks a local (g, dropped) tombstone: a re-create is applied over a +// drop. +func TestApplyDatabaseOp_LiveAtNextGenerationBeatsDropped(t *testing.T) { + dm, _ := setupTestDatabaseManager(t) + defer dm.Close() + + if err := dm.CreateDatabase("recreatebeatsdrop"); err != nil { + t.Fatalf("Failed to create database: %v", err) + } + if err := dm.DropDatabase("recreatebeatsdrop"); err != nil { + t.Fatalf("Failed to drop database: %v", err) + } + + changed, err := dm.ApplyDatabaseOp("recreatebeatsdrop", DatabaseRegistryKey{Generation: 2, Dropped: false}) + if err != nil { + t.Fatalf("ApplyDatabaseOp failed: %v", err) + } + if !changed { + t.Error("Expected (2, live) to outrank and replace local (1, dropped)") + } + if !dm.DatabaseExists("recreatebeatsdrop") { + t.Error("Expected database to be re-created") + } +} + +// TestApplyDatabaseOp_PeerLiveNeverResurrectsLocalDropped verifies a peer's +// (g, live) entry can never resurrect a local (g, dropped) tombstone - the +// core convergence guarantee (ApplyDatabaseOp's proof). +func TestApplyDatabaseOp_PeerLiveNeverResurrectsLocalDropped(t *testing.T) { + dm, _ := setupTestDatabaseManager(t) + defer dm.Close() + + if err := dm.CreateDatabase("neverresurrect"); err != nil { + t.Fatalf("Failed to create database: %v", err) + } + if err := dm.DropDatabase("neverresurrect"); err != nil { + t.Fatalf("Failed to drop database: %v", err) + } + + // A peer that missed the DROP still reports (1, live) - the key from + // before the drop. + changed, err := dm.ApplyDatabaseOp("neverresurrect", DatabaseRegistryKey{Generation: 1, Dropped: false}) + if err != nil { + t.Fatalf("ApplyDatabaseOp failed: %v", err) + } + if changed { + t.Error("Expected a peer's (1, live) to lose to local (1, dropped)") + } + if dm.DatabaseExists("neverresurrect") { + t.Error("Database must stay dropped") + } +} + +// TestApplyDatabaseOp_MissedCreate_CreatesDatabase models a node that missed +// the original CREATE DATABASE entirely: it has never seen the name (local +// key (0, dropped)), and a peer's key creates it. +func TestApplyDatabaseOp_MissedCreate_CreatesDatabase(t *testing.T) { + dm, _ := setupTestDatabaseManager(t) + defer dm.Close() + + if dm.DatabaseExists("missedcreate") { + t.Fatal("Database must not exist before ApplyDatabaseOp") + } + + changed, err := dm.ApplyDatabaseOp("missedcreate", DatabaseRegistryKey{Generation: 1, Dropped: false}) + if err != nil { + t.Fatalf("ApplyDatabaseOp failed: %v", err) + } + if !changed { + t.Error("Expected changed=true") + } + if !dm.DatabaseExists("missedcreate") { + t.Error("Expected the missed database to be created") + } +} + +// TestApplyDatabaseOp_MissedDrop_DropsDatabase models a node that missed a +// DROP DATABASE: it still has the database live locally, and a peer's +// tombstone key drops it. +func TestApplyDatabaseOp_MissedDrop_DropsDatabase(t *testing.T) { + dm, _ := setupTestDatabaseManager(t) + defer dm.Close() + + if err := dm.CreateDatabase("misseddrop"); err != nil { + t.Fatalf("Failed to create database: %v", err) + } + + changed, err := dm.ApplyDatabaseOp("misseddrop", DatabaseRegistryKey{Generation: 1, Dropped: true}) + if err != nil { + t.Fatalf("ApplyDatabaseOp failed: %v", err) + } + if !changed { + t.Error("Expected changed=true") + } + if dm.DatabaseExists("misseddrop") { + t.Error("Expected the missed drop to be applied") + } +} + +// TestApplyDatabaseOp_RefusesSystemAndDefaultDatabase verifies ApplyDatabaseOp +// keeps the same drop protections DropDatabase enforces directly. +func TestApplyDatabaseOp_RefusesSystemAndDefaultDatabase(t *testing.T) { + dm, _ := setupTestDatabaseManager(t) + defer dm.Close() + + if _, err := dm.ApplyDatabaseOp(SystemDatabaseName, DatabaseRegistryKey{Generation: 1, Dropped: false}); err == nil { + t.Error("Expected an error applying a create op to the system database") + } + if _, err := dm.ApplyDatabaseOp(SystemDatabaseName, DatabaseRegistryKey{Generation: 1, Dropped: true}); err == nil { + t.Error("Expected an error applying a drop op to the system database") + } + + defaultKey, err := dm.RegistryKey(DefaultDatabaseName) + if err != nil { + t.Fatalf("RegistryKey failed: %v", err) + } + if _, err := dm.ApplyDatabaseOp(DefaultDatabaseName, DatabaseRegistryKey{Generation: defaultKey.Generation + 1, Dropped: true}); err == nil { + t.Error("Expected an error applying a drop op to the default database") + } + if !dm.DatabaseExists(DefaultDatabaseName) { + t.Error("Default database must survive a refused drop") + } +} + +// TestApplyDatabaseOpClearsLegacyOnAStampedCreate (see ApplyDatabaseOp's doc +// comment): a legacy registry row (migrated from an older release's registry, +// no real generation +// history) stops being legacy the moment a real, post-upgrade CREATE merges +// a higher generation into it, so it reconciles as an ordinary live key from +// then on - including on a peer that has no row for the name at all, which +// a still-legacy row must never create (reconcileRegistryWithPeer, +// grpc/anti_entropy.go). +func TestApplyDatabaseOpClearsLegacyOnAStampedCreate(t *testing.T) { + dm, _ := setupTestDatabaseManager(t) + defer dm.Close() + + if err := dm.CreateDatabase("legacydb"); err != nil { + t.Fatalf("CreateDatabase failed: %v", err) + } + // Simulate a row migrated from an older release's registry (migrateDatabaseRegistrySchema). + if _, err := dm.systemDB.GetDB().Exec("UPDATE __marmot_databases SET legacy = 1 WHERE name = ?", "legacydb"); err != nil { + t.Fatalf("failed to mark row legacy: %v", err) + } + + // A peer's real, post-upgrade CREATE at a higher generation - merged the + // same way anti-entropy's registry reconciliation would. + changed, err := dm.ApplyDatabaseOp("legacydb", DatabaseRegistryKey{Generation: 5, Dropped: false}) + if err != nil { + t.Fatalf("ApplyDatabaseOp failed: %v", err) + } + if !changed { + t.Fatal("expected the higher-generation key to be adopted") + } + + entries, err := dm.RegistryEntries() + if err != nil { + t.Fatalf("RegistryEntries failed: %v", err) + } + var found bool + for _, e := range entries { + if e.Name != "legacydb" { + continue + } + found = true + if e.Legacy { + t.Error("a stamped CREATE merged via ApplyDatabaseOp did not clear the legacy flag") + } + if e.Key.Generation != 5 || e.Key.Dropped { + t.Errorf("expected key (5, live), got (%d, dropped=%v)", e.Key.Generation, e.Key.Dropped) + } + } + if !found { + t.Fatal("legacydb not found in RegistryEntries") + } +} diff --git a/db/db_integration.go b/db/db_integration.go index 2d0257b7..ca7b987f 100644 --- a/db/db_integration.go +++ b/db/db_integration.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strings" + "sync/atomic" "time" "github.com/maxpert/marmot/cfg" @@ -37,6 +38,9 @@ type ReplicatedDatabase struct { batchCommitter *SQLiteBatchCommitter schemaCache *SchemaCache // Shared schema cache for preupdate hooks gate *writeGate // refuses every commit once the database leaves service + + dbName string // database name, "" for the system database + schemaVersion atomic.Uint64 // cached __marmot_schema_version value; unused for the system database } // ReplicationFunc is called to replicate transactions to other nodes @@ -50,6 +54,29 @@ type replicatedDatabaseOptions struct { driverName string synchronous string batchCommit bool + + dbName string // "" when unnamed (the system database): no legacy schema version to migrate + legacySchemaVersion func(dbName string) (int64, error) // migration read source, nil if not applicable +} + +// WithDatabaseName names the database being opened: the name its +// __marmot_schema_version table migrates a legacy value under, and the +// name its applied-transaction repair records. The system database is opened +// without it; its table stays at 0, since no replicated DDL targets it. +func WithDatabaseName(name string) ReplicatedDatabaseOption { + return func(o *replicatedDatabaseOptions) { + o.dbName = name + } +} + +// WithLegacySchemaVersionSource supplies the retiring pebble-stored schema +// version counter, read once at open to migrate a pre-existing database's +// version into __marmot_schema_version. Only consulted when the +// SQLite file already existed before this open; see ensureSchemaVersionTable. +func WithLegacySchemaVersionSource(read func(dbName string) (int64, error)) ReplicatedDatabaseOption { + return func(o *replicatedDatabaseOptions) { + o.legacySchemaVersion = read + } } // WithDurableCommits makes every commit on the database survive an OS crash @@ -198,15 +225,37 @@ func NewReplicatedDatabase(dbPath string, nodeID uint64, clock *hlc.Clock, metaS } } + // The migration rule for __marmot_schema_version turns on whether + // __marmot_applied_txn already existed before this open, so it must be + // checked before ensureAppliedTxnTable creates it. + filePreexisted := false + if !isMemoryDB { + if existed, err := sqliteTableExists(writeDB, "__marmot_applied_txn"); err != nil { + closeAll() + return nil, err + } else { + filePreexisted = existed + } + } + if err := ensureAppliedTxnTable(writeDB); err != nil { closeAll() return nil, err } - if err := repairAppliedTxnMetadata(writeDB, metaStore, ""); err != nil { + if err := repairAppliedTxnMetadata(writeDB, metaStore, o.dbName); err != nil { closeAll() return nil, err } + // Every database file carries __marmot_schema_version, whichever path + // opened it: a commit that bumps it must never find it missing. Only a + // named user database migrates a legacy value into it. + schemaVersion, err := ensureSchemaVersionTable(writeDB, o.dbName, filePreexisted, o.legacySchemaVersion) + if err != nil { + closeAll() + return nil, fmt.Errorf("failed to prepare schema version table: %w", err) + } + // Create schema cache (shared by TransactionManager and preupdate hooks) schemaCache := NewSchemaCache() @@ -246,7 +295,13 @@ func NewReplicatedDatabase(dbPath string, nodeID uint64, clock *hlc.Clock, metaS batchCommitter: batchCommitter, schemaCache: schemaCache, gate: gate, + dbName: o.dbName, + } + mdb.schemaVersion.Store(schemaVersion) + if o.dbName != "" { + txnMgr.SetSchemaVersionBumped(func(v uint64) { mdb.advanceSchemaVersion(v) }) } + txnMgr.SetAppliedMarkerCheck(mdb.appliedMarkerExists) // Wire batch committer to transaction manager if batchCommitter != nil { @@ -265,6 +320,16 @@ func NewReplicatedDatabase(dbPath string, nodeID uint64, clock *hlc.Clock, metaS return mdb, nil } +// appliedMarkerExists reports whether __marmot_applied_txn holds txnID, +// read through the read pool so it never waits on the single writer. +func (mdb *ReplicatedDatabase) appliedMarkerExists(txnID uint64) (bool, error) { + readDB := mdb.readDB + if readDB == nil { + return false, errors.New("database connections are closed") + } + return markerExists(readDB, txnID) +} + // SetReplicationFunc sets the replication function func (mdb *ReplicatedDatabase) SetReplicationFunc(fn ReplicationFunc) { mdb.replicationFn = fn @@ -541,6 +606,39 @@ func (mdb *ReplicatedDatabase) GetMetaStore() MetaStore { return mdb.metaStore } +// DatabaseName returns the name this database was opened under ("" for the +// system database). +func (mdb *ReplicatedDatabase) DatabaseName() string { + return mdb.dbName +} + +// SchemaVersion returns the cached __marmot_schema_version value. It +// is always 0 for the system database, which has no such table. +func (mdb *ReplicatedDatabase) SchemaVersion() uint64 { + return mdb.schemaVersion.Load() +} + +// advanceSchemaVersion raises the cached __marmot_schema_version value to v, +// never lowering it: every post-commit store of a value read +// from the database itself - the replay path and the 2PC non-DML commit +// callback - goes through this instead of a plain Store, because two such +// commits can finish in either order (replay's anti-entropy goroutine versus +// this node's own DDL committer) and an out-of-order Store would regress the +// cache below a value SQLite itself already has committed, wrongly declining +// the PREPARE gate until the next DDL. The initial Store at open (a restore +// reopen legitimately resets the cache) is exempt and stays a plain Store. +func (mdb *ReplicatedDatabase) advanceSchemaVersion(v uint64) { + for { + cur := mdb.schemaVersion.Load() + if v <= cur { + return + } + if mdb.schemaVersion.CompareAndSwap(cur, v) { + return + } + } +} + // GetCachedTableSchema returns the cached schema for a table. // This uses the in-memory schema cache and does NOT query SQLite. func (mdb *ReplicatedDatabase) GetCachedTableSchema(tableName string) (*TableSchema, error) { diff --git a/db/ddl_prepare_validate.go b/db/ddl_prepare_validate.go index 949027ff..8cf5bd16 100644 --- a/db/ddl_prepare_validate.go +++ b/db/ddl_prepare_validate.go @@ -37,6 +37,15 @@ var transientSQLiteCodes = map[sqlite3.ErrNo]bool{ sqlite3.ErrReadonly: true, // SQLITE_READONLY: database (or this connection) is read-only right now } +// IsTransientSQLiteError reports whether err carries one of +// transientSQLiteCodes: a condition on this node right now - typically the +// single writer held by another connection - not a verdict on the work that +// hit it. +func IsTransientSQLiteError(err error) bool { + var sqliteErr sqlite3.Error + return errors.As(err, &sqliteErr) && transientSQLiteCodes[sqliteErr.Code] +} + // AutoIncWidthExceededError reports that MAX(id) in an existing table exceeds // the range its explicitly declared AUTO_INCREMENT column can hold - e.g. an // ALTER that narrows the column to TINYINT after 200 rows already exist. diff --git a/db/ddl_prepare_validate_test.go b/db/ddl_prepare_validate_test.go index 4bbb5ced..69ee25e5 100644 --- a/db/ddl_prepare_validate_test.go +++ b/db/ddl_prepare_validate_test.go @@ -432,3 +432,12 @@ func TestValidateDDLStatements_OverWidthTableKeepsAcceptingDDL(t *testing.T) { `ALTER TABLE legacy ADD COLUMN note TEXT`, }), "a DDL that does not re-declare the column's width was refused over ids the width never held") } + +// TestIsTransientSQLiteError pins that a wrapped busy or locked SQLite error +// is transient and a verdict such as a constraint violation is not. +func TestIsTransientSQLiteError(t *testing.T) { + require.True(t, IsTransientSQLiteError(fmt.Errorf("commit: %w", sqlite3.Error{Code: sqlite3.ErrBusy}))) + require.True(t, IsTransientSQLiteError(fmt.Errorf("commit: %w", sqlite3.Error{Code: sqlite3.ErrLocked}))) + require.False(t, IsTransientSQLiteError(fmt.Errorf("commit: %w", sqlite3.Error{Code: sqlite3.ErrConstraint}))) + require.False(t, IsTransientSQLiteError(errors.New("not a SQLite error"))) +} diff --git a/db/detach_drain_test.go b/db/detach_drain_test.go index 270c0e26..b9636762 100644 --- a/db/detach_drain_test.go +++ b/db/detach_drain_test.go @@ -30,18 +30,20 @@ func newDetachTestManager(t *testing.T) *DatabaseManager { return dm } -// holdGCPassBeforeTheManagerLock restarts tm's GC with a short interval and a -// refresh step that blocks the first pass until release is closed. When the -// returned channel closes, a pass is inside that step; its next call is the -// manager's GetMinAppliedTxnID, which takes DatabaseManager.mu. +// holdGCPassBeforeTheManagerLock restarts tm's GC with a short interval and +// a GC-safe-position step that blocks the first pass until release is +// closed. When the returned channel closes, a pass is inside that step; a +// real GCSafePositionFunc implementation is expected to take +// DatabaseManager.mu to aggregate consumed positions across databases, the +// same way the GetMinAppliedTxnID callback it replaced did. func holdGCPassBeforeTheManagerLock(tm *TransactionManager, release <-chan struct{}) <-chan struct{} { entered := make(chan struct{}) var once sync.Once tm.StopGarbageCollection() - tm.SetRefreshReplicationStatesFunc(func(context.Context) error { + tm.SetGCSafePositionFunc(func() (LogPosition, bool) { once.Do(func() { close(entered) }) <-release - return nil + return LogPosition{}, false }) tm.gcInterval = 5 * time.Millisecond tm.StartGarbageCollection() @@ -57,10 +59,11 @@ func gcStopRequested(tm *TransactionManager) bool { // TestClosingADatabaseDoesNotDeadlockWithAGCPassWaitingOnTheManager: detach, // drop and shutdown each stop a database's GC and wait for its pass to end, -// while that pass may be about to take DatabaseManager.mu -// (GetMinAppliedTxnID). None of them may hold mu while it waits. The pass is -// held in its refresh step until the operation is waiting for it, then let go -// towards the lock. +// while that pass may be about to take DatabaseManager.mu (a +// GCSafePositionFunc implementation aggregating consumed positions across +// databases). None of them may hold mu while it waits. The pass is held in +// its GC-safe-position step until the operation is waiting for it, then let +// go towards the lock. // // Mutation: hold dm.mu across DetachDatabase's drain (or DropDatabase's // db.Close, or Close's database loop). The pass then waits on the lock the @@ -197,14 +200,14 @@ func TestSnapshotsRefuseWhileADatabaseIsDetached(t *testing.T) { _, _, err := dm.TakeSnapshot() require.ErrorIs(t, err, ErrDatabaseDetached, "a snapshot omitted a detached database (TakeSnapshot)") - _, _, _, err = dm.TakeSnapshotToDir(t.TempDir()) + _, _, err = dm.TakeSnapshotToDir(t.TempDir()) require.ErrorIs(t, err, ErrDatabaseDetached, "a snapshot omitted a detached database (TakeSnapshotToDir)") require.NoError(t, dm.AttachDatabase("app")) infos, _, err := dm.TakeSnapshot() require.NoError(t, err) require.Contains(t, snapshotNames(infos), "app") - infos, _, _, err = dm.TakeSnapshotToDir(t.TempDir()) + infos, _, err = dm.TakeSnapshotToDir(t.TempDir()) require.NoError(t, err) require.Contains(t, snapshotNames(infos), "app") } @@ -317,7 +320,13 @@ func TestDropDatabaseDuringARestore(t *testing.T) { _, err = dm.GetDatabase("app") require.Error(t, err) require.NotErrorIs(t, err, ErrDatabaseDetached) - require.NotContains(t, registryNames(t, dm), "app", "the registry still names a dropped database") + // The registry keeps the row as a tombstone (dropped=1) rather than + // deleting it (DatabaseRegistryKey), so a later + // CREATE can fence a stale peer with a strictly higher generation. + require.Contains(t, registryNames(t, dm), "app", "the registry must keep a dropped database's row as a tombstone") + key, err := dm.RegistryKey("app") + require.NoError(t, err) + require.True(t, key.Dropped, "a dropped database's registry row must be tombstoned") require.ErrorIs(t, dm.CreateDatabase("app"), ErrDatabaseDetached, "a CREATE raced the files of a drop in progress") require.NoError(t, dm.DropDatabase("app"), "a repeated DROP must stay idempotent") diff --git a/db/gc_log_test.go b/db/gc_log_test.go new file mode 100644 index 00000000..3a35bfbd --- /dev/null +++ b/db/gc_log_test.go @@ -0,0 +1,363 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "testing" + "time" + + "github.com/maxpert/marmot/hlc" +) + +// commitTestTxn begins and commits a simple transaction and returns its +// LogPosition. +func commitTestTxn(t *testing.T, store *PebbleMetaStore, txnID, nodeID uint64) LogPosition { + t.Helper() + clock := hlc.NewClock(nodeID) + ts := clock.Now() + if err := store.BeginTransaction(txnID, nodeID, ts); err != nil { + t.Fatalf("BeginTransaction(%d): %v", txnID, err) + } + if err := store.CommitTransaction(txnID, ts, nil, "testdb", "", 0, 0); err != nil { + t.Fatalf("CommitTransaction(%d): %v", txnID, err) + } + rec, err := store.GetTransaction(txnID) + if err != nil || rec == nil { + t.Fatalf("GetTransaction(%d): rec=%v err=%v", txnID, rec, err) + } + return LogPosition{Seq: rec.SeqNum, TxnID: txnID} +} + +func TestListCommittedLogOrderingLimitMoreStable(t *testing.T) { + store, cleanup := createTestPebbleMetaStore(t) + defer cleanup() + + var positions []LogPosition + for i := uint64(1); i <= 5; i++ { + positions = append(positions, commitTestTxn(t, store, 1000+i, 1)) + } + + // First page: limit 2, from the zero position. + entries, stable, more, err := store.ListCommittedLog(LogPosition{}, 2) + if err != nil { + t.Fatalf("ListCommittedLog: %v", err) + } + if len(entries) != 2 { + t.Fatalf("page 1 len = %d, want 2", len(entries)) + } + if entries[0] != positions[0] || entries[1] != positions[1] { + t.Fatalf("page 1 = %v, want %v", entries, positions[:2]) + } + if !more { + t.Fatal("expected more=true with 3 entries left") + } + if stable < positions[len(positions)-1].Seq { + t.Fatalf("stable=%d should cover every committed entry (last seq %d)", stable, positions[len(positions)-1].Seq) + } + + // Second page: continue after the last entry of page 1. + entries2, _, more2, err := store.ListCommittedLog(entries[len(entries)-1], 10) + if err != nil { + t.Fatalf("ListCommittedLog page 2: %v", err) + } + if len(entries2) != 3 { + t.Fatalf("page 2 len = %d, want 3", len(entries2)) + } + for i, want := range positions[2:] { + if entries2[i] != want { + t.Fatalf("page 2[%d] = %v, want %v", i, entries2[i], want) + } + } + if more2 { + t.Fatal("expected more=false once every entry has been listed") + } + + // Listing strictly after the last entry returns nothing. + entries3, _, more3, err := store.ListCommittedLog(positions[len(positions)-1], 10) + if err != nil { + t.Fatalf("ListCommittedLog page 3: %v", err) + } + if len(entries3) != 0 || more3 { + t.Fatalf("page 3 = %v more=%v, want empty/false", entries3, more3) + } +} + +func TestListCommittedLogExcludesUncommitted(t *testing.T) { + store, cleanup := createTestPebbleMetaStore(t) + defer cleanup() + + committed := commitTestTxn(t, store, 2001, 1) + + // A PENDING transaction has no seq-index entry, so it can't appear. + clock := hlc.NewClock(1) + if err := store.BeginTransaction(2002, 1, clock.Now()); err != nil { + t.Fatalf("BeginTransaction: %v", err) + } + + entries, _, _, err := store.ListCommittedLog(LogPosition{}, 10) + if err != nil { + t.Fatalf("ListCommittedLog: %v", err) + } + if len(entries) != 1 || entries[0] != committed { + t.Fatalf("entries = %v, want only %v", entries, committed) + } +} + +func TestPullCursorAndConsumedPositionAreSeparateAndAsIs(t *testing.T) { + store, cleanup := createTestPebbleMetaStore(t) + defer cleanup() + + peer := uint64(42) + + zero, err := store.GetPullCursor(peer) + if err != nil || zero != (LogPosition{}) { + t.Fatalf("GetPullCursor before any Set = %v, %v; want zero, nil", zero, err) + } + + high := LogPosition{Seq: 100, TxnID: 5} + if err := store.SetPullCursor(peer, high); err != nil { + t.Fatalf("SetPullCursor: %v", err) + } + got, err := store.GetPullCursor(peer) + if err != nil || got != high { + t.Fatalf("GetPullCursor = %v, %v; want %v", got, err, high) + } + + // Consumed positions are stored under a distinct namespace: setting one + // for the same node id must not change the pull cursor, and vice versa. + if err := store.SetConsumedPosition(peer, LogPosition{Seq: 7, TxnID: 1}); err != nil { + t.Fatalf("SetConsumedPosition: %v", err) + } + stillHigh, err := store.GetPullCursor(peer) + if err != nil || stillHigh != high { + t.Fatalf("pull cursor changed by SetConsumedPosition: got %v, want %v", stillHigh, high) + } + + // SetConsumedPosition stores the value as is, not as a max: a lower + // position must overwrite a higher one (needed after a restore). + low := LogPosition{Seq: 3, TxnID: 1} + if err := store.SetConsumedPosition(peer, low); err != nil { + t.Fatalf("SetConsumedPosition (regress): %v", err) + } + all, err := store.ConsumedPositions() + if err != nil { + t.Fatalf("ConsumedPositions: %v", err) + } + if all[peer] != low { + t.Fatalf("ConsumedPositions[%d] = %v, want %v (as-is, not max)", peer, all[peer], low) + } + + if err := store.DeleteConsumedPosition(peer); err != nil { + t.Fatalf("DeleteConsumedPosition: %v", err) + } + all2, err := store.ConsumedPositions() + if err != nil { + t.Fatalf("ConsumedPositions after delete: %v", err) + } + if _, ok := all2[peer]; ok { + t.Fatalf("consumed position for %d should be gone after delete", peer) + } +} + +func TestGCDeletesCommittedAtOrBelowSafeOlderThanMinRetention(t *testing.T) { + store, cleanup := createTestPebbleMetaStore(t) + defer cleanup() + + pos := commitTestTxn(t, store, 3001, 1) + time.Sleep(2 * time.Millisecond) + + // minRetention=0 => already "older than min retention"; maxRetention + // large => the unconditional branch doesn't apply. safe covers pos. + n, err := store.CleanupOldTransactionRecords(0, time.Hour, pos) + if err != nil { + t.Fatalf("CleanupOldTransactionRecords: %v", err) + } + if n != 1 { + t.Fatalf("deleted = %d, want 1", n) + } + + rec, err := store.GetTransaction(3001) + if err != nil { + t.Fatalf("GetTransaction: %v", err) + } + if rec != nil { + t.Fatalf("expected the record to be gone, got %+v", rec) + } + + through, err := store.TruncatedThrough() + if err != nil { + t.Fatalf("TruncatedThrough: %v", err) + } + if through != pos { + t.Fatalf("TruncatedThrough = %v, want %v", through, pos) + } +} + +func TestGCNeverDeletesAnEntryNoMemberHasConsumed(t *testing.T) { + store, cleanup := createTestPebbleMetaStore(t) + defer cleanup() + + commitTestTxn(t, store, 3101, 1) + time.Sleep(2 * time.Millisecond) + + // safe is the zero position: nothing has been reported consumed, so + // the min-retention branch must not fire regardless of age. + n, err := store.CleanupOldTransactionRecords(0, time.Hour, LogPosition{}) + if err != nil { + t.Fatalf("CleanupOldTransactionRecords: %v", err) + } + if n != 0 { + t.Fatalf("deleted = %d, want 0 (nothing consumed yet)", n) + } + + rec, err := store.GetTransaction(3101) + if err != nil || rec == nil { + t.Fatalf("record should survive: rec=%v err=%v", rec, err) + } +} + +func TestGCDeletesOlderThanMaxRetentionRegardlessOfSafe(t *testing.T) { + store, cleanup := createTestPebbleMetaStore(t) + defer cleanup() + + commitTestTxn(t, store, 3201, 1) + time.Sleep(2 * time.Millisecond) + + // maxRetention=0 forces unconditional deletion; minRetention huge so + // that branch alone would not fire; safe is zero (would refuse via the + // min-retention branch, but must not block the max-retention branch). + n, err := store.CleanupOldTransactionRecords(time.Hour, 0, LogPosition{}) + if err != nil { + t.Fatalf("CleanupOldTransactionRecords: %v", err) + } + if n != 1 { + t.Fatalf("deleted = %d, want 1 (max retention is unconditional)", n) + } +} + +// TestGCStopsAtFirstNonDeletableEntryPrefixOnly enforces the no-holes +// invariant: GC only ever deletes a prefix of the log in position order, so +// TruncatedThrough always means "everything at or below this is gone." +func TestGCStopsAtFirstNonDeletableEntryPrefixOnly(t *testing.T) { + store, cleanup := createTestPebbleMetaStore(t) + defer cleanup() + + posA := commitTestTxn(t, store, 3301, 1) + posB := commitTestTxn(t, store, 3302, 1) + posC := commitTestTxn(t, store, 3303, 1) + time.Sleep(2 * time.Millisecond) + + // safe covers only A; B (and, behind it, C) must stop the walk even + // though C individually would also be old enough. + n, err := store.CleanupOldTransactionRecords(0, time.Hour, posA) + if err != nil { + t.Fatalf("CleanupOldTransactionRecords: %v", err) + } + if n != 1 { + t.Fatalf("deleted = %d, want 1 (only A)", n) + } + + if rec, _ := store.GetTransaction(3301); rec != nil { + t.Fatal("A should be deleted") + } + if rec, _ := store.GetTransaction(3302); rec == nil { + t.Fatal("B must survive: it is not covered by safe") + } + if rec, _ := store.GetTransaction(3303); rec == nil { + t.Fatal("C must survive: the walk must stop at B, before reaching C") + } + + through, err := store.TruncatedThrough() + if err != nil { + t.Fatalf("TruncatedThrough: %v", err) + } + if through != posA { + t.Fatalf("TruncatedThrough = %v, want %v", through, posA) + } + _ = posB + _ = posC +} + +func TestTruncatedThroughNeverDecreases(t *testing.T) { + store, cleanup := createTestPebbleMetaStore(t) + defer cleanup() + + posA := commitTestTxn(t, store, 3401, 1) + posB := commitTestTxn(t, store, 3402, 1) + time.Sleep(2 * time.Millisecond) + + if _, err := store.CleanupOldTransactionRecords(0, time.Hour, posB); err != nil { + t.Fatalf("first GC: %v", err) + } + first, err := store.TruncatedThrough() + if err != nil { + t.Fatalf("TruncatedThrough: %v", err) + } + if first != posB { + t.Fatalf("TruncatedThrough = %v, want %v", first, posB) + } + _ = posA + + // A second pass with nothing left to delete must not regress T. + if _, err := store.CleanupOldTransactionRecords(0, time.Hour, LogPosition{}); err != nil { + t.Fatalf("second GC: %v", err) + } + second, err := store.TruncatedThrough() + if err != nil { + t.Fatalf("TruncatedThrough (2): %v", err) + } + if second != first { + t.Fatalf("TruncatedThrough regressed: %v -> %v", first, second) + } +} + +// TestGCSkipsANonCommittedSeqEntryUntilMaxRetention pins that a seq-index +// entry whose status is no longer COMMITTED (a record a late PREPARE's begin +// overwrote before BeginTransactionWithID refused that) never stops GC +// forever: the entries after it are still deleted, and it is deleted itself +// once past max retention. +func TestGCSkipsANonCommittedSeqEntryUntilMaxRetention(t *testing.T) { + store, cleanup := createTestPebbleMetaStore(t) + defer cleanup() + + commitTestTxn(t, store, 3401, 1) + commitTestTxn(t, store, 3402, 1) + posC := commitTestTxn(t, store, 3403, 1) + if err := store.db.Set(pebbleTxnStatusKey(3402), []byte{byte(TxnStatusPending)}, nil); err != nil { + t.Fatalf("overwrite status: %v", err) + } + time.Sleep(2 * time.Millisecond) + + n, err := store.CleanupOldTransactionRecords(0, time.Hour, posC) + if err != nil { + t.Fatalf("CleanupOldTransactionRecords: %v", err) + } + if n != 2 { + t.Fatalf("deleted = %d, want 2 (the committed entries around the stray one)", n) + } + if rec, _ := store.GetTransaction(3403); rec != nil { + t.Fatal("the committed entry after the stray one must be deleted") + } + if rec, _ := store.GetTransaction(3402); rec == nil { + t.Fatal("the stray entry must survive until max retention") + } + through, err := store.TruncatedThrough() + if err != nil { + t.Fatalf("TruncatedThrough: %v", err) + } + if through != posC { + t.Fatalf("TruncatedThrough = %v, want %v", through, posC) + } + + n, err = store.CleanupOldTransactionRecords(0, 0, posC) + if err != nil { + t.Fatalf("CleanupOldTransactionRecords: %v", err) + } + if n != 1 { + t.Fatalf("deleted = %d, want 1 (the stray entry, past max retention)", n) + } + if rec, _ := store.GetTransaction(3402); rec != nil { + t.Fatal("the stray entry must be deleted past max retention") + } +} diff --git a/db/gc_membership_test.go b/db/gc_membership_test.go new file mode 100644 index 00000000..8d537a5c --- /dev/null +++ b/db/gc_membership_test.go @@ -0,0 +1,151 @@ +package db + +import ( + "testing" + "time" + + "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" +) + +// TestGCSafePositionNeverDeletesAnEntryAMemberHasNotConsumed drives the real +// TransactionManager GC (cleanupOldTransactionRecords, via the +// GCSafePositionFunc DatabaseManager.wireGCCoordination wires in) through a +// live DatabaseManager, not a bare MetaStore call: member 2 is current but +// has never reported a consumed position, which must pin GC regardless of +// the entry's age: no member may lose an entry it has not pulled. +func TestGCSafePositionNeverDeletesAnEntryAMemberHasNotConsumed(t *testing.T) { + dbMgr, mdb, _ := newReplayTestDatabase(t) + tm := mdb.GetTransactionManager() + + // Short retentions so the test does not wait out real hours; the min + // retention branch is what this test exercises. + tm.mu.Lock() + tm.gcMinRetention = time.Millisecond + tm.gcMaxRetention = time.Hour + tm.mu.Unlock() + + // Membership: self (1) and members 2 and 3, which have not called + // SetConsumedPosition yet. + dbMgr.SetGCMembershipFunc(func() []uint64 { return []uint64{1, 2, 3} }) + + txn := &ReplayTxn{TxnID: 9001, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: time.Now().UnixNano(), NodeID: 1}, Rows: []*EncodedCapturedRow{insertRow(1, "a")}} + applied, err := mdb.ApplyReplayedTxn(t.Context(), txn, true) + require.NoError(t, err) + require.True(t, applied) + + time.Sleep(5 * time.Millisecond) // exceed gcMinRetention + + n, err := tm.cleanupOldTransactionRecords() + require.NoError(t, err) + require.Equal(t, 0, n, "member 2 has never reported a consumed position; GC must not delete") + require.True(t, rowExists(t, mdb, 1)) + + // Member 2 now reports it has consumed through the txn's position. + rec, err := mdb.GetMetaStore().GetTransaction(9001) + require.NoError(t, err) + require.NotNil(t, rec) + require.NoError(t, mdb.GetMetaStore().SetConsumedPosition(2, LogPosition{Seq: rec.SeqNum, TxnID: 9001})) + + n, err = tm.cleanupOldTransactionRecords() + require.NoError(t, err) + require.Equal(t, 0, n, "member 3 has not consumed the entry; GC must not delete it (the safe position is a min)") + + require.NoError(t, mdb.GetMetaStore().SetConsumedPosition(3, LogPosition{Seq: rec.SeqNum, TxnID: 9001})) + n, err = tm.cleanupOldTransactionRecords() + require.NoError(t, err) + require.Equal(t, 1, n, "every member has now consumed through the entry; GC must delete it") +} + +// TestGCPermanentlyDownMemberPinsUntilMaxRetentionThenDeletes drives the +// real TransactionManager GC through a live DatabaseManager: a member that +// never reports a consumed position pins deletion only until gcMaxRetention +// elapses, after which GC deletes unconditionally (the documented bound: a +// member down longer than gc_max_retention_hours restores by snapshot). +func TestGCPermanentlyDownMemberPinsUntilMaxRetentionThenDeletes(t *testing.T) { + dbMgr, mdb, _ := newReplayTestDatabase(t) + tm := mdb.GetTransactionManager() + + tm.mu.Lock() + tm.gcMinRetention = time.Hour // keep the min-retention branch out of the way + tm.gcMaxRetention = 30 * time.Millisecond + tm.mu.Unlock() + + // Member 2 is a current member that never reports a consumed position - + // permanently down. + dbMgr.SetGCMembershipFunc(func() []uint64 { return []uint64{1, 2} }) + + txn := &ReplayTxn{TxnID: 9101, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: time.Now().UnixNano(), NodeID: 1}, Rows: []*EncodedCapturedRow{insertRow(2, "b")}} + applied, err := mdb.ApplyReplayedTxn(t.Context(), txn, true) + require.NoError(t, err) + require.True(t, applied) + + n, err := tm.cleanupOldTransactionRecords() + require.NoError(t, err) + require.Equal(t, 0, n, "before max retention elapses, a down member's missing position must still pin GC") + + time.Sleep(60 * time.Millisecond) // exceed gcMaxRetention + + n, err = tm.cleanupOldTransactionRecords() + require.NoError(t, err) + require.Equal(t, 1, n, "past max retention, GC must delete unconditionally even though member 2 never reported") + + through, err := mdb.GetMetaStore().TruncatedThrough() + require.NoError(t, err) + require.NotEqual(t, LogPosition{}, through, "TruncatedThrough must advance past the deleted entry") +} + +// TestGCSafePositionExcludesDepartedMembersConsumedPosition drives the real +// wiring's cleanup of a departed member's consumed position: once a member +// is no longer current (SetGCMembershipFunc stops listing it), its stale +// R[self,m,d] must no longer pin GC, and wireGCCoordination's closure +// deletes it via MetaStore.DeleteConsumedPosition. +func TestGCSafePositionExcludesDepartedMembersConsumedPosition(t *testing.T) { + dbMgr, mdb, _ := newReplayTestDatabase(t) + tm := mdb.GetTransactionManager() + + tm.mu.Lock() + tm.gcMinRetention = time.Millisecond + tm.gcMaxRetention = time.Hour + tm.mu.Unlock() + + // Three-member cluster: member 3 stays current and catches all the way + // up; member 2 is about to depart while stuck at the zero position, so + // the test can tell "still pinning while current" apart from "no longer + // pins once departed" without also depending on the single-member + // (no-other-member) fallback case, which GCSafePositionFunc documents + // separately as "unknown - defer to max retention". + dbMgr.SetGCMembershipFunc(func() []uint64 { return []uint64{1, 2, 3} }) + + txn := &ReplayTxn{TxnID: 9201, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: time.Now().UnixNano(), NodeID: 1}, Rows: []*EncodedCapturedRow{insertRow(3, "c")}} + applied, err := mdb.ApplyReplayedTxn(t.Context(), txn, true) + require.NoError(t, err) + require.True(t, applied) + + rec, err := mdb.GetMetaStore().GetTransaction(9201) + require.NoError(t, err) + require.NotNil(t, rec) + entryPos := LogPosition{Seq: rec.SeqNum, TxnID: 9201} + + // Member 3 has fully caught up; member 2 is stuck at the zero position, + // which would pin GC if it were still a member. + require.NoError(t, mdb.GetMetaStore().SetConsumedPosition(2, LogPosition{})) + require.NoError(t, mdb.GetMetaStore().SetConsumedPosition(3, entryPos)) + + time.Sleep(5 * time.Millisecond) + + n, err := tm.cleanupOldTransactionRecords() + require.NoError(t, err) + require.Equal(t, 0, n, "member 2's stale (zero) consumed position must still pin GC while it is current") + + // Member 2 leaves membership entirely. + dbMgr.SetGCMembershipFunc(func() []uint64 { return []uint64{1, 3} }) + + n, err = tm.cleanupOldTransactionRecords() + require.NoError(t, err) + require.Equal(t, 1, n, "a departed member's consumed position must no longer pin GC") + + positions, err := mdb.GetMetaStore().ConsumedPositions() + require.NoError(t, err) + require.NotContains(t, positions, uint64(2), "a departed member's consumed position must be deleted, not merely ignored") +} diff --git a/db/log_position.go b/db/log_position.go new file mode 100644 index 00000000..52d9d6f7 --- /dev/null +++ b/db/log_position.go @@ -0,0 +1,355 @@ +package db + +import ( + "encoding/binary" + "fmt" + "runtime" + "sync" + "sync/atomic" + + "github.com/cockroachdb/pebble" + "github.com/rs/zerolog/log" +) + +// LogPosition identifies an entry in a database's local commit log: the +// store-wide local commit sequence paired with the transaction that +// produced it. Ordering is lexicographic (Seq, TxnID), which is the +// ordering of the existing seq-index key pebbleTxnSeqKey(seq, txnID). The +// zero value precedes every real entry, since the sequence never hands out +// 0 (see newLogSeqAllocator). +type LogPosition struct { + Seq uint64 + TxnID uint64 +} + +// Compare returns -1, 0 or 1 as p sorts before, equal to, or after o. +func (p LogPosition) Compare(o LogPosition) int { + switch { + case p.Seq < o.Seq: + return -1 + case p.Seq > o.Seq: + return 1 + case p.TxnID < o.TxnID: + return -1 + case p.TxnID > o.TxnID: + return 1 + default: + return 0 + } +} + +// Less reports whether p sorts strictly before o. +func (p LogPosition) Less(o LogPosition) bool { + return p.Compare(o) < 0 +} + +// encodeLogPosition serializes p as the 16-byte big-endian (Seq, TxnID) +// pair used both by the seq index key suffix and by every persisted +// LogPosition value (cursors, consumed positions, truncation point). +func encodeLogPosition(p LogPosition) []byte { + buf := make([]byte, 16) + binary.BigEndian.PutUint64(buf, p.Seq) + binary.BigEndian.PutUint64(buf[8:], p.TxnID) + return buf +} + +// decodeLogPosition is the inverse of encodeLogPosition. A short buffer +// (missing key, corrupt value) decodes as the zero position. +func decodeLogPosition(buf []byte) LogPosition { + if len(buf) < 16 { + return LogPosition{} + } + return LogPosition{ + Seq: binary.BigEndian.Uint64(buf), + TxnID: binary.BigEndian.Uint64(buf[8:]), + } +} + +const ( + // logSeqBandwidth is the number of sequence numbers durably reserved by + // each lease extension of the store-wide local commit sequence. + logSeqBandwidth = 10000 + + // logSeqRingSize bounds how many sequence numbers a process may have + // allocated but not yet marked done at once. It must be a power of two; + // allocation spins (no I/O) until the watermark frees a slot, which + // only happens under an implausibly deep backlog of concurrent writers. + logSeqRingSize = 1 << 16 +) + +// logSeqTracker is a lock-free, exact record of which sequence numbers a +// process has finished allocating. It replaces a time-based "eps" staleness +// assumption: a position is never silently +// skipped, because nothing is ever reported stable until every seq up to it +// is known, exactly, to be done. +// +// Proof that StableSeq() is safe for a reader to snapshot before it opens +// its pebble iterator: a reader takes stable = StableSeq() before opening +// the iterator. Pebble makes every completed batch.Commit visible to any +// iterator opened after that Commit call returns. For every seq <= stable, +// markDone(seq) has already run; markDone is invoked by a defer that covers +// every exit path of CommitTransaction and StoreReplayedTransaction, so by +// the time it runs, either the commit-record batch has already committed +// (in which case it is visible to the reader's iterator, opened afterwards) +// or the writer gave up without committing (in which case that seq's +// record was never written, and never will be, since a seq is allocated +// exactly once). So every seq <= stable is either visible now or will never +// exist. Nothing becomes visible at or below stable afterwards, because +// stable only ever increases and a seq is never reused. Entries written +// before this process started are, by construction, below this process's +// first allocated seq, hence already covered by the initial watermark (see +// newLogSeqAllocator). No wall-clock reading appears anywhere in this +// argument. +type logSeqTracker struct { + // slots[i] holds the seq value most recently marked done at index + // i = seq % logSeqRingSize, or 0 if none has been yet. Storing the + // actual seq (not a done bit) lets advance() tell a stale value left + // behind by an earlier generation apart from the one it is looking for. + slots [logSeqRingSize]atomic.Uint64 + + // watermark is StableSeq(): the highest seq s such that every seq <= s + // allocated by this process has been marked done. + watermark atomic.Uint64 +} + +// markDone records that seq has finished (successfully or not) and +// advances the watermark over any newly-stable contiguous prefix. +func (t *logSeqTracker) markDone(seq uint64) { + t.slots[seq%logSeqRingSize].Store(seq) + for { + w := t.watermark.Load() + next := w + 1 + if t.slots[next%logSeqRingSize].Load() != next { + return + } + if !t.watermark.CompareAndSwap(w, next) { + continue // another marker moved the watermark; retry from its value + } + } +} + +// stable returns the current watermark (StableSeq). +func (t *logSeqTracker) stable() uint64 { + return t.watermark.Load() +} + +// waitForSlot spins, without doing any I/O, until seq's ring slot has been +// vacated by the watermark advancing past seq-ringSize. This is what bounds +// the number of seqs a process may have in flight at once; by the time seq +// is allocated, the watermark must already have passed seq-ringSize, so the +// slot's previous occupant is guaranteed done. +func (t *logSeqTracker) waitForSlot(seq uint64) { + for seq-t.watermark.Load() >= logSeqRingSize { + runtime.Gosched() + } +} + +// logSeqAllocator mints the store-wide local commit sequence (LogPosition.Seq) +// for one PebbleMetaStore. It replaces the per-origin AtomicSequence leases +// keyed by /seq/{nodeID}: every committed transaction in a database's meta +// store, whatever its origin, now draws from this single sequence, so the +// seq index totally orders the store's local log. +// +// The lease end is persisted with the store's durable write option +// (pebble.Sync, or pebble.NoSync for a store opened with DisableWAL, tests +// only, which has no WAL to lose), so a seq is never reissued even across a +// crash that lost NoSync commit records: allocate() never returns a seq at +// or above the last durably persisted end without a new, durably persisted +// end past it existing first. The lease is extended asynchronously, in the +// background, once half of the current lease has been consumed, so its +// fsync never sits on the allocation path; allocate() only extends +// synchronously in the rare case where consumption outruns that extension +// and reaches the durable end before it completes. +type logSeqAllocator struct { + db *pebble.DB + key []byte + syncWrite *pebble.WriteOptions // pebble.Sync, or pebble.NoSync for a WAL-disabled store (tests only) + + next atomic.Uint64 // next seq to hand out + + durableEnd atomic.Uint64 // durable lease end; allocate() blocks here + extendThreshold atomic.Uint64 // seq at which the next extension is triggered + extending atomic.Bool // guards a single in-flight async extension + extendWG sync.WaitGroup + extendMu sync.Mutex // serialises lease extensions (extendPast) + + tracker logSeqTracker +} + +// newLogSeqAllocator opens the store-wide sequence, resuming from its +// persisted durable end if the key already exists, or establishing one at +// the value initialFn computes (evaluated only on this first-ever open, to +// avoid rescanning the store on every restart) otherwise. initialFn must +// return a value strictly above every seq this store, or its per-origin +// predecessor, has ever handed out. +// +// syncWrite is the store's durable write option: pebble.Sync normally, or +// pebble.NoSync for a store opened with DisableWAL (tests only), which has +// no durable write to offer and refuses a synced one. +func newLogSeqAllocator(db *pebble.DB, key []byte, syncWrite *pebble.WriteOptions, initialFn func() (uint64, error)) (*logSeqAllocator, error) { + a := &logSeqAllocator{db: db, key: key, syncWrite: syncWrite} + + var end uint64 + val, closer, err := db.Get(key) + switch { + case err == nil: + if len(val) >= 8 { + end = binary.BigEndian.Uint64(val) + } + closer.Close() + case err == pebble.ErrNotFound: + end, err = initialFn() + if err != nil { + return nil, fmt.Errorf("failed to derive initial log sequence: %w", err) + } + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, end) + if err := db.Set(key, buf, syncWrite); err != nil { + return nil, fmt.Errorf("failed to persist initial log sequence: %w", err) + } + default: + return nil, fmt.Errorf("failed to read log sequence: %w", err) + } + + a.next.Store(end) + a.durableEnd.Store(end) + a.extendThreshold.Store(end) // first allocate() triggers the first extension + a.tracker.watermark.Store(end - 1) + return a, nil +} + +// allocate reserves and returns the next seq. It performs no I/O while the +// durable lease covers the seq; an extension is started in the background +// once half the lease is consumed. Only a seq at or past the durable end - +// consumption outran the background extension - extends the lease +// synchronously, and a failure there is returned: the seq is marked done +// (never written) and the caller's commit fails instead of waiting on a +// write that cannot succeed. +func (a *logSeqAllocator) allocate() (uint64, error) { + seq := a.next.Add(1) - 1 + + a.tracker.waitForSlot(seq) + + if seq >= a.extendThreshold.Load() && a.extending.CompareAndSwap(false, true) { + a.extendWG.Add(1) + go func() { + defer a.extendWG.Done() + defer a.extending.Store(false) + if err := a.extendPast(a.durableEnd.Load()); err != nil { + log.Error().Err(err).Msg("failed to extend durable log sequence lease") + } + }() + } + + if seq < a.durableEnd.Load() { + return seq, nil + } + if err := a.extendPast(seq); err != nil { + a.tracker.markDone(seq) + return 0, fmt.Errorf("extend durable log sequence lease: %w", err) + } + return seq, nil +} + +// extendPast durably extends the lease, one bandwidth at a time, until it +// covers seq. Extensions are serialised by extendMu, which no allocation +// holds except while it waits for exactly this. +func (a *logSeqAllocator) extendPast(seq uint64) error { + a.extendMu.Lock() + defer a.extendMu.Unlock() + for seq >= a.durableEnd.Load() { + newEnd := a.durableEnd.Load() + logSeqBandwidth + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, newEnd) + if err := a.db.Set(a.key, buf, a.syncWrite); err != nil { + return err + } + a.durableEnd.Store(newEnd) + a.extendThreshold.Store(newEnd - logSeqBandwidth/2) + } + return nil +} + +// markDone records that seq finished its commit-record write, successfully +// or not. It must be called exactly once per value allocate() returned, on +// every exit path (see logSeqTracker's doc comment for why). +func (a *logSeqAllocator) markDone(seq uint64) { + a.tracker.markDone(seq) +} + +// StableSeq returns the highest seq s such that every seq <= s this process +// has allocated has finished (logSeqTracker's doc comment has the proof). +func (a *logSeqAllocator) StableSeq() uint64 { + return a.tracker.stable() +} + +// close waits for any in-flight async extension to finish before the +// underlying pebble.DB is closed. +func (a *logSeqAllocator) close() { + a.extendWG.Wait() +} + +// maxSeqNumFromIndex scans the seq index for the highest seq recorded, +// shared by PebbleMetaStore.GetMaxSeqNum and the store-wide sequence's +// migration path (computeInitialLogSeq). +func maxSeqNumFromIndex(db *pebble.DB) (uint64, error) { + var maxSeq uint64 + prefix := []byte(pebblePrefixTxnSeq) + + iter, err := db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: prefixUpperBound(prefix), + }) + if err != nil { + return 0, err + } + defer iter.Close() + + if iter.Last() { + key := iter.Key() + if len(key) >= len(pebblePrefixTxnSeq)+8 { + maxSeq = binary.BigEndian.Uint64(key[len(pebblePrefixTxnSeq):]) + } + } + return maxSeq, iter.Error() +} + +// computeInitialLogSeq derives the first value the new store-wide log +// sequence may hand out, on the first run after the per-origin-sequence +// upgrade: strictly above every existing per-origin /seq/{nodeID} lease end +// and above the highest seq already recorded in the seq index, so the new +// sequence can never reissue a seq a pre-upgrade node already used. +func computeInitialLogSeq(db *pebble.DB) (uint64, error) { + max := uint64(0) + + prefix := []byte(pebblePrefixSeq) + iter, err := db.NewIter(&pebble.IterOptions{LowerBound: prefix, UpperBound: prefixUpperBound(prefix)}) + if err != nil { + return 0, err + } + for iter.SeekGE(prefix); iter.Valid(); iter.Next() { + val, err := iter.ValueAndErr() + if err != nil { + _ = iter.Close() + return 0, err + } + if len(val) >= 8 { + if v := binary.BigEndian.Uint64(val); v > max { + max = v + } + } + } + if err := iter.Close(); err != nil { + return 0, err + } + + maxIndexed, err := maxSeqNumFromIndex(db) + if err != nil { + return 0, err + } + if maxIndexed > max { + max = maxIndexed + } + + return max + 1, nil +} diff --git a/db/log_position_test.go b/db/log_position_test.go new file mode 100644 index 00000000..2ad4b64f --- /dev/null +++ b/db/log_position_test.go @@ -0,0 +1,425 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "encoding/binary" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/cockroachdb/pebble" +) + +func TestLogPositionCompareAndLess(t *testing.T) { + cases := []struct { + name string + a, b LogPosition + want int + }{ + {"equal", LogPosition{1, 1}, LogPosition{1, 1}, 0}, + {"lower seq", LogPosition{1, 5}, LogPosition{2, 1}, -1}, + {"higher seq", LogPosition{3, 1}, LogPosition{2, 5}, 1}, + {"same seq lower txn", LogPosition{5, 1}, LogPosition{5, 2}, -1}, + {"same seq higher txn", LogPosition{5, 9}, LogPosition{5, 2}, 1}, + {"zero precedes real entry", LogPosition{}, LogPosition{1, 1}, -1}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := c.a.Compare(c.b); got != c.want { + t.Fatalf("Compare(%v,%v) = %d, want %d", c.a, c.b, got, c.want) + } + wantLess := c.want < 0 + if got := c.a.Less(c.b); got != wantLess { + t.Fatalf("Less(%v,%v) = %v, want %v", c.a, c.b, got, wantLess) + } + }) + } +} + +func TestLogPositionEncodeDecodeRoundTrip(t *testing.T) { + p := LogPosition{Seq: 12345, TxnID: 67890} + got := decodeLogPosition(encodeLogPosition(p)) + if got != p { + t.Fatalf("round trip = %v, want %v", got, p) + } + if z := decodeLogPosition(nil); z != (LogPosition{}) { + t.Fatalf("short buffer decode = %v, want zero value", z) + } +} + +func openTestPebbleDB(t *testing.T) *pebble.DB { + t.Helper() + dir := filepath.Join(t.TempDir(), "logseq.pebble") + db, err := pebble.Open(dir, &pebble.Options{}) + if err != nil { + t.Fatalf("pebble.Open: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return db +} + +// TestLogSeqAllocatorInitAboveOldLeasesAndMaxSeqNum enforces the migration +// rule: the very first allocation of the +// new store-wide sequence must be strictly above every pre-upgrade +// per-origin /seq/{nodeID} lease and above GetMaxSeqNum. +func TestLogSeqAllocatorInitAboveOldLeasesAndMaxSeqNum(t *testing.T) { + db := openTestPebbleDB(t) + + // Simulate two pre-upgrade per-origin leases. + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, 500) + if err := db.Set(pebbleSeqKey(1), buf, pebble.Sync); err != nil { + t.Fatalf("seed lease 1: %v", err) + } + binary.BigEndian.PutUint64(buf, 900) + if err := db.Set(pebbleSeqKey(2), buf, pebble.Sync); err != nil { + t.Fatalf("seed lease 2: %v", err) + } + // Simulate an existing seq-index entry above both leases. + if err := db.Set(pebbleTxnSeqKey(1200, 42), nil, pebble.Sync); err != nil { + t.Fatalf("seed seq index: %v", err) + } + + alloc, err := newLogSeqAllocator(db, []byte(pebblePrefixLogSeq), pebble.Sync, func() (uint64, error) { + return computeInitialLogSeq(db) + }) + if err != nil { + t.Fatalf("newLogSeqAllocator: %v", err) + } + defer alloc.close() + + first := mustAllocate(t, alloc) + if first <= 900 || first <= 1200 { + t.Fatalf("first allocated seq %d must be strictly above the old leases (900) and GetMaxSeqNum (1200)", first) + } +} + +// TestLogSeqAllocatorResumesAtDurableEndAcrossReopen enforces the crash +// safety rule (Q3): a seq is never reissued across a restart, even though +// commit records themselves are written with NoSync. +func TestLogSeqAllocatorResumesAtDurableEndAcrossReopen(t *testing.T) { + dir := filepath.Join(t.TempDir(), "logseq.pebble") + db, err := pebble.Open(dir, &pebble.Options{}) + if err != nil { + t.Fatalf("pebble.Open: %v", err) + } + + alloc, err := newLogSeqAllocator(db, []byte(pebblePrefixLogSeq), pebble.Sync, func() (uint64, error) { return 1, nil }) + if err != nil { + t.Fatalf("newLogSeqAllocator: %v", err) + } + seen := make(map[uint64]bool) + for i := 0; i < 5; i++ { + s := mustAllocate(t, alloc) + alloc.markDone(s) + seen[s] = true + } + alloc.close() + if err := db.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + db2, err := pebble.Open(dir, &pebble.Options{}) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer db2.Close() + alloc2, err := newLogSeqAllocator(db2, []byte(pebblePrefixLogSeq), pebble.Sync, func() (uint64, error) { + t.Fatal("initialFn must not run once the key exists") + return 0, nil + }) + if err != nil { + t.Fatalf("newLogSeqAllocator (reopen): %v", err) + } + defer alloc2.close() + + for i := 0; i < 20; i++ { + s := mustAllocate(t, alloc2) + alloc2.markDone(s) + if seen[s] { + t.Fatalf("seq %d was reissued after reopen", s) + } + } +} + +// TestLogSeqAllocatorExtendsAsyncAndDoesNotReissue exercises many +// allocations across the bandwidth boundary, forcing at least one async +// lease extension, and checks every seq is unique and increasing. +func TestLogSeqAllocatorExtendsAsyncAndDoesNotReissue(t *testing.T) { + db := openTestPebbleDB(t) + alloc, err := newLogSeqAllocator(db, []byte(pebblePrefixLogSeq), pebble.Sync, func() (uint64, error) { return 1, nil }) + if err != nil { + t.Fatalf("newLogSeqAllocator: %v", err) + } + defer alloc.close() + + const n = logSeqBandwidth * 3 + var last uint64 + first := true + for i := 0; i < n; i++ { + s := mustAllocate(t, alloc) + alloc.markDone(s) + if !first && s != last+1 { + t.Fatalf("seq not contiguous: got %d after %d", s, last) + } + last = s + first = false + } +} + +func TestLogSeqTrackerStableSeqExactness(t *testing.T) { + var tr logSeqTracker + tr.watermark.Store(0) + + // Out-of-order completion: 2 finishes before 1: stable must stay 0. + tr.markDone(2) + if got := tr.stable(); got != 0 { + t.Fatalf("stable after marking 2 (not 1) = %d, want 0", got) + } + tr.markDone(1) + if got := tr.stable(); got != 2 { + t.Fatalf("stable after marking 1 and 2 = %d, want 2", got) + } + tr.markDone(3) + if got := tr.stable(); got != 3 { + t.Fatalf("stable after marking 1,2,3 = %d, want 3", got) + } +} + +// TestLogSeqTrackerStableSeqRaceStress is the -race stress test required by +// the design: concurrent writers allocate and mark done in overlapping +// order, while a reader repeatedly lists "committed" entries below a +// previously observed stable point and asserts that set never changes on a +// later re-list. This is the same guarantee ListCommittedLog relies on. +func TestLogSeqTrackerStableSeqRaceStress(t *testing.T) { + var alloc logSeqAllocator + alloc.next.Store(1) + alloc.durableEnd.Store(1 << 30) // no lease extension needed for this test + alloc.extendThreshold.Store(1 << 30) + alloc.tracker.watermark.Store(0) + + const writers = 8 + const perWriter = 2000 + + var committed sync.Map // seq -> struct{}: "durable" entries, written before markDone + var wg sync.WaitGroup + stop := make(chan struct{}) + + for w := 0; w < writers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < perWriter; i++ { + seq, err := alloc.allocate() + if err != nil { + t.Error(err) + return + } + committed.Store(seq, struct{}{}) + alloc.markDone(seq) + } + }() + } + + var readerErr atomic.Value + var readerWG sync.WaitGroup + readerWG.Add(1) + go func() { + defer readerWG.Done() + var prevStable uint64 + var prevSet map[uint64]bool + for { + select { + case <-stop: + return + default: + } + stable := alloc.StableSeq() + if stable < prevStable { + readerErr.Store("stable regressed") + return + } + set := make(map[uint64]bool) + committed.Range(func(k, _ any) bool { + seq := k.(uint64) + if seq <= stable { + set[seq] = true + } + return true + }) + // Every seq the previous, lower-or-equal snapshot listed at or + // below prevStable must still be listed now: nothing observed + // stable ever disappears. + if prevSet != nil { + for seq := range prevSet { + if seq <= prevStable && !set[seq] { + readerErr.Store("entry at or below a previously observed stable point vanished") + return + } + } + } + prevStable = stable + prevSet = set + } + }() + + wg.Wait() + close(stop) + readerWG.Wait() + + if v := readerErr.Load(); v != nil { + t.Fatalf("reader observed violation: %v", v) + } + if got := alloc.StableSeq(); got != writers*perWriter { + t.Fatalf("final StableSeq = %d, want %d (all writers done)", got, writers*perWriter) + } +} + +func BenchmarkLogSeqAllocate(b *testing.B) { + dir, err := os.MkdirTemp("", "logseq_bench") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(dir) + pdb, err := pebble.Open(filepath.Join(dir, "p"), &pebble.Options{}) + if err != nil { + b.Fatal(err) + } + defer pdb.Close() + + alloc, err := newLogSeqAllocator(pdb, []byte(pebblePrefixLogSeq), pebble.Sync, func() (uint64, error) { return 1, nil }) + if err != nil { + b.Fatal(err) + } + defer alloc.close() + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + s := mustAllocate(b, alloc) + alloc.markDone(s) + } + }) +} + +// mustAllocate allocates the next seq or fails the test. +func mustAllocate(tb testing.TB, a *logSeqAllocator) uint64 { + tb.Helper() + seq, err := a.allocate() + if err != nil { + tb.Fatalf("allocate: %v", err) + } + return seq +} + +// TestLogSeqAllocatorFailsInsteadOfSpinningWhenTheLeaseCannotBeExtended: a +// store whose lease cannot be written (here read-only; in production a full +// disk or an I/O error) fails the allocation with an error, and marks the +// seq done so the stable point is not held back by it. Before, allocate() +// spun forever, starting a new extension goroutine on every spin, and every +// commit of the store hung behind it. +// +// Mutation: drop the error return and spin until the durable end moves. +// The test times out. +func TestLogSeqAllocatorFailsInsteadOfSpinningWhenTheLeaseCannotBeExtended(t *testing.T) { + dir := filepath.Join(t.TempDir(), "logseq.pebble") + db, err := pebble.Open(dir, &pebble.Options{}) + if err != nil { + t.Fatalf("pebble.Open: %v", err) + } + if _, err := newLogSeqAllocator(db, []byte(pebblePrefixLogSeq), pebble.Sync, func() (uint64, error) { return 1, nil }); err != nil { + t.Fatalf("newLogSeqAllocator: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + ro, err := pebble.Open(dir, &pebble.Options{ReadOnly: true}) + if err != nil { + t.Fatalf("reopen read-only: %v", err) + } + defer ro.Close() + alloc, err := newLogSeqAllocator(ro, []byte(pebblePrefixLogSeq), pebble.Sync, func() (uint64, error) { + t.Fatal("initialFn must not run once the key exists") + return 0, nil + }) + if err != nil { + t.Fatalf("newLogSeqAllocator (read-only): %v", err) + } + defer alloc.close() + + done := make(chan error, 1) + go func() { + _, err := alloc.allocate() + done <- err + }() + select { + case err := <-done: + if err == nil { + t.Fatal("allocate succeeded past a lease that cannot be persisted") + } + case <-time.After(10 * time.Second): + t.Fatal("allocate hung on a lease that cannot be extended") + } + if got := alloc.StableSeq(); got != 1 { + t.Fatalf("StableSeq = %d after the failed seq 1 was marked done, want 1", got) + } +} + +// TestLogSeqAllocatorReturnsAnErrorWhenTheLeaseCannotBeExtended: once the +// durable lease is used up and extending it fails (here a read-only store), +// allocate returns the error, promptly, instead of spinning on an extension +// that cannot succeed; the seq it took is marked done, so the stable point +// is not held back by it. +func TestLogSeqAllocatorReturnsAnErrorWhenTheLeaseCannotBeExtended(t *testing.T) { + dir := filepath.Join(t.TempDir(), "logseq.pebble") + db, err := pebble.Open(dir, &pebble.Options{}) + if err != nil { + t.Fatalf("pebble.Open: %v", err) + } + alloc, err := newLogSeqAllocator(db, []byte(pebblePrefixLogSeq), pebble.Sync, func() (uint64, error) { return 1, nil }) + if err != nil { + t.Fatalf("newLogSeqAllocator: %v", err) + } + alloc.close() + if err := db.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + ro, err := pebble.Open(dir, &pebble.Options{ReadOnly: true}) + if err != nil { + t.Fatalf("reopen read-only: %v", err) + } + alloc, err = newLogSeqAllocator(ro, []byte(pebblePrefixLogSeq), pebble.Sync, func() (uint64, error) { + t.Fatal("initialFn must not run once the key exists") + return 0, nil + }) + if err != nil { + t.Fatalf("newLogSeqAllocator (read-only): %v", err) + } + + done := make(chan error, 1) + go func() { + _, err := alloc.allocate() + done <- err + }() + select { + case err := <-done: + alloc.close() + ro.Close() + if err == nil { + t.Fatal("allocate past a lease that cannot be extended returned no error") + } + case <-time.After(5 * time.Second): + // The store stays open: the spinning allocation still uses it. + t.Fatal("allocate spun on a lease extension that cannot succeed") + } + if got := alloc.StableSeq(); got != 1 { + t.Fatalf("the failed seq held the stable point back: StableSeq=%d, want 1", got) + } +} diff --git a/db/meta_schema.go b/db/meta_schema.go index 07efb7ee..f093222f 100644 --- a/db/meta_schema.go +++ b/db/meta_schema.go @@ -103,28 +103,6 @@ func (s TxnStatus) String() string { } } -// SyncStatus represents replication sync state -type SyncStatus uint8 - -const ( - SyncStatusSynced SyncStatus = 0 - SyncStatusCatchingUp SyncStatus = 1 - SyncStatusFailed SyncStatus = 2 -) - -func (s SyncStatus) String() string { - switch s { - case SyncStatusSynced: - return "SYNCED" - case SyncStatusCatchingUp: - return "CATCHING_UP" - case SyncStatusFailed: - return "FAILED" - default: - return "UNKNOWN" - } -} - // DatabaseOpType represents CREATE/DROP DATABASE operations type DatabaseOpType uint8 @@ -150,6 +128,12 @@ type DatabaseOperationSnapshot struct { Timestamp int64 `msgpack:"timestamp"` DatabaseName string `msgpack:"database_name"` Operation DatabaseOpType `msgpack:"operation"` + // Generation is the registry generation PREPARE resolved for this op (the + // coordinator's stamp, or - when unstamped - this participant's own + // locally computed value; see ReplicationEngine.prepareDatabaseOperation). + // COMMIT combines it with Operation to form the DatabaseRegistryKey it + // applies through DatabaseManager.ApplyDatabaseOp. + Generation uint64 `msgpack:"generation"` } // DDLSnapshot is a typed struct for DDL operation intents diff --git a/db/meta_store.go b/db/meta_store.go index 6dd26589..7d57afd6 100644 --- a/db/meta_store.go +++ b/db/meta_store.go @@ -14,6 +14,46 @@ import ( // ErrStopIteration signals scan callbacks to stop iteration without error var ErrStopIteration = errors.New("stop iteration") +// ErrAbortCommitted is returned by AbortTransaction when the transaction is +// already COMMITTED: the local log is append-only, so only GC may ever remove +// a committed entry. +var ErrAbortCommitted = errors.New("cannot abort a committed transaction") + +// ErrNotAbandonedBegin is returned by DiscardAbandonedBegin for a transaction +// whose record is not PendingBegunAbandoned. +var ErrNotAbandonedBegin = errors.New("transaction record is not an abandoned begin") + +// PreparedPayload is what a transaction's durable prepare +// (DurablyPrepareTransaction) recorded it holds: its captured row count and +// its persisted (non-DML) intent count. A commit compares what it finds +// against it, so a transaction whose payload something deleted after the +// prepare is refused rather than committed empty. +type PreparedPayload struct { + Rows uint64 `msgpack:"r"` + Intents uint64 `msgpack:"i"` +} + +// PendingKind classifies a transaction's local PENDING record by how far its +// PREPARE got. A transaction some peer's committed log holds was decided +// COMMITTED, and the log puller resolves this node's PENDING record for it +// by kind: commit a prepared one through the local commit path, discard an +// abandoned begin and replay, and leave a live begin to finish first. +type PendingKind uint8 + +const ( + // PendingNone: no PENDING record (absent, COMMITTED or ABORTED). + PendingNone PendingKind = iota + // PendingBegunLive: begun but not durably prepared, and still tracked by + // this process - a PREPARE executing right now. + PendingBegunLive + // PendingBegunAbandoned: begun but never durably prepared, and not + // tracked by this process - left by a PREPARE that died with the + // process (kill -9) before its durable prepare. + PendingBegunAbandoned + // PendingPrepared: durably prepared (DurablyPrepareTransaction). + PendingPrepared +) + // CapturedRowCursor iterates over raw captured rows for a transaction. // Must call Close() when done to release resources. type CapturedRowCursor interface { @@ -41,10 +81,26 @@ type MetaStore interface { GetPendingTransactions() ([]*TransactionRecord, error) Heartbeat(txnID uint64) error + // PreparedPayload returns what txnID's durable prepare recorded it holds + // (PreparedPayload); found is false for a transaction with no such record + // (never durably prepared, or prepared before the record existed). + PreparedPayload(txnID uint64) (payload PreparedPayload, found bool, err error) + + // ClassifyPending reports how far txnID's local PENDING record got (see + // PendingKind); PendingNone when there is no PENDING record. + ClassifyPending(txnID uint64) (PendingKind, error) + + // DiscardAbandonedBegin deletes txnID's PendingBegunAbandoned record with + // every row lock, intent and captured row it holds. It refuses, with + // ErrNotAbandonedBegin, a record in any other state. + DiscardAbandonedBegin(txnID uint64) error + // StoreReplayedTransaction inserts a fully-committed transaction record directly. - // Used by delta sync to record transactions that were replayed from other nodes. - // Unlike CommitTransaction, this doesn't require a prior BeginTransaction call. - StoreReplayedTransaction(txnID, nodeID uint64, commitTS hlc.Timestamp, dbName string, rowCount uint32) error + // Used to record transactions that were replayed (pulled) from another + // node's log. Unlike CommitTransaction, this doesn't require a prior + // BeginTransaction call. originNodeID is the transaction's coordinator + // (persisted as the immutable NodeID), not the replaying node. + StoreReplayedTransaction(txnID, originNodeID uint64, commitTS hlc.Timestamp, dbName string, rowCount uint32, requiredSchemaVersion uint64) error // Write intents (distributed locks) WriteIntent(txnID uint64, intentType IntentType, tableName, intentKey string, op OpType, sqlStmt string, data []byte, ts hlc.Timestamp, nodeID uint64) error @@ -54,21 +110,56 @@ type MetaStore interface { GetIntentsByTxn(txnID uint64) ([]*WriteIntentRecord, error) GetIntent(tableName, intentKey string) (*WriteIntentRecord, error) - // Replication state - GetReplicationState(peerNodeID uint64, dbName string) (*ReplicationStateRecord, error) - UpdateReplicationState(peerNodeID uint64, dbName string, lastTxnID uint64, lastTS hlc.Timestamp) error - GetMinAppliedTxnID(dbName string) (uint64, error) - GetAllReplicationStates() ([]*ReplicationStateRecord, error) - - // Sequence numbers for gap-free replication - GetNextSeqNum(nodeID uint64) (uint64, error) + // GetMaxSeqNum returns the highest seq ever recorded in this store's + // seq index (LogPosition.Seq), including entries from before this + // process's own allocations. GetMaxSeqNum() (uint64, error) - GetMinAppliedSeqNum(dbName string) (uint64, error) - // Schema/DDL + // StableSeq returns the highest seq s such that every seq <= s this + // process has allocated from the store-wide local commit sequence has + // finished, successfully or not (see logSeqTracker's doc comment in + // log_position.go for the proof this is safe to read before listing). + StableSeq() uint64 + + // ListCommittedLog returns this store's local log entries strictly + // after `after`, in position order, restricted to COMMITTED + // transactions with Seq <= the stable point read at the start of the + // call, up to limit. more is true when further stable entries exist + // beyond the returned page. + ListCommittedLog(after LogPosition, limit int) (entries []LogPosition, stable uint64, more bool, err error) + + // GetPullCursor and SetPullCursor persist C[self,peer,d]: this node's + // pull position in peerNodeID's log for this database. + GetPullCursor(peerNodeID uint64) (LogPosition, error) + SetPullCursor(peerNodeID uint64, pos LogPosition) error + + // SetConsumedPosition stores R[self,requester,d]: the `after` position + // requesterNodeID last sent when listing this store's log, as is, not + // as a max. ConsumedPositions returns every requester's last + // reported position; DeleteConsumedPosition removes a departed member's + // so it stops pinning GC. + SetConsumedPosition(requesterNodeID uint64, pos LogPosition) error + ConsumedPositions() (map[uint64]LogPosition, error) + DeleteConsumedPosition(nodeID uint64) error + + // TruncatedThrough returns T[self,d]: the highest LogPosition this + // store's GC has deleted through. It never decreases. + TruncatedThrough() (LogPosition, error) + + // SetReapplyPending durably records whether this database's local log + // must be re-applied to its SQLite file (set before a snapshot restore + // replaces the file, cleared once ReapplyLocalLog succeeds), so a crash + // between the two never loses a transaction only this node held. + SetReapplyPending(pending bool) error + ReapplyPending() (bool, error) + + // GetSchemaVersion is kept only as the migration read for the + // __marmot_schema_version table now living in each user database's own + // SQLite file: NewReplicatedDatabase consults it once, through the + // system database's store, the first time it opens a pre-existing + // database file. There is no write path any more - UpdateSchemaVersion is + // removed - and no other production caller should read it. GetSchemaVersion(dbName string) (int64, error) - UpdateSchemaVersion(dbName string, version int64, ddlSQL string, txnID uint64) error - GetAllSchemaVersions() (map[string]int64, error) TryAcquireDDLLock(dbName string, nodeID uint64, leaseDuration time.Duration) (bool, error) ReleaseDDLLock(dbName string, nodeID uint64) error @@ -96,9 +187,23 @@ type MetaStore interface { HasCDCRowLocksForTable(tableName string) (bool, error) // For DDL to check if DML in progress GetCDCTableDDLLock(tableName string) (uint64, error) // Returns txnID or 0 if no lock - // GC - CleanupStaleTransactions(timeout time.Duration) (int, error) - CleanupOldTransactionRecords(minRetention, maxRetention time.Duration, minAppliedTxnID, minAppliedSeqNum uint64) (int, error) + // Stale-transaction GC. StaleTransactionIDs lists the PENDING + // transactions whose heartbeat is older than timeout. + // AbortStaleTransaction re-checks one of them and, if it is still PENDING + // and stale, deletes its intents and captured rows and aborts it, + // reporting whether it did. The caller must hold the transaction's commit + // guard (TransactionManager.acquireCommit) across the call, so the abort + // can never interleave with a commit of the same transaction. + StaleTransactionIDs(timeout time.Duration) ([]uint64, error) + AbortStaleTransaction(txnID uint64, timeout time.Duration) (bool, error) + + // CleanupOldTransactionRecords deletes committed log entries whose + // position is <= safe and whose CommittedAt is older than minRetention, + // or whose CommittedAt is older than maxRetention regardless of safe. + // It only ever deletes a prefix of the log in position order (see the + // implementation's doc comment) and advances TruncatedThrough to the + // highest position it deleted. + CleanupOldTransactionRecords(minRetention, maxRetention time.Duration, safe LogPosition) (int, error) // Aggregation queries for anti-entropy GetMaxCommittedTxnID() (uint64, error) @@ -137,6 +242,7 @@ type TransactionRecord struct { TablesInvolved string DatabaseName string RequiredSchemaVersion uint64 // Minimum schema version required for this transaction + RowCount uint32 // Number of captured rows for this transaction (0 until committed) } // TxnImmutableRecord contains fields set once at transaction start (never modified) @@ -181,17 +287,6 @@ type IntentLock struct { TxnID uint64 } -// ReplicationStateRecord represents replication state for a peer -type ReplicationStateRecord struct { - PeerNodeID uint64 - DatabaseName string - LastAppliedTxnID uint64 - LastAppliedTSWall int64 - LastAppliedTSLogical int32 - LastSyncTime int64 - SyncStatus SyncStatus -} - // NewMetaStore creates a MemoryMetaStore (which wraps PebbleMetaStore). // basePath is the path to the user database (e.g., "/data/mydb.db"). // Creates {basePath}_meta.pebble/ directory diff --git a/db/meta_store_memory.go b/db/meta_store_memory.go index 60c84e88..eee168ef 100644 --- a/db/meta_store_memory.go +++ b/db/meta_store_memory.go @@ -44,8 +44,8 @@ func NewMemoryMetaStore(pebble *PebbleMetaStore) *MemoryMetaStore { // BeginTransaction writes immutable record to Pebble and stores status/heartbeat in memory. func (m *MemoryMetaStore) BeginTransaction(txnID, nodeID uint64, startTS hlc.Timestamp) error { - // Write immutable record to Pebble /txn/{txnID} - if err := m.pebble.writeImmutableTxnRecord(txnID, nodeID, startTS); err != nil { + // Write immutable record and pending-index key to Pebble + if err := m.pebble.writeBegunTxnRecord(txnID, nodeID, startTS); err != nil { return err } @@ -68,6 +68,36 @@ func (m *MemoryMetaStore) DurablyPrepareTransaction(txnID uint64) error { return m.pebble.DurablyPrepareTransaction(txnID) } +// ClassifyPending implements MetaStore: a begin Pebble alone would call +// abandoned is live while this process still tracks it PENDING. +func (m *MemoryMetaStore) ClassifyPending(txnID uint64) (PendingKind, error) { + kind, err := m.pebble.ClassifyPending(txnID) + if err != nil || kind != PendingBegunAbandoned { + return kind, err + } + if state, found := m.txnStore.Get(txnID); found { + if state.Status == TxnStatusPending { + return PendingBegunLive, nil + } + return PendingNone, nil + } + return PendingBegunAbandoned, nil +} + +// DiscardAbandonedBegin implements MetaStore. +func (m *MemoryMetaStore) DiscardAbandonedBegin(txnID uint64) error { + if _, found := m.txnStore.Get(txnID); found { + return fmt.Errorf("transaction %d: %w", txnID, ErrNotAbandonedBegin) + } + m.lockStore.ReleaseByTxn(txnID) + return m.pebble.DiscardAbandonedBegin(txnID) +} + +// PreparedPayload delegates to Pebble. +func (m *MemoryMetaStore) PreparedPayload(txnID uint64) (PreparedPayload, bool, error) { + return m.pebble.PreparedPayload(txnID) +} + // GetTransaction reconstructs TransactionRecord from Pebble and memory state. func (m *MemoryMetaStore) GetTransaction(txnID uint64) (*TransactionRecord, error) { // Read immutable from Pebble @@ -111,6 +141,7 @@ func (m *MemoryMetaStore) GetTransaction(txnID uint64) (*TransactionRecord, erro rec.TablesInvolved = commit.TablesInvolved rec.DatabaseName = commit.DatabaseName rec.RequiredSchemaVersion = commit.RequiredSchemaVersion + rec.RowCount = commit.RowCount } } @@ -153,7 +184,12 @@ func (m *MemoryMetaStore) CommitTransaction(txnID uint64, commitTS hlc.Timestamp return nil } -// AbortTransaction updates memory status and cleans up Pebble records. +// AbortTransaction updates memory status and cleans up Pebble records. A +// transaction already COMMITTED is refused (ErrAbortCommitted): the +// memory tier removes a transaction from txnStore as soon as it commits +// (see CommitTransaction), so finding it here still marked committed is a +// narrow race with that removal, not the common case of "not tracked in +// memory" below. func (m *MemoryMetaStore) AbortTransaction(txnID uint64) error { // Check if transaction exists in memory state, found := m.txnStore.Get(txnID) @@ -161,12 +197,15 @@ func (m *MemoryMetaStore) AbortTransaction(txnID uint64) error { // Not in memory - might be a committed/replayed transaction, delegate to Pebble return m.pebble.AbortTransaction(txnID) } + if state.Status == TxnStatusCommitted { + return ErrAbortCommitted + } // Update memory status to aborted m.txnStore.UpdateStatus(txnID, TxnStatusAborted) // Manually clean up Pebble keys (can't call pebble.AbortTransaction because it expects status in Pebble) - if err := m.pebble.deleteTransactionKeys(txnID, state.Status == TxnStatusCommitted); err != nil { + if err := m.pebble.deleteTransactionKeys(txnID); err != nil { return err } @@ -177,8 +216,8 @@ func (m *MemoryMetaStore) AbortTransaction(txnID uint64) error { } // StoreReplayedTransaction delegates to Pebble for replayed transactions. -func (m *MemoryMetaStore) StoreReplayedTransaction(txnID, nodeID uint64, commitTS hlc.Timestamp, dbName string, rowCount uint32) error { - return m.pebble.StoreReplayedTransaction(txnID, nodeID, commitTS, dbName, rowCount) +func (m *MemoryMetaStore) StoreReplayedTransaction(txnID, originNodeID uint64, commitTS hlc.Timestamp, dbName string, rowCount uint32, requiredSchemaVersion uint64) error { + return m.pebble.StoreReplayedTransaction(txnID, originNodeID, commitTS, dbName, rowCount, requiredSchemaVersion) } // WriteIntent delegates to Pebble (not migrated to memory yet). @@ -211,54 +250,66 @@ func (m *MemoryMetaStore) GetIntent(tableName, intentKey string) (*WriteIntentRe return m.pebble.GetIntent(tableName, intentKey) } -// GetReplicationState delegates to Pebble. -func (m *MemoryMetaStore) GetReplicationState(peerNodeID uint64, dbName string) (*ReplicationStateRecord, error) { - return m.pebble.GetReplicationState(peerNodeID, dbName) +// GetMaxSeqNum delegates to Pebble. +func (m *MemoryMetaStore) GetMaxSeqNum() (uint64, error) { + return m.pebble.GetMaxSeqNum() } -// UpdateReplicationState delegates to Pebble. -func (m *MemoryMetaStore) UpdateReplicationState(peerNodeID uint64, dbName string, lastTxnID uint64, lastTS hlc.Timestamp) error { - return m.pebble.UpdateReplicationState(peerNodeID, dbName, lastTxnID, lastTS) +// StableSeq delegates to Pebble. +func (m *MemoryMetaStore) StableSeq() uint64 { + return m.pebble.StableSeq() } -// GetMinAppliedTxnID delegates to Pebble. -func (m *MemoryMetaStore) GetMinAppliedTxnID(dbName string) (uint64, error) { - return m.pebble.GetMinAppliedTxnID(dbName) +// ListCommittedLog delegates to Pebble. +func (m *MemoryMetaStore) ListCommittedLog(after LogPosition, limit int) ([]LogPosition, uint64, bool, error) { + return m.pebble.ListCommittedLog(after, limit) } -// GetAllReplicationStates delegates to Pebble. -func (m *MemoryMetaStore) GetAllReplicationStates() ([]*ReplicationStateRecord, error) { - return m.pebble.GetAllReplicationStates() +// GetPullCursor delegates to Pebble. +func (m *MemoryMetaStore) GetPullCursor(peerNodeID uint64) (LogPosition, error) { + return m.pebble.GetPullCursor(peerNodeID) } -// GetNextSeqNum delegates to Pebble. -func (m *MemoryMetaStore) GetNextSeqNum(nodeID uint64) (uint64, error) { - return m.pebble.GetNextSeqNum(nodeID) +// SetPullCursor delegates to Pebble. +func (m *MemoryMetaStore) SetPullCursor(peerNodeID uint64, pos LogPosition) error { + return m.pebble.SetPullCursor(peerNodeID, pos) } -// GetMaxSeqNum delegates to Pebble. -func (m *MemoryMetaStore) GetMaxSeqNum() (uint64, error) { - return m.pebble.GetMaxSeqNum() +// SetConsumedPosition delegates to Pebble. +func (m *MemoryMetaStore) SetConsumedPosition(requesterNodeID uint64, pos LogPosition) error { + return m.pebble.SetConsumedPosition(requesterNodeID, pos) } -// GetMinAppliedSeqNum delegates to Pebble. -func (m *MemoryMetaStore) GetMinAppliedSeqNum(dbName string) (uint64, error) { - return m.pebble.GetMinAppliedSeqNum(dbName) +// ConsumedPositions delegates to Pebble. +func (m *MemoryMetaStore) ConsumedPositions() (map[uint64]LogPosition, error) { + return m.pebble.ConsumedPositions() } -// GetSchemaVersion delegates to Pebble. -func (m *MemoryMetaStore) GetSchemaVersion(dbName string) (int64, error) { - return m.pebble.GetSchemaVersion(dbName) +// DeleteConsumedPosition delegates to Pebble. +func (m *MemoryMetaStore) DeleteConsumedPosition(nodeID uint64) error { + return m.pebble.DeleteConsumedPosition(nodeID) +} + +// TruncatedThrough delegates to Pebble. +func (m *MemoryMetaStore) TruncatedThrough() (LogPosition, error) { + return m.pebble.TruncatedThrough() } -// UpdateSchemaVersion delegates to Pebble. -func (m *MemoryMetaStore) UpdateSchemaVersion(dbName string, version int64, ddlSQL string, txnID uint64) error { - return m.pebble.UpdateSchemaVersion(dbName, version, ddlSQL, txnID) +// SetReapplyPending delegates to Pebble. +func (m *MemoryMetaStore) SetReapplyPending(pending bool) error { + return m.pebble.SetReapplyPending(pending) } -// GetAllSchemaVersions delegates to Pebble. -func (m *MemoryMetaStore) GetAllSchemaVersions() (map[string]int64, error) { - return m.pebble.GetAllSchemaVersions() +// ReapplyPending delegates to Pebble. +func (m *MemoryMetaStore) ReapplyPending() (bool, error) { + return m.pebble.ReapplyPending() +} + +// GetSchemaVersion delegates to Pebble. Kept only as the schema-version +// migration read; +// see MetaStore.GetSchemaVersion's doc comment. +func (m *MemoryMetaStore) GetSchemaVersion(dbName string) (int64, error) { + return m.pebble.GetSchemaVersion(dbName) } // TryAcquireDDLLock delegates to Pebble. @@ -362,44 +413,50 @@ func (m *MemoryMetaStore) GetCDCTableDDLLock(tableName string) (uint64, error) { return txnID, nil } -// CleanupStaleTransactions iterates memory txnStore instead of Pebble /txn_idx/pend/. -func (m *MemoryMetaStore) CleanupStaleTransactions(timeout time.Duration) (int, error) { +// StaleTransactionIDs implements MetaStore from the memory txnStore: every +// PENDING transaction whose last heartbeat is older than timeout. +func (m *MemoryMetaStore) StaleTransactionIDs(timeout time.Duration) ([]uint64, error) { cutoff := time.Now().Add(-timeout).UnixNano() - cleaned := 0 - - var staleTxnIDs []uint64 - - // Iterate memory txnStore for pending transactions + var stale []uint64 m.txnStore.RangePending(func(txnID uint64) bool { - state, found := m.txnStore.Get(txnID) - if !found { - return true - } - - if state.LastHeartbeat < cutoff { - staleTxnIDs = append(staleTxnIDs, txnID) + if m.isStale(txnID, cutoff) { + stale = append(stale, txnID) } return true }) + return stale, nil +} - // Abort each stale transaction and clean up intents - for _, txnID := range staleTxnIDs { - // Clean up write intents first - _ = m.DeleteIntentsByTxn(txnID) - _ = m.pebble.DeleteIntentEntries(txnID) - - // Then abort the transaction - if err := m.AbortTransaction(txnID); err == nil { - cleaned++ - } +// AbortStaleTransaction implements MetaStore: it re-checks, under the +// caller's per-txn commit guard, that txnID is still PENDING with a stale +// heartbeat, and only then deletes its intents and captured rows and aborts +// it. +func (m *MemoryMetaStore) AbortStaleTransaction(txnID uint64, timeout time.Duration) (bool, error) { + if !m.isStale(txnID, time.Now().Add(-timeout).UnixNano()) { + return false, nil + } + if err := m.DeleteIntentsByTxn(txnID); err != nil { + return false, fmt.Errorf("delete intents of stale txn %d: %w", txnID, err) } + if err := m.pebble.DeleteIntentEntries(txnID); err != nil { + return false, fmt.Errorf("delete captured rows of stale txn %d: %w", txnID, err) + } + if err := m.AbortTransaction(txnID); err != nil { + return false, fmt.Errorf("abort stale txn %d: %w", txnID, err) + } + return true, nil +} - return cleaned, nil +// isStale reports whether txnID is PENDING in the memory txnStore with a +// last heartbeat before cutoff. +func (m *MemoryMetaStore) isStale(txnID uint64, cutoff int64) bool { + state, found := m.txnStore.Get(txnID) + return found && state.Status == TxnStatusPending && state.LastHeartbeat < cutoff } // CleanupOldTransactionRecords delegates to Pebble. -func (m *MemoryMetaStore) CleanupOldTransactionRecords(minRetention, maxRetention time.Duration, minAppliedTxnID, minAppliedSeqNum uint64) (int, error) { - return m.pebble.CleanupOldTransactionRecords(minRetention, maxRetention, minAppliedTxnID, minAppliedSeqNum) +func (m *MemoryMetaStore) CleanupOldTransactionRecords(minRetention, maxRetention time.Duration, safe LogPosition) (int, error) { + return m.pebble.CleanupOldTransactionRecords(minRetention, maxRetention, safe) } // GetMaxCommittedTxnID delegates to Pebble. @@ -450,6 +507,11 @@ func (m *MemoryMetaStore) IntentStats() (pendingIntents int, err error) { // unless its COMMIT or ABORT arrives first; until then its captured rows are // kept for that COMMIT. The captured rows of any other transaction without a // commit record are orphans of a crash and are deleted. +// +// A begin the previous process died holding before its durable prepare +// (PendingBegunAbandoned) promised nothing to its coordinator and is +// discarded, so it can never read PENDING forever and block the log pull of +// the transaction a peer committed under its id. func (m *MemoryMetaStore) ReconstructFromPebble() error { now := time.Now().UnixNano() for _, rec := range m.pebble.takeRecoveredPrepared() { @@ -462,6 +524,19 @@ func (m *MemoryMetaStore) ReconstructFromPebble() error { }) } + abandoned, err := m.pebble.abandonedBeginTxnIDs() + if err != nil { + return fmt.Errorf("failed to find abandoned transaction begins: %w", err) + } + for _, txnID := range abandoned { + if err := m.DiscardAbandonedBegin(txnID); err != nil { + return fmt.Errorf("failed to discard abandoned begin %d: %w", txnID, err) + } + } + if len(abandoned) > 0 { + log.Info().Int("count", len(abandoned)).Msg("Discarded transaction begins abandoned before their durable prepare") + } + candidates, err := m.pebble.findOrphanedCDCRawTxnIDs() if err != nil { return fmt.Errorf("failed to find orphaned CDC txnIDs: %w", err) diff --git a/db/meta_store_memory_test.go b/db/meta_store_memory_test.go index edc711b4..d7aceb44 100644 --- a/db/meta_store_memory_test.go +++ b/db/meta_store_memory_test.go @@ -233,6 +233,30 @@ func TestMemoryMetaStore_AbortTransaction(t *testing.T) { require.Nil(t, rec) } +// TestMemoryMetaStore_AbortRefusesCommitted enforces the append-only local +// log through the memory tier: once CommitTransaction removes a transaction from the +// in-memory store, AbortTransaction delegates to Pebble, which must refuse. +func TestMemoryMetaStore_AbortRefusesCommitted(t *testing.T) { + store, cleanup := setupMemoryMetaStore(t) + defer cleanup() + + txnID := uint64(601) + nodeID := uint64(6) + startTS := hlc.Timestamp{WallTime: time.Now().UnixNano(), Logical: 0} + + require.NoError(t, store.BeginTransaction(txnID, nodeID, startTS)) + commitTS := hlc.Timestamp{WallTime: time.Now().UnixNano(), Logical: 1} + require.NoError(t, store.CommitTransaction(txnID, commitTS, nil, "testdb", "", 0, 0)) + + err := store.AbortTransaction(txnID) + require.ErrorIs(t, err, ErrAbortCommitted) + + rec, err := store.GetTransaction(txnID) + require.NoError(t, err) + require.NotNil(t, rec) + require.Equal(t, TxnStatusCommitted, rec.Status) +} + func TestMemoryMetaStore_CDCRowLocks(t *testing.T) { store, cleanup := setupMemoryMetaStore(t) defer cleanup() @@ -340,10 +364,18 @@ func TestMemoryMetaStore_CleanupStaleTransactions(t *testing.T) { state3, _ := store.txnStore.Get(txn3) state3.LastHeartbeat = time.Now().Add(-3 * time.Hour).UnixNano() - // Cleanup stale transactions (older than 1 hour) - cleaned, err := store.CleanupStaleTransactions(1 * time.Hour) + // Stale transactions (older than 1 hour): txn1 and txn3 + stale, err := store.StaleTransactionIDs(1 * time.Hour) + require.NoError(t, err) + require.ElementsMatch(t, []uint64{txn1, txn3}, stale) + for _, txnID := range stale { + aborted, err := store.AbortStaleTransaction(txnID, 1*time.Hour) + require.NoError(t, err) + require.True(t, aborted) + } + aborted, err := store.AbortStaleTransaction(txn2, 1*time.Hour) require.NoError(t, err) - require.Equal(t, 2, cleaned) // txn1 and txn3 + require.False(t, aborted, "a fresh transaction must not be aborted") // Verify txn1 and txn3 are removed _, found1 := store.txnStore.Get(txn1) @@ -368,23 +400,33 @@ func TestMemoryMetaStore_DelegationMethods(t *testing.T) { require.Equal(t, int64(0), version) }) - t.Run("UpdateSchemaVersion", func(t *testing.T) { - err := store.UpdateSchemaVersion("testdb", 1, "CREATE TABLE test", 1) - require.NoError(t, err) + // UpdateSchemaVersion is removed: schema versions now live in each + // user database's own __marmot_schema_version SQLite table, not here. + // GetSchemaVersion above is kept only as the migration read. - version, err := store.GetSchemaVersion("testdb") - require.NoError(t, err) - require.Equal(t, int64(1), version) - }) + // The store-wide log sequence (formerly GetNextSeqNum(nodeID), now + // internal to CommitTransaction/StoreReplayedTransaction) is exercised + // through the public API: consecutive commits get increasing positions, + // and StableSeq catches up once both have finished. + t.Run("LogSequenceAdvancesAcrossCommits", func(t *testing.T) { + nodeID := uint64(1) + startTS := hlc.Timestamp{WallTime: time.Now().UnixNano(), Logical: 0} - t.Run("GetNextSeqNum", func(t *testing.T) { - seq1, err := store.GetNextSeqNum(1) + txnA := uint64(9001) + require.NoError(t, store.BeginTransaction(txnA, nodeID, startTS)) + require.NoError(t, store.CommitTransaction(txnA, startTS, nil, "testdb", "", 0, 0)) + recA, err := store.GetTransaction(txnA) require.NoError(t, err) - require.Greater(t, seq1, uint64(0)) + require.Greater(t, recA.SeqNum, uint64(0)) - seq2, err := store.GetNextSeqNum(1) + txnB := uint64(9002) + require.NoError(t, store.BeginTransaction(txnB, nodeID, startTS)) + require.NoError(t, store.CommitTransaction(txnB, startTS, nil, "testdb", "", 0, 0)) + recB, err := store.GetTransaction(txnB) require.NoError(t, err) - require.Equal(t, seq1+1, seq2) + require.Greater(t, recB.SeqNum, recA.SeqNum) + + require.GreaterOrEqual(t, store.StableSeq(), recB.SeqNum) }) } @@ -483,10 +525,81 @@ func TestReconstructFromPebble_CleansOrphanedCDCRaw(t *testing.T) { cursor.Close() require.Equal(t, 0, count, "Orphaned CDC raw data should be deleted") - // Verify immutable record still exists + // The begin was never durably prepared: the whole record is discarded, so + // it can never read PENDING forever after the restart. immutable, err := newStore.pebble.readImmutableTxnRecord(txnID) require.NoError(t, err) - require.NotNil(t, immutable, "Immutable record should still exist") + require.Nil(t, immutable, "an abandoned begin's record must be discarded at reconstruction") + rec, err := newStore.GetTransaction(txnID) + require.NoError(t, err) + require.Nil(t, rec) +} + +// TestReconstructFromPebble_KeepsDurablyPreparedBegin pins the other side of +// the abandoned-begin discard: a begin that reached its durable prepare is +// kept PENDING across the restart, for its COMMIT. +func TestReconstructFromPebble_KeepsDurablyPreparedBegin(t *testing.T) { + store, cleanup := setupMemoryMetaStore(t) + defer cleanup() + + txnID := uint64(1150) + require.NoError(t, store.BeginTransaction(txnID, 11, hlc.Timestamp{WallTime: time.Now().UnixNano()})) + require.NoError(t, store.DurablyPrepareTransaction(txnID)) + store.txnStore.Remove(txnID) + + newStore := NewMemoryMetaStore(store.pebble) + require.NoError(t, newStore.ReconstructFromPebble()) + + kind, err := newStore.ClassifyPending(txnID) + require.NoError(t, err) + require.Equal(t, PendingPrepared, kind) + rec, err := newStore.GetTransaction(txnID) + require.NoError(t, err) + require.NotNil(t, rec) + require.Equal(t, TxnStatusPending, rec.Status) +} + +// TestMemoryMetaStore_ClassifyPendingAndDiscard pins each PendingKind and +// that DiscardAbandonedBegin refuses every record that is not abandoned. +func TestMemoryMetaStore_ClassifyPendingAndDiscard(t *testing.T) { + store, cleanup := setupMemoryMetaStore(t) + defer cleanup() + ts := hlc.Timestamp{WallTime: time.Now().UnixNano()} + + kind, err := store.ClassifyPending(1) + require.NoError(t, err) + require.Equal(t, PendingNone, kind, "no record") + + require.NoError(t, store.BeginTransaction(2, 1, ts)) + kind, err = store.ClassifyPending(2) + require.NoError(t, err) + require.Equal(t, PendingBegunLive, kind) + require.ErrorIs(t, store.DiscardAbandonedBegin(2), ErrNotAbandonedBegin, "a live begin must not be discarded") + + store.txnStore.Remove(2) + kind, err = store.ClassifyPending(2) + require.NoError(t, err) + require.Equal(t, PendingBegunAbandoned, kind) + require.NoError(t, store.WriteCapturedRow(2, 1, []byte("row"))) + require.NoError(t, store.DiscardAbandonedBegin(2)) + rec, err := store.GetTransaction(2) + require.NoError(t, err) + require.Nil(t, rec) + require.False(t, store.HasCapturedRows(2), "discard must drop the begin's captured rows") + + require.NoError(t, store.BeginTransaction(3, 1, ts)) + require.NoError(t, store.DurablyPrepareTransaction(3)) + kind, err = store.ClassifyPending(3) + require.NoError(t, err) + require.Equal(t, PendingPrepared, kind) + store.txnStore.Remove(3) + require.ErrorIs(t, store.DiscardAbandonedBegin(3), ErrNotAbandonedBegin, "a prepared txn must not be discarded") + + require.NoError(t, store.BeginTransaction(4, 1, ts)) + require.NoError(t, store.CommitTransaction(4, ts, nil, "db", "", 0, 0)) + kind, err = store.ClassifyPending(4) + require.NoError(t, err) + require.Equal(t, PendingNone, kind, "committed") } func TestReconstructFromPebble_KeepsCommittedCDCRaw(t *testing.T) { diff --git a/db/meta_store_pebble.go b/db/meta_store_pebble.go index 39d34b08..5fc623d0 100644 --- a/db/meta_store_pebble.go +++ b/db/meta_store_pebble.go @@ -31,7 +31,6 @@ const ( pebblePrefixTxnByID = "/txn_idx/txnid/" // /txn_idx/txnid/{8 bytes txnID} - primary index for streaming pebblePrefixCDCRaw = "/cdc/raw/" // legacy per-row CDC prefix; rejected on open pebblePrefixCDCManifest = "/cdc/manifest/" // /cdc/manifest/{8 bytes txnID} - pebblePrefixRepl = "/repl/" // /repl/{8 bytes peerNodeID}/{dbName} pebblePrefixSchema = "/schema/" // /schema/{dbName} pebblePrefixDDLLock = "/ddl/" // /ddl/{dbName} pebblePrefixSeq = "/seq/" // /seq/{8 bytes nodeID} @@ -39,6 +38,31 @@ const ( pebblePrefixCounter = "/meta/" // /meta/{counterName} pebblePrefixTxnCommit = "/txn_commit/" // /txn_commit/{8 bytes txnID} pebblePrefixTxnStatus = "/txn_status/" // /txn_status/{8 bytes txnID} + pebblePrefixTxnPrepared = "/txn_prepared/" // /txn_prepared/{8 bytes txnID} -> PreparedPayload + + // pebblePrefixLogSeq is the single, store-wide key holding the durable + // end of the local commit sequence's current lease (logSeqAllocator). It + // replaces the per-origin /seq/{nodeID} leases as the source of + // LogPosition.Seq for every committed transaction, whatever its origin. + pebblePrefixLogSeq = "/log_seq/current" + + // pebblePrefixPullCursor stores C[n,p,d]: this node's pull position in + // peer p's log for this database, kept apart from the per-peer consumed + // positions. + pebblePrefixPullCursor = "/log_pull_cursor/" // /log_pull_cursor/{8 bytes peerNodeID} + + // pebblePrefixConsumedPos stores R[self,requester,d]: the last `after` + // position a requester reported when listing this store's log, kept as + // is rather than as a max. + pebblePrefixConsumedPos = "/log_consumed_pos/" // /log_consumed_pos/{8 bytes requesterNodeID} + + // pebbleKeyTruncatedThrough stores T[self,d]: the highest LogPosition + // this store's GC has deleted through. It only ever increases. + pebbleKeyTruncatedThrough = "/log_truncated_through" + + // pebbleKeyReapplyPending marks that this database's local log must be + // re-applied to a restored SQLite file. Present means pending. + pebbleKeyReapplyPending = "/log_reapply_pending" ) // Sharded lock for WriteIntent serialization (prevents TOCTOU race) @@ -56,9 +80,9 @@ type PebbleMetaStore struct { // Idempotent close closed atomic.Bool - // Sequence generators (Pebble doesn't have native Sequence API) - sequences map[uint64]*AtomicSequence - seqMu sync.Mutex + // logSeq mints the store-wide local commit sequence (LogPosition.Seq) + // shared by every origin's committed transactions (log_position.go). + logSeq *logSeqAllocator // Persistent counters for O(1) lookups counters *PebbleCounter @@ -228,7 +252,6 @@ func NewPebbleMetaStore(path string, opts PebbleMetaStoreOptions) (*PebbleMetaSt store := &PebbleMetaStore{ db: db, path: path, - sequences: make(map[uint64]*AtomicSequence), rowLocks: NewRowLockStore(), dmlIntents: xsync.NewMapOf[uint64, *xsync.MapOf[string, *WriteIntentRecord]](), persistedIntentTxns: xsync.NewMapOf[uint64, struct{}](), @@ -242,6 +265,15 @@ func NewPebbleMetaStore(path string, opts PebbleMetaStoreOptions) (*PebbleMetaSt // Initialize persistent counters store.counters = NewPebbleCounter(db, pebblePrefixCounter, 10) + logSeq, err := newLogSeqAllocator(db, []byte(pebblePrefixLogSeq), store.syncWrite, func() (uint64, error) { + return computeInitialLogSeq(db) + }) + if err != nil { + db.Close() + return nil, fmt.Errorf("failed to open log sequence: %w", err) + } + store.logSeq = logSeq + cdcLog, err := openCDCSegmentLog(path) if err != nil { db.Close() @@ -512,13 +544,8 @@ func (s *PebbleMetaStore) Close() error { return nil // Already closed } - // Release all sequence generators (persist final values) - s.seqMu.Lock() - for _, seq := range s.sequences { - seq.Close() - } - s.sequences = nil - s.seqMu.Unlock() + // Wait for any in-flight async lease extension before the pebble.DB closes. + s.logSeq.close() if err := s.cdcLog.close(); err != nil { log.Warn().Err(err).Str("path", filepath.Join(s.path, cdcSegmentDirName)).Msg("Failed to close CDC segment log") @@ -532,82 +559,6 @@ func (s *PebbleMetaStore) Checkpoint() error { return nil } -// AtomicSequence provides contention-free sequence number generation. -// Pre-allocates batches of IDs to minimize disk writes. -type AtomicSequence struct { - db *pebble.DB - key []byte - bandwidth uint64 // IDs to pre-allocate (e.g., 1000) - - mu sync.Mutex - nextVal uint64 // Next value to return - leaseEnd uint64 // End of current lease -} - -// NewAtomicSequence creates a new sequence generator. -// On startup, reads the persisted lease end and continues from there. -func NewAtomicSequence(db *pebble.DB, key []byte, bandwidth uint64) (*AtomicSequence, error) { - var leaseEnd uint64 - - val, closer, err := db.Get(key) - if err == nil { - if len(val) >= 8 { - leaseEnd = binary.BigEndian.Uint64(val) - } - closer.Close() - } else if err != pebble.ErrNotFound { - return nil, fmt.Errorf("failed to read sequence: %w", err) - } - - return &AtomicSequence{ - db: db, - key: key, - bandwidth: bandwidth, - nextVal: leaseEnd, // Resume from where we left off - leaseEnd: leaseEnd, - }, nil -} - -// Next returns the next sequence number. -// Pre-allocates a batch of IDs when the current lease is exhausted. -func (s *AtomicSequence) Next() (uint64, error) { - s.mu.Lock() - defer s.mu.Unlock() - - if s.nextVal >= s.leaseEnd { - // Allocate new batch - newLease := s.leaseEnd + s.bandwidth - - // Persist new lease end to disk - buf := make([]byte, 8) - binary.BigEndian.PutUint64(buf, newLease) - if err := s.db.Set(s.key, buf, pebble.NoSync); err != nil { - return 0, fmt.Errorf("failed to persist sequence: %w", err) - } - - s.leaseEnd = newLease - } - - val := s.nextVal - s.nextVal++ - return val, nil -} - -// Close persists any unused portion of the current lease. -// This minimizes gaps on restart. -func (s *AtomicSequence) Close() error { - s.mu.Lock() - defer s.mu.Unlock() - - // Persist current nextVal so we don't have gaps on restart - buf := make([]byte, 8) - binary.BigEndian.PutUint64(buf, s.nextVal) - return s.db.Set(s.key, buf, pebble.NoSync) -} - -// seqBandwidth is the number of sequence numbers to pre-allocate at once -const pebbleSeqBandwidth = 1000 - // Key helper functions - use binary encoding for uint64 (8 bytes vs 16 hex chars) // Big-endian preserves lexicographic sort order for range scans @@ -658,6 +609,10 @@ func pebbleTxnCommitKey(txnID uint64) []byte { return buildKeyUint64(pebblePrefixTxnCommit, txnID) } +func pebbleTxnPreparedKey(txnID uint64) []byte { + return buildKeyUint64(pebblePrefixTxnPrepared, txnID) +} + func pebbleTxnStatusKey(txnID uint64) []byte { return buildKeyUint64(pebblePrefixTxnStatus, txnID) } @@ -683,16 +638,6 @@ func pebbleCDCManifestKey(txnID uint64) []byte { return buildKeyUint64(pebblePrefixCDCManifest, txnID) } -func pebbleReplKey(peerNodeID uint64, dbName string) []byte { - key := make([]byte, len(pebblePrefixRepl)+8+1+len(dbName)) - n := copy(key, pebblePrefixRepl) - binary.BigEndian.PutUint64(key[n:], peerNodeID) - n += 8 - key[n] = '/' - copy(key[n+1:], dbName) - return key -} - func pebbleSchemaKey(dbName string) []byte { return buildKeyString(pebblePrefixSchema, dbName) } @@ -705,6 +650,14 @@ func pebbleSeqKey(nodeID uint64) []byte { return buildKeyUint64(pebblePrefixSeq, nodeID) } +func pebblePullCursorKey(peerNodeID uint64) []byte { + return buildKeyUint64(pebblePrefixPullCursor, peerNodeID) +} + +func pebbleConsumedPosKey(requesterNodeID uint64) []byte { + return buildKeyUint64(pebblePrefixConsumedPos, requesterNodeID) +} + // prefixUpperBound returns prefix + 0xFF... for range iteration func prefixUpperBound(prefix []byte) []byte { upper := make([]byte, len(prefix)+8) @@ -763,6 +716,127 @@ func (s *PebbleMetaStore) writeImmutableTxnRecord(txnID, nodeID uint64, startTS return s.db.Set(pebbleTxnKey(txnID), native.Bytes(), pebble.NoSync) } +// writeBegunTxnRecord writes a memory-tier begin: the immutable record plus +// the pending-index key, in one batch. The index key is what lets +// ReconstructFromPebble find a begin the process died holding before its +// durable prepare (PendingBegunAbandoned) without scanning every /txn/ key. +func (s *PebbleMetaStore) writeBegunTxnRecord(txnID, nodeID uint64, startTS hlc.Timestamp) error { + immutable := &TxnImmutableRecord{ + TxnID: txnID, + NodeID: nodeID, + StartTSWall: startTS.WallTime, + StartTSLogical: startTS.Logical, + CreatedAt: time.Now().UnixNano(), + } + native, err := encoding.MarshalNative(immutable) + if err != nil { + return fmt.Errorf("failed to marshal immutable record: %w", err) + } + defer native.Dispose() + + batch := s.db.NewBatch() + defer batch.Close() + if err := batch.Set(pebbleTxnKey(txnID), native.Bytes(), nil); err != nil { + return err + } + if err := batch.Set(pebbleTxnPendingKey(txnID), nil, nil); err != nil { + return err + } + return batch.Commit(pebble.NoSync) +} + +// ClassifyPending implements MetaStore. Without a memory tier a record whose +// status key is missing is always abandoned: PebbleMetaStore.BeginTransaction +// writes the PENDING status with the record itself. +func (s *PebbleMetaStore) ClassifyPending(txnID uint64) (PendingKind, error) { + immutable, err := s.readImmutableTxnRecord(txnID) + if err != nil || immutable == nil { + return PendingNone, err + } + status, err := s.readTxnStatus(txnID) + if err == pebble.ErrNotFound { + commit, cerr := s.readCommitRecord(txnID) + if cerr != nil { + return PendingNone, cerr + } + if commit != nil { + return PendingNone, nil + } + return PendingBegunAbandoned, nil + } + if err != nil { + return PendingNone, err + } + if status == TxnStatusPending { + return PendingPrepared, nil + } + return PendingNone, nil +} + +// DiscardAbandonedBegin implements MetaStore. +func (s *PebbleMetaStore) DiscardAbandonedBegin(txnID uint64) error { + kind, err := s.ClassifyPending(txnID) + if err != nil { + return err + } + if kind != PendingBegunAbandoned { + return fmt.Errorf("transaction %d: %w", txnID, ErrNotAbandonedBegin) + } + s.rowLocks.ReleaseByTxn(txnID) + if err := s.DeleteIntentsByTxn(txnID); err != nil { + return fmt.Errorf("delete intents of abandoned txn %d: %w", txnID, err) + } + if err := s.DeleteCapturedRows(txnID); err != nil { + return fmt.Errorf("delete captured rows of abandoned txn %d: %w", txnID, err) + } + s.dmlIntents.Delete(txnID) + s.persistedIntentTxns.Delete(txnID) + + batch := s.db.NewBatch() + defer batch.Close() + if err := batch.Delete(pebbleTxnKey(txnID), nil); err != nil { + return err + } + if err := batch.Delete(pebbleTxnPendingKey(txnID), nil); err != nil { + return err + } + return batch.Commit(pebble.NoSync) +} + +// abandonedBeginTxnIDs lists every pending-index entry whose record is +// PendingBegunAbandoned from this store's own view - the caller excludes +// begins it still tracks. +func (s *PebbleMetaStore) abandonedBeginTxnIDs() ([]uint64, error) { + prefix := []byte(pebblePrefixTxnPending) + iter, err := s.db.NewIter(&pebble.IterOptions{LowerBound: prefix, UpperBound: prefixUpperBound(prefix)}) + if err != nil { + return nil, err + } + var candidates []uint64 + for iter.SeekGE(prefix); iter.Valid(); iter.Next() { + key := iter.Key() + if len(key) < len(pebblePrefixTxnPending)+8 { + continue + } + candidates = append(candidates, binary.BigEndian.Uint64(key[len(pebblePrefixTxnPending):])) + } + if err := iter.Close(); err != nil { + return nil, err + } + + abandoned := candidates[:0] + for _, txnID := range candidates { + kind, err := s.ClassifyPending(txnID) + if err != nil { + return nil, err + } + if kind == PendingBegunAbandoned { + abandoned = append(abandoned, txnID) + } + } + return abandoned, nil +} + // readImmutableTxnRecord reads the immutable transaction record from /txn/{txnID}. func (s *PebbleMetaStore) readImmutableTxnRecord(txnID uint64) (*TxnImmutableRecord, error) { val, closer, err := s.db.Get(pebbleTxnKey(txnID)) @@ -801,9 +875,12 @@ func (s *PebbleMetaStore) readCommitRecord(txnID uint64) (*TxnCommitRecord, erro return &commit, nil } -// deleteTransactionKeys deletes transaction records from Pebble. -// Used by MemoryMetaStore which manages status/heartbeat in memory. -func (s *PebbleMetaStore) deleteTransactionKeys(txnID uint64, isCommitted bool) error { +// deleteTransactionKeys deletes a non-committed transaction's records from +// Pebble. Used by MemoryMetaStore, which manages status/heartbeat in +// memory, to abort a transaction it still holds PENDING; the memory tier +// refuses to abort a COMMITTED one before ever calling this, so it +// never touches the seq index, which only GC may delete. +func (s *PebbleMetaStore) deleteTransactionKeys(txnID uint64) error { s.dmlIntents.Delete(txnID) s.persistedIntentTxns.Delete(txnID) batch := s.db.NewBatch() @@ -816,6 +893,9 @@ func (s *PebbleMetaStore) deleteTransactionKeys(txnID uint64, isCommitted bool) // Delete commit record if exists _ = batch.Delete(pebbleTxnCommitKey(txnID), nil) + // A memory-tier begin writes the pending-index key too (writeBegunTxnRecord). + _ = batch.Delete(pebbleTxnPendingKey(txnID), nil) + _ = batch.Delete(pebbleTxnPreparedKey(txnID), nil) // A prepared transaction's status stays behind as ABORTED: a segment // prepare record outlives the abort, and recovery must not take it for a @@ -824,20 +904,10 @@ func (s *PebbleMetaStore) deleteTransactionKeys(txnID uint64, isCommitted bool) if err := batch.Set(pebbleTxnStatusKey(txnID), []byte{byte(TxnStatusAborted)}, nil); err != nil { return err } - _ = batch.Delete(pebbleTxnPendingKey(txnID), nil) } else if err != pebble.ErrNotFound { return err } - // If committed, try to remove from sequence index - if isCommitted { - // Try to read commit record to get seqNum before deleting - commit, err := s.readCommitRecord(txnID) - if err == nil && commit != nil { - _ = batch.Delete(pebbleTxnSeqKey(commit.SeqNum, txnID), nil) - } - } - return batch.Commit(pebble.NoSync) } @@ -896,14 +966,81 @@ func (s *PebbleMetaStore) BeginTransaction(txnID, nodeID uint64, startTS hlc.Tim // have been written. The default path queues segment fsync through the grouped // CDC syncer to match Pebble NoSync hot-path behavior; set // MARMOT_CDC_PREPARE_SYNC=strict to wait for fsync before PREPARE ACK. +// +// It also records the transaction's PreparedPayload, in the same batch as the +// PENDING status, for its commit to check against (PreparedPayload). func (s *PebbleMetaStore) DurablyPrepareTransaction(txnID uint64) error { - statusBuf := []byte{byte(TxnStatusPending)} - if err := s.db.Set(pebbleTxnStatusKey(txnID), statusBuf, pebble.NoSync); err != nil { + payload, err := s.currentPayload(txnID) + if err != nil { + return err + } + encoded, err := encoding.Marshal(payload) + if err != nil { + return fmt.Errorf("encode prepared payload: %w", err) + } + batch := s.db.NewBatch() + defer batch.Close() + if err := batch.Set(pebbleTxnStatusKey(txnID), []byte{byte(TxnStatusPending)}, nil); err != nil { + return err + } + if err := batch.Set(pebbleTxnPreparedKey(txnID), encoded, nil); err != nil { + return err + } + if err := batch.Commit(pebble.NoSync); err != nil { return err } return s.sealCapturedRows(txnID, cdcPrepareSyncStrict()) } +// currentPayload counts what txnID holds right now: its captured rows +// (pending or already sealed) and its persisted intents. +func (s *PebbleMetaStore) currentPayload(txnID uint64) (PreparedPayload, error) { + var payload PreparedPayload + if rows, pending := s.cdcLog.pendingRowCount(txnID); pending { + payload.Rows = rows + } else { + data, err := s.getValueCopy(pebbleCDCManifestKey(txnID)) + if err != nil && err != pebble.ErrNotFound { + return payload, err + } + if err == nil { + var manifest cdcSegmentTxnManifest + if err := encoding.Unmarshal(data, &manifest); err != nil { + return payload, fmt.Errorf("decode CDC manifest txn %d: %w", txnID, err) + } + payload.Rows = manifest.RowCount + } + } + if _, persisted := s.persistedIntentTxns.Load(txnID); !persisted { + return payload, nil + } + prefix := pebbleIntentByTxnPrefix(txnID) + iter, err := s.db.NewIter(&pebble.IterOptions{LowerBound: prefix, UpperBound: prefixUpperBound(prefix)}) + if err != nil { + return payload, err + } + for iter.SeekGE(prefix); iter.Valid(); iter.Next() { + payload.Intents++ + } + return payload, iter.Close() +} + +// PreparedPayload implements MetaStore. +func (s *PebbleMetaStore) PreparedPayload(txnID uint64) (PreparedPayload, bool, error) { + var payload PreparedPayload + data, err := s.getValueCopy(pebbleTxnPreparedKey(txnID)) + if err == pebble.ErrNotFound { + return payload, false, nil + } + if err != nil { + return payload, false, err + } + if err := encoding.Unmarshal(data, &payload); err != nil { + return payload, false, fmt.Errorf("decode prepared payload of txn %d: %w", txnID, err) + } + return payload, true, nil +} + // CommitTransaction marks a transaction as COMMITTED func (s *PebbleMetaStore) CommitTransaction(txnID uint64, commitTS hlc.Timestamp, statements []byte, dbName, tablesInvolved string, requiredSchemaVersion uint64, rowCount uint32) error { log.Debug(). @@ -914,28 +1051,27 @@ func (s *PebbleMetaStore) CommitTransaction(txnID uint64, commitTS hlc.Timestamp Uint64("required_schema_version", requiredSchemaVersion). Msg("CDC: CommitTransaction") - if err := s.SealCapturedRows(txnID); err != nil { - return fmt.Errorf("seal captured rows: %w", err) - } - - // Step 1: Read ONLY immutable record to get NodeID - immutableData, err := s.getValueCopy(pebbleTxnKey(txnID)) - if err == pebble.ErrNotFound { - return fmt.Errorf("transaction %d not found", txnID) - } + // Allocate the store-wide log position first, and mark it done on every + // exit path from here on (logSeqTracker): a seq that turns out unused (an + // early return below) is simply a gap, never reissued, and the tracker + // still reports it done immediately since no commit record was written + // for it. + seqNum, err := s.nextLogSeq() if err != nil { return err } + defer s.logSeq.markDone(seqNum) - var immutable TxnImmutableRecord - if err := encoding.Unmarshal(immutableData, &immutable); err != nil { - return err + if err := s.SealCapturedRows(txnID); err != nil { + return fmt.Errorf("seal captured rows: %w", err) } - // Step 2: Get sequence number using contention-free Sequence API - seqNum, err := s.GetNextSeqNum(immutable.NodeID) - if err != nil { - return fmt.Errorf("failed to get next seq_num: %w", err) + // Step 1: Confirm the transaction exists. + if _, err := s.getValueCopy(pebbleTxnKey(txnID)); err != nil { + if err == pebble.ErrNotFound { + return fmt.Errorf("transaction %d not found", txnID) + } + return err } now := time.Now().UnixNano() @@ -974,10 +1110,14 @@ func (s *PebbleMetaStore) CommitTransaction(txnID uint64, commitTS hlc.Timestamp // Heartbeat is in-memory only - no need to write at commit - // Remove from pending index + // Remove from pending index, and the prepared payload the commit no + // longer needs. if err := batch.Delete(pebbleTxnPendingKey(txnID), nil); err != nil { return err } + if err := batch.Delete(pebbleTxnPreparedKey(txnID), nil); err != nil { + return err + } // Add to sequence index (kept for backward compatibility and GC) if err := batch.Set(pebbleTxnSeqKey(seqNum, txnID), nil, nil); err != nil { @@ -1000,24 +1140,31 @@ func (s *PebbleMetaStore) CommitTransaction(txnID uint64, commitTS hlc.Timestamp return batch.Commit(pebble.NoSync) // Rely on replication for durability (like CDB/TiKV/FDB) } -// StoreReplayedTransaction inserts a fully-committed transaction record directly. -func (s *PebbleMetaStore) StoreReplayedTransaction(txnID, nodeID uint64, commitTS hlc.Timestamp, dbName string, rowCount uint32) error { +// StoreReplayedTransaction inserts a fully-committed transaction record +// directly, from a txn replayed from another node's log. originNodeID is +// persisted as the immutable NodeID (the owner of the DDL incarnations the +// txn creates, which must be the same on every node that replays it); it +// is the node that coordinated the transaction, not the node replaying it. +// requiredSchemaVersion is persisted on the commit record exactly as the +// originating commit recorded it. +func (s *PebbleMetaStore) StoreReplayedTransaction(txnID, originNodeID uint64, commitTS hlc.Timestamp, dbName string, rowCount uint32, requiredSchemaVersion uint64) error { log.Debug(). Uint64("txn_id", txnID). - Uint64("node_id", nodeID). + Uint64("origin_node_id", originNodeID). Int64("commit_ts", commitTS.WallTime). Str("database", dbName). Uint32("row_count", rowCount). + Uint64("required_schema_version", requiredSchemaVersion). Msg("StoreReplayedTransaction: storing replayed transaction") - if err := s.SealCapturedRows(txnID); err != nil { - return fmt.Errorf("seal replayed rows: %w", err) + seqNum, err := s.nextLogSeq() + if err != nil { + return err } + defer s.logSeq.markDone(seqNum) - // Get sequence number - seqNum, err := s.GetNextSeqNum(nodeID) - if err != nil { - return fmt.Errorf("failed to get next seq_num: %w", err) + if err := s.SealCapturedRows(txnID); err != nil { + return fmt.Errorf("seal replayed rows: %w", err) } now := time.Now().UnixNano() @@ -1025,7 +1172,7 @@ func (s *PebbleMetaStore) StoreReplayedTransaction(txnID, nodeID uint64, commitT // Create immutable record immutable := &TxnImmutableRecord{ TxnID: txnID, - NodeID: nodeID, + NodeID: originNodeID, StartTSWall: commitTS.WallTime, StartTSLogical: commitTS.Logical, CreatedAt: now, @@ -1045,7 +1192,7 @@ func (s *PebbleMetaStore) StoreReplayedTransaction(txnID, nodeID uint64, commitT CommittedAt: now, TablesInvolved: "", DatabaseName: dbName, - RequiredSchemaVersion: 0, + RequiredSchemaVersion: requiredSchemaVersion, RowCount: rowCount, } @@ -1096,7 +1243,10 @@ func (s *PebbleMetaStore) StoreReplayedTransaction(txnID, nodeID uint64, commitT return batch.Commit(pebble.NoSync) } -// AbortTransaction deletes a transaction record +// AbortTransaction deletes a PENDING transaction's record. A COMMITTED +// transaction is refused with ErrAbortCommitted: the local log is +// append-only, and only GC ever removes a +// committed entry's seq-index key or records. func (s *PebbleMetaStore) AbortTransaction(txnID uint64) error { defer s.dmlIntents.Delete(txnID) defer s.persistedIntentTxns.Delete(txnID) @@ -1109,20 +1259,8 @@ func (s *PebbleMetaStore) AbortTransaction(txnID uint64) error { return err } - var seqNum uint64 - // If committed, read commit record to get SeqNum for cleanup if status == TxnStatusCommitted { - commitData, err := s.getValueCopy(pebbleTxnCommitKey(txnID)) - if err != nil && err != pebble.ErrNotFound { - return err - } - if err == nil { - var commit TxnCommitRecord - if err := encoding.Unmarshal(commitData, &commit); err != nil { - return err - } - seqNum = commit.SeqNum - } + return ErrAbortCommitted } batch := s.db.NewBatch() @@ -1138,13 +1276,9 @@ func (s *PebbleMetaStore) AbortTransaction(txnID uint64) error { return err } - // Remove from pending index (best-effort cleanup) + // Remove from pending index and the prepared payload (best-effort cleanup) _ = batch.Delete(pebbleTxnPendingKey(txnID), nil) - - // Remove from sequence index if it had one (best-effort cleanup) - if seqNum > 0 { - _ = batch.Delete(pebbleTxnSeqKey(seqNum, txnID), nil) - } + _ = batch.Delete(pebbleTxnPreparedKey(txnID), nil) // NoSync: AbortTransaction is cleanup. Idempotent - can be redone. return batch.Commit(pebble.NoSync) @@ -1168,9 +1302,14 @@ func (s *PebbleMetaStore) GetTransaction(txnID uint64) (*TransactionRecord, erro return nil, err } - // Read status from /txn_status/ + // Read status from /txn_status/. A transaction begun through the memory + // tier has only its immutable record here until it is durably prepared + // or committed; one found that way after a restart was abandoned while + // PENDING, and reads as PENDING rather than failing the lookup. status, err := s.readTxnStatus(txnID) - if err != nil { + if err == pebble.ErrNotFound { + status = TxnStatusPending + } else if err != nil { return nil, err } @@ -1204,6 +1343,7 @@ func (s *PebbleMetaStore) GetTransaction(txnID uint64) (*TransactionRecord, erro rec.TablesInvolved = commit.TablesInvolved rec.DatabaseName = commit.DatabaseName rec.RequiredSchemaVersion = commit.RequiredSchemaVersion + rec.RowCount = commit.RowCount } } @@ -1270,8 +1410,7 @@ func (s *PebbleMetaStore) WriteIntent(txnID uint64, intentType IntentType, table if !acquired { // Race condition telemetry.WriteConflictsTotal.With("intent", "resolve_race").Inc() - return fmt.Errorf("write-write conflict: row %s:%s locked by transaction %d (current txn: %d)", - tableName, intentKey, existingTxnID, txnID) + return writeConflictError(tableName, intentKey, existingTxnID, txnID) } } @@ -1323,6 +1462,15 @@ func (s *PebbleMetaStore) WriteIntent(txnID uint64, intentType IntentType, table return nil } +// writeConflictError reports txnID's write-write conflict with holderTxnID on +// one row. The intent key is binary (an encoded primary key), and the message +// travels as a protobuf string, which must be valid UTF-8, so the key is +// hex-encoded rather than printed raw. +func writeConflictError(tableName, intentKey string, holderTxnID, txnID uint64) error { + return fmt.Errorf("write-write conflict: row %s:%x locked by transaction %d (current txn: %d)", + tableName, intentKey, holderTxnID, txnID) +} + // resolveIntentConflictPebble decides whether the intent existingTxnID holds on // a row may be overwritten by txnID. Only an ended holder yields: one with no // transaction record, or one that committed or aborted. @@ -1363,8 +1511,7 @@ func (s *PebbleMetaStore) resolveIntentConflictPebble(existingTxnID, txnID uint6 Msg("Cleaning up intent from aborted transaction") default: - return fmt.Errorf("write-write conflict: row %s:%s locked by transaction %d (current txn: %d)", - tableName, intentKey, existingTxnID, txnID) + return writeConflictError(tableName, intentKey, existingTxnID, txnID) } // Delete /intent_txn/ index for the overwritten transaction @@ -1585,115 +1732,173 @@ func (s *PebbleMetaStore) GetIntent(tableName, intentKey string) (*WriteIntentRe return intent, nil } -// GetReplicationState retrieves replication state for a peer -func (s *PebbleMetaStore) GetReplicationState(peerNodeID uint64, dbName string) (*ReplicationStateRecord, error) { - val, closer, err := s.db.Get(pebbleReplKey(peerNodeID, dbName)) - if err == pebble.ErrNotFound { - return nil, nil +// ListCommittedLog returns this store's local log entries strictly after +// `after`, in position order, restricted to COMMITTED transactions with +// Seq <= the stable point read at the start of the call (StableSeq, taken +// before the iterator opens; see logSeqTracker's doc comment for why that +// snapshot is safe), up to limit. +func (s *PebbleMetaStore) ListCommittedLog(after LogPosition, limit int) ([]LogPosition, uint64, bool, error) { + stable := s.StableSeq() + if limit <= 0 { + return nil, stable, false, nil } + + lower := buildKeyUint64x2(pebblePrefixTxnSeq, after.Seq, after.TxnID) + prefix := []byte(pebblePrefixTxnSeq) + iter, err := s.db.NewIter(&pebble.IterOptions{LowerBound: prefix, UpperBound: prefixUpperBound(prefix)}) if err != nil { - return nil, err + return nil, stable, false, err } - defer closer.Close() + defer iter.Close() - state := &ReplicationStateRecord{} - if err := encoding.Unmarshal(val, state); err != nil { - return nil, err + var entries []LogPosition + more := false + for iter.SeekGE(lower); iter.Valid(); iter.Next() { + key := iter.Key() + if len(key) < len(pebblePrefixTxnSeq)+16 { + continue + } + pos := LogPosition{ + Seq: binary.BigEndian.Uint64(key[len(pebblePrefixTxnSeq):]), + TxnID: binary.BigEndian.Uint64(key[len(pebblePrefixTxnSeq)+8:]), + } + if !after.Less(pos) { + continue + } + if pos.Seq > stable { + break + } + + status, statusErr := s.readTxnStatus(pos.TxnID) + if statusErr != nil && statusErr != pebble.ErrNotFound { + return nil, stable, false, statusErr + } + if statusErr == pebble.ErrNotFound || status != TxnStatusCommitted { + continue + } + + if len(entries) >= limit { + more = true + break + } + entries = append(entries, pos) + } + if err := iter.Error(); err != nil { + return nil, stable, false, err } - return state, nil + return entries, stable, more, nil } -// UpdateReplicationState updates replication state for a peer -func (s *PebbleMetaStore) UpdateReplicationState(peerNodeID uint64, dbName string, lastTxnID uint64, lastTS hlc.Timestamp) error { - state := &ReplicationStateRecord{ - PeerNodeID: peerNodeID, - DatabaseName: dbName, - LastAppliedTxnID: lastTxnID, - LastAppliedTSWall: lastTS.WallTime, - LastAppliedTSLogical: lastTS.Logical, - LastSyncTime: time.Now().UnixNano(), - SyncStatus: SyncStatusSynced, +// GetPullCursor returns C[self,peer,d]: this store's pull position in +// peerNodeID's log for this database. The zero position means nothing has +// been pulled from that peer yet. +func (s *PebbleMetaStore) GetPullCursor(peerNodeID uint64) (LogPosition, error) { + val, closer, err := s.db.Get(pebblePullCursorKey(peerNodeID)) + if err == pebble.ErrNotFound { + return LogPosition{}, nil } - - data, err := encoding.Marshal(state) if err != nil { - return err + return LogPosition{}, err } + defer closer.Close() + return decodeLogPosition(val), nil +} - return s.db.Set(pebbleReplKey(peerNodeID, dbName), data, pebble.NoSync) +// SetPullCursor persists C[self,peer,d]. +func (s *PebbleMetaStore) SetPullCursor(peerNodeID uint64, pos LogPosition) error { + return s.db.Set(pebblePullCursorKey(peerNodeID), encodeLogPosition(pos), pebble.NoSync) } -// GetMinAppliedTxnID returns the minimum applied txn_id across all peers for a database -func (s *PebbleMetaStore) GetMinAppliedTxnID(dbName string) (uint64, error) { - var minTxnID uint64 = ^uint64(0) // Max uint64 - found := false - prefix := []byte(pebblePrefixRepl) +// SetConsumedPosition stores R[self,requesterNodeID,d]: the `after` +// position requesterNodeID last sent when listing this store's log. It is +// stored as is, not as a max: after a restore, a requester's cursor +// can regress, and it must be able to pull entries again that a stale, +// higher recorded position would otherwise make this store's GC treat as +// already consumed. +func (s *PebbleMetaStore) SetConsumedPosition(requesterNodeID uint64, pos LogPosition) error { + return s.db.Set(pebbleConsumedPosKey(requesterNodeID), encodeLogPosition(pos), pebble.NoSync) +} - iter, err := s.db.NewIter(&pebble.IterOptions{ - LowerBound: prefix, - UpperBound: prefixUpperBound(prefix), - }) +// ConsumedPositions returns every requester's last-reported consumed +// position in this store's log. +func (s *PebbleMetaStore) ConsumedPositions() (map[uint64]LogPosition, error) { + result := make(map[uint64]LogPosition) + prefix := []byte(pebblePrefixConsumedPos) + iter, err := s.db.NewIter(&pebble.IterOptions{LowerBound: prefix, UpperBound: prefixUpperBound(prefix)}) if err != nil { - return 0, err + return nil, err } defer iter.Close() - for iter.SeekGE(prefix); iter.Valid(); iter.Next() { - val, err := iter.ValueAndErr() - if err != nil { - continue - } - - var state ReplicationStateRecord - if err := encoding.Unmarshal(val, &state); err != nil { + key := iter.Key() + if len(key) < len(pebblePrefixConsumedPos)+8 { continue } - - if state.DatabaseName == dbName { - found = true - if state.LastAppliedTxnID < minTxnID { - minTxnID = state.LastAppliedTxnID - } + nodeID := binary.BigEndian.Uint64(key[len(pebblePrefixConsumedPos):]) + val, err := iter.ValueAndErr() + if err != nil { + return nil, err } + result[nodeID] = decodeLogPosition(val) } + return result, iter.Error() +} - if err := iter.Error(); err != nil { - return 0, err +// DeleteConsumedPosition removes a departed member's consumed position so +// it no longer pins GC. +func (s *PebbleMetaStore) DeleteConsumedPosition(nodeID uint64) error { + return s.db.Delete(pebbleConsumedPosKey(nodeID), pebble.NoSync) +} + +// TruncatedThrough returns T[self,d]: the highest LogPosition this store's +// GC has deleted through. The zero position means GC has never run. +func (s *PebbleMetaStore) TruncatedThrough() (LogPosition, error) { + val, closer, err := s.db.Get([]byte(pebbleKeyTruncatedThrough)) + if err == pebble.ErrNotFound { + return LogPosition{}, nil } - if !found { - return 0, nil + if err != nil { + return LogPosition{}, err } - return minTxnID, nil + defer closer.Close() + return decodeLogPosition(val), nil } -// GetAllReplicationStates returns all replication state records -func (s *PebbleMetaStore) GetAllReplicationStates() ([]*ReplicationStateRecord, error) { - var states []*ReplicationStateRecord - prefix := []byte(pebblePrefixRepl) +// SetReapplyPending durably sets or clears the re-apply-pending mark. It is +// written with the store's durable write option: the mark must survive the +// crash it exists for. +func (s *PebbleMetaStore) SetReapplyPending(pending bool) error { + if pending { + return s.db.Set([]byte(pebbleKeyReapplyPending), []byte{1}, s.syncWrite) + } + return s.db.Delete([]byte(pebbleKeyReapplyPending), s.syncWrite) +} - iter, err := s.db.NewIter(&pebble.IterOptions{ - LowerBound: prefix, - UpperBound: prefixUpperBound(prefix), - }) +// ReapplyPending reports whether the re-apply-pending mark is set. +func (s *PebbleMetaStore) ReapplyPending() (bool, error) { + _, closer, err := s.db.Get([]byte(pebbleKeyReapplyPending)) + if err == pebble.ErrNotFound { + return false, nil + } if err != nil { - return nil, err + return false, err } - defer iter.Close() - - for iter.SeekGE(prefix); iter.Valid(); iter.Next() { - val, err := iter.ValueAndErr() - if err != nil { - continue - } + closer.Close() + return true, nil +} - state := &ReplicationStateRecord{} - if err := encoding.Unmarshal(val, state); err != nil { - continue - } - states = append(states, state) +// advanceTruncatedThroughInBatch adds a Set for TruncatedThrough to batch +// iff candidate sorts after the currently persisted value, so that T never +// decreases. +func (s *PebbleMetaStore) advanceTruncatedThroughInBatch(batch *pebble.Batch, candidate LogPosition) error { + current, err := s.TruncatedThrough() + if err != nil { + return err } - - return states, iter.Error() + if !current.Less(candidate) { + return nil + } + return batch.Set([]byte(pebbleKeyTruncatedThrough), encodeLogPosition(candidate), nil) } // pebbleSchemaVersionRecord is internal storage for schema @@ -1704,7 +1909,12 @@ type pebbleSchemaVersionRecord struct { UpdatedAt int64 } -// GetSchemaVersion retrieves the schema version for a database +// GetSchemaVersion retrieves the database's schema version as last written by +// the retired pebble-backed counter. Kept only as the one-time migration +// read into __marmot_schema_version: +// NewReplicatedDatabase's __marmot_schema_version table is now the source of +// truth, and nothing writes this key any more (UpdateSchemaVersion is +// removed). See MetaStore.GetSchemaVersion's doc comment. func (s *PebbleMetaStore) GetSchemaVersion(dbName string) (int64, error) { val, closer, err := s.db.Get(pebbleSchemaKey(dbName)) if err == pebble.ErrNotFound { @@ -1722,63 +1932,6 @@ func (s *PebbleMetaStore) GetSchemaVersion(dbName string) (int64, error) { return rec.Version, nil } -// UpdateSchemaVersion updates the schema version for a database. -// -// Unlike the transaction records, the write is synced. The DDL it counts is -// already durable in SQLite, and a version lost to a crash is not healed by -// replication: the node reports a lower version than the schema it runs, and -// every peer's transaction that requires the newer version is refused on it. -// Every schema-version write goes through here - a DDL's increment, catch-up -// and a snapshot restore - and DDL is rare, so the sync stays off the hot path. -func (s *PebbleMetaStore) UpdateSchemaVersion(dbName string, version int64, ddlSQL string, txnID uint64) error { - rec := &pebbleSchemaVersionRecord{ - Version: version, - LastDDL: ddlSQL, - TxnID: txnID, - UpdatedAt: time.Now().UnixNano(), - } - - data, err := encoding.Marshal(rec) - if err != nil { - return err - } - - return s.db.Set(pebbleSchemaKey(dbName), data, s.syncWrite) -} - -// GetAllSchemaVersions returns all schema versions indexed by database name -func (s *PebbleMetaStore) GetAllSchemaVersions() (map[string]int64, error) { - versions := make(map[string]int64) - prefix := []byte(pebblePrefixSchema) - - iter, err := s.db.NewIter(&pebble.IterOptions{ - LowerBound: prefix, - UpperBound: prefixUpperBound(prefix), - }) - if err != nil { - return nil, err - } - defer iter.Close() - - for iter.SeekGE(prefix); iter.Valid(); iter.Next() { - key := iter.Key() - dbName := string(key[len(prefix):]) - - val, err := iter.ValueAndErr() - if err != nil { - continue - } - - var rec pebbleSchemaVersionRecord - if err := encoding.Unmarshal(val, &rec); err != nil { - continue - } - versions[dbName] = rec.Version - } - - return versions, iter.Error() -} - // pebbleDdlLockRecord is internal storage for DDL locks type pebbleDdlLockRecord struct { NodeID uint64 @@ -2094,263 +2247,271 @@ func (s *PebbleMetaStore) cleanupCDCSegmentFiles() (int, error) { return s.cdcLog.pruneUnreferencedSegments(retained) } -// CleanupStaleTransactions aborts transactions that haven't had a heartbeat within the timeout -func (s *PebbleMetaStore) CleanupStaleTransactions(timeout time.Duration) (int, error) { - // Check if store is closed - return early to avoid pebble: closed panic +// StaleTransactionIDs implements MetaStore. Without a memory tier there +// are no heartbeats, so a PENDING transaction is stale once its record's +// CreatedAt is older than timeout. +func (s *PebbleMetaStore) StaleTransactionIDs(timeout time.Duration) ([]uint64, error) { if s.closed.Load() { - return 0, nil + return nil, nil } - - cleaned := 0 - - // Phase 1: Find stale PENDING transactions - var staleTxnIDs []uint64 prefix := []byte(pebblePrefixTxnPending) - - iter, err := s.db.NewIter(&pebble.IterOptions{ - LowerBound: prefix, - UpperBound: prefixUpperBound(prefix), - }) + iter, err := s.db.NewIter(&pebble.IterOptions{LowerBound: prefix, UpperBound: prefixUpperBound(prefix)}) if err != nil { - return 0, err + return nil, err } - + var candidates []uint64 for iter.SeekGE(prefix); iter.Valid(); iter.Next() { key := iter.Key() if len(key) < len(pebblePrefixTxnPending)+8 { continue } - txnID := binary.BigEndian.Uint64(key[len(pebblePrefixTxnPending):]) + candidates = append(candidates, binary.BigEndian.Uint64(key[len(pebblePrefixTxnPending):])) + } + if err := iter.Close(); err != nil { + return nil, err + } - // Heartbeats are in-memory only - for PebbleMetaStore direct use, - // use CreatedAt to determine staleness (MemoryMetaStore uses in-memory heartbeats) - rec, err := s.GetTransaction(txnID) - if err != nil || rec == nil { - continue + cutoff := time.Now().Add(-timeout).UnixNano() + stale := candidates[:0] + for _, txnID := range candidates { + ok, err := s.isStale(txnID, cutoff) + if err != nil { + return nil, err } - - nowNs := time.Now().UnixNano() - ageNs := nowNs - rec.CreatedAt - if ageNs > timeout.Nanoseconds() { - staleTxnIDs = append(staleTxnIDs, txnID) - log.Warn(). - Uint64("txn_id", txnID). - Int64("age_ms", ageNs/1e6). - Int64("timeout_ms", timeout.Milliseconds()). - Msg("GC: Found stale PENDING transaction (no active heartbeat)") + if ok { + stale = append(stale, txnID) } } - if err := iter.Close(); err != nil { - return 0, err + return stale, nil +} + +// AbortStaleTransaction implements MetaStore. +func (s *PebbleMetaStore) AbortStaleTransaction(txnID uint64, timeout time.Duration) (bool, error) { + ok, err := s.isStale(txnID, time.Now().Add(-timeout).UnixNano()) + if err != nil || !ok { + return false, err + } + log.Warn().Uint64("txn_id", txnID).Msg("GC: Aborting stale PENDING transaction (no active heartbeat)") + if err := s.AbortTransaction(txnID); err != nil { + return false, fmt.Errorf("abort stale txn %d: %w", txnID, err) + } + if err := s.DeleteIntentsByTxn(txnID); err != nil { + return false, fmt.Errorf("delete intents of stale txn %d: %w", txnID, err) + } + if err := s.DeleteIntentEntries(txnID); err != nil { + return false, fmt.Errorf("delete captured rows of stale txn %d: %w", txnID, err) } + return true, nil +} - // Delete stale transactions and their intents (best-effort cleanup) - for _, txnID := range staleTxnIDs { - log.Warn(). - Uint64("txn_id", txnID). - Msg("GC: Aborting and deleting stale transaction") - _ = s.AbortTransaction(txnID) - _ = s.DeleteIntentsByTxn(txnID) - _ = s.DeleteIntentEntries(txnID) - cleaned++ +// isStale reports whether txnID is PENDING with a record created before +// cutoff. +func (s *PebbleMetaStore) isStale(txnID uint64, cutoff int64) (bool, error) { + rec, err := s.GetTransaction(txnID) + if err != nil || rec == nil { + return false, err } + return rec.Status == TxnStatusPending && rec.CreatedAt < cutoff, nil +} - // Phase 2: No orphaned intent cleanup needed - RowLockStore is ephemeral - // ReleaseByTxn in Phase 1 already cleaned up in-memory locks +// gcEntryVerdict is what one CleanupOldTransactionRecords pass does with one +// seq-index entry. +type gcEntryVerdict uint8 - if cleaned > 0 { - log.Info(). - Int("stale_txns", len(staleTxnIDs)). - Msg("MetaStore GC: Cleaned up stale transactions") - } +const ( + gcDelete gcEntryVerdict = iota // delete it and keep walking + gcSkip // keep it and keep walking + gcStop // keep it and end the pass +) - return cleaned, nil +// gcVerdict decides one seq-index entry's fate. A COMMITTED entry is +// deleted once past max retention, or past min retention at or below safe; +// otherwise it ends the pass, which keeps the deleted set a prefix of the +// committed log. An entry whose status is not COMMITTED (a record a late +// PREPARE overwrote before BeginTransactionWithID refused that) is not a log +// entry ListCommittedLog ever serves, so it can neither end the pass forever +// nor pin TruncatedThrough: it is skipped until its own begin is past max +// retention, and then deleted. +func (s *PebbleMetaStore) gcVerdict(pos LogPosition, minCutoff, maxCutoff int64, safe LogPosition) (gcEntryVerdict, error) { + status, err := s.readTxnStatus(pos.TxnID) + if err != nil && err != pebble.ErrNotFound { + return gcStop, err + } + if err == nil && status == TxnStatusCommitted { + commit, err := s.readCommitRecord(pos.TxnID) + if err != nil { + return gcStop, err + } + if commit != nil { + if commit.CommittedAt < maxCutoff || (pos.Compare(safe) <= 0 && commit.CommittedAt < minCutoff) { + return gcDelete, nil + } + return gcStop, nil + } + } + immutable, err := s.readImmutableTxnRecord(pos.TxnID) + if err != nil { + return gcStop, err + } + if immutable != nil && immutable.CreatedAt >= maxCutoff { + return gcSkip, nil + } + return gcDelete, nil } -// CleanupOldTransactionRecords removes old COMMITTED/ABORTED transaction records -func (s *PebbleMetaStore) CleanupOldTransactionRecords(minRetention, maxRetention time.Duration, minAppliedTxnID, minAppliedSeqNum uint64) (int, error) { +// gcMaxEntriesPerPass bounds how many log entries one +// CleanupOldTransactionRecords pass deletes, so one batch stays bounded; the +// next pass continues from the new truncation point. +const gcMaxEntriesPerPass = 10000 + +// CleanupOldTransactionRecords deletes a prefix of the committed log. +// +// It reads each entry's actual status key and commit record: decoding a +// /txn/ value (TxnImmutableRecord, which has no Status field) into +// TransactionRecord would read every status as the zero value +// TxnStatusPending, and nothing would ever be deleted. +// +// It walks the seq index in position order (the log's own order) and stops +// at the first committed entry that is not deletable, so that "every +// committed entry below TruncatedThrough is deleted" holds without creating +// holes: a later pass cannot delete an entry whose position is higher than +// one it left behind this time, because doing so would let TruncatedThrough +// advance past a gap. In practice this costs nothing, because max-retention +// entries are the oldest and so are already a prefix; a younger committed +// entry (newer than max retention and above safe) simply ends this pass's +// walk, and is revisited on the next one. A seq-index entry whose status is +// not COMMITTED is not a log entry at all and never ends the walk (gcVerdict). +// +// ABORTED transactions have no seq-index entry (AbortTransaction never adds +// one, and refuses to touch one that exists for a COMMITTED txn), so they +// are never visited by this walk. Their /txn_status/ key is left in place: +// a known limitation, nothing garbage-collects it. +func (s *PebbleMetaStore) CleanupOldTransactionRecords(minRetention, maxRetention time.Duration, safe LogPosition) (int, error) { // Check if store is closed - return early to avoid pebble: closed panic if s.closed.Load() { return 0, nil } now := time.Now() - minRetentionCutoff := now.Add(-minRetention).UnixNano() - maxRetentionCutoff := now.Add(-maxRetention).UnixNano() - deleted := 0 - committedDeleted := 0 + minCutoff := now.Add(-minRetention).UnixNano() + maxCutoff := now.Add(-maxRetention).UnixNano() - prefix := []byte(pebblePrefixTxn) - var keysToDelete [][]byte - var seqKeysToDelete [][]byte - var txnIDsToClean []uint64 + type deletable struct { + pos LogPosition + } + var toDelete []deletable + var highest LogPosition - iter, err := s.db.NewIter(&pebble.IterOptions{ - LowerBound: prefix, - UpperBound: prefixUpperBound(prefix), - }) + prefix := []byte(pebblePrefixTxnSeq) + iter, err := s.db.NewIter(&pebble.IterOptions{LowerBound: prefix, UpperBound: prefixUpperBound(prefix)}) if err != nil { return 0, err } for iter.SeekGE(prefix); iter.Valid(); iter.Next() { - val, err := iter.ValueAndErr() - if err != nil { + key := iter.Key() + if len(key) < len(pebblePrefixTxnSeq)+16 { continue } - - var rec TransactionRecord - if err := encoding.Unmarshal(val, &rec); err != nil { - continue + pos := LogPosition{ + Seq: binary.BigEndian.Uint64(key[len(pebblePrefixTxnSeq):]), + TxnID: binary.BigEndian.Uint64(key[len(pebblePrefixTxnSeq)+8:]), } - // Only clean COMMITTED or ABORTED - if rec.Status != TxnStatusCommitted && rec.Status != TxnStatusAborted { + verdict, err := s.gcVerdict(pos, minCutoff, maxCutoff, safe) + if err != nil { + _ = iter.Close() + return 0, err + } + if verdict == gcSkip { continue } - - // Check deletion criteria - shouldDelete := false - if rec.CreatedAt < maxRetentionCutoff { - shouldDelete = true - } else if rec.CreatedAt < minRetentionCutoff { - if minAppliedTxnID > 0 && minAppliedSeqNum > 0 { - if rec.TxnID < minAppliedTxnID && (rec.SeqNum == 0 || rec.SeqNum < minAppliedSeqNum) { - shouldDelete = true - } - } else if minAppliedTxnID > 0 { - if rec.TxnID < minAppliedTxnID { - shouldDelete = true - } - } else if minAppliedSeqNum > 0 { - if rec.SeqNum == 0 || rec.SeqNum < minAppliedSeqNum { - shouldDelete = true - } - } + if verdict == gcStop || len(toDelete) >= gcMaxEntriesPerPass { + break } - if shouldDelete { - key := make([]byte, len(iter.Key())) - copy(key, iter.Key()) - keysToDelete = append(keysToDelete, key) - if rec.SeqNum > 0 { - seqKeysToDelete = append(seqKeysToDelete, pebbleTxnSeqKey(rec.SeqNum, rec.TxnID)) - } - if rec.Status == TxnStatusCommitted { - committedDeleted++ - txnIDsToClean = append(txnIDsToClean, rec.TxnID) - } + toDelete = append(toDelete, deletable{pos: pos}) + if highest.Less(pos) { + highest = pos } } if err := iter.Close(); err != nil { return 0, err } - if len(keysToDelete) == 0 { + if len(toDelete) == 0 { return 0, nil } batch := s.db.NewBatch() defer batch.Close() - for _, key := range keysToDelete { - if err := batch.Delete(key, nil); err == nil { - deleted++ + for _, d := range toDelete { + if err := batch.Delete(pebbleTxnKey(d.pos.TxnID), nil); err != nil { + return 0, err + } + if err := batch.Delete(pebbleTxnCommitKey(d.pos.TxnID), nil); err != nil { + return 0, err + } + if err := batch.Delete(pebbleTxnStatusKey(d.pos.TxnID), nil); err != nil { + return 0, err + } + if err := batch.Delete(pebbleTxnByIDKey(d.pos.TxnID), nil); err != nil { + return 0, err + } + if err := batch.Delete(pebbleTxnSeqKey(d.pos.Seq, d.pos.TxnID), nil); err != nil { + return 0, err + } + // Only a stray entry (see gcVerdict) can still have one. + if err := batch.Delete(pebbleTxnPendingKey(d.pos.TxnID), nil); err != nil { + return 0, err } } - for _, key := range seqKeysToDelete { - _ = batch.Delete(key, nil) + // Every entry here, a stray one included, was counted when it committed. + if err := s.counters.DecInBatch(batch, "committed_txn_count", int64(len(toDelete))); err != nil { + return 0, err + } + if err := s.advanceTruncatedThroughInBatch(batch, highest); err != nil { + return 0, err } if err := batch.Commit(pebble.NoSync); err != nil { return 0, err } - // Clean up CDC raw captured rows for deleted committed transactions - for _, txnID := range txnIDsToClean { - if err := s.DeleteCapturedRows(txnID); err != nil { - log.Debug().Err(err).Uint64("txn_id", txnID).Msg("MetaStore GC: Failed to delete CDC raw rows") + // Clean up CDC raw captured rows for deleted transactions (best-effort; + // the metadata keys above are the source of truth for "deleted"). + for _, d := range toDelete { + if err := s.DeleteCapturedRows(d.pos.TxnID); err != nil { + log.Debug().Err(err).Uint64("txn_id", d.pos.TxnID).Msg("MetaStore GC: Failed to delete CDC raw rows") } } if _, err := s.cleanupCDCSegmentFiles(); err != nil { log.Debug().Err(err).Msg("MetaStore GC: Failed to delete unreferenced CDC segment files") } - // Decrement committed transaction counter - if committedDeleted > 0 { - if _, err := s.counters.Dec("committed_txn_count", int64(committedDeleted)); err != nil { - log.Warn().Err(err).Int("count", committedDeleted).Msg("MetaStore GC: Failed to decrement counter") - } - } - - if deleted > 0 { - log.Info().Int("deleted_records", deleted).Int("committed_deleted", committedDeleted).Msg("MetaStore GC: Cleaned up old transaction records") - } - - return deleted, nil + log.Info().Int("deleted_records", len(toDelete)).Msg("MetaStore GC: Cleaned up old transaction records") + return len(toDelete), nil } -// GetNextSeqNum returns the next sequence number for a node -func (s *PebbleMetaStore) GetNextSeqNum(nodeID uint64) (uint64, error) { - seq, err := s.getOrCreateSequence(nodeID) - if err != nil { - return 0, err - } - num, err := seq.Next() - if err != nil { - return 0, err - } - // We want 1-based sequence numbers - return num + 1, nil +// nextLogSeq allocates the next value of the store-wide local commit +// sequence (LogPosition.Seq), shared by every origin's committed +// transactions in this store. Callers must call s.logSeq.markDone(seq) via +// defer on every exit path once allocated (see logSeqTracker). +func (s *PebbleMetaStore) nextLogSeq() (uint64, error) { + return s.logSeq.allocate() } -// getOrCreateSequence returns or creates an AtomicSequence for the given nodeID -func (s *PebbleMetaStore) getOrCreateSequence(nodeID uint64) (*AtomicSequence, error) { - s.seqMu.Lock() - defer s.seqMu.Unlock() - - if seq, ok := s.sequences[nodeID]; ok { - return seq, nil - } - - seq, err := NewAtomicSequence(s.db, pebbleSeqKey(nodeID), pebbleSeqBandwidth) - if err != nil { - return nil, fmt.Errorf("failed to create sequence for node %d: %w", nodeID, err) - } - - s.sequences[nodeID] = seq - return seq, nil +// StableSeq returns the highest seq s such that every seq <= s this +// process has allocated from the store-wide log sequence has finished +// (logSeqTracker's doc comment has the full proof). +func (s *PebbleMetaStore) StableSeq() uint64 { + return s.logSeq.StableSeq() } // GetMaxSeqNum returns the maximum sequence number across all committed transactions func (s *PebbleMetaStore) GetMaxSeqNum() (uint64, error) { - var maxSeq uint64 - prefix := []byte(pebblePrefixTxnSeq) - - iter, err := s.db.NewIter(&pebble.IterOptions{ - LowerBound: prefix, - UpperBound: prefixUpperBound(prefix), - }) - if err != nil { - return 0, err - } - defer iter.Close() - - // Go to last key in range (key format: prefix + 8 bytes seqNum + 8 bytes txnID) - if iter.Last() { - key := iter.Key() - if len(key) >= len(pebblePrefixTxnSeq)+8 { - maxSeq = binary.BigEndian.Uint64(key[len(pebblePrefixTxnSeq):]) - } - } - - return maxSeq, iter.Error() -} - -// GetMinAppliedSeqNum returns the minimum applied sequence number across all peers for a database -func (s *PebbleMetaStore) GetMinAppliedSeqNum(dbName string) (uint64, error) { - // Proxy through GetMinAppliedTxnID since we track by txn_id - return s.GetMinAppliedTxnID(dbName) + return maxSeqNumFromIndex(s.db) } // GetMaxCommittedTxnID returns the maximum committed transaction ID diff --git a/db/meta_store_pebble_test.go b/db/meta_store_pebble_test.go index 7ac20e52..0a101e60 100644 --- a/db/meta_store_pebble_test.go +++ b/db/meta_store_pebble_test.go @@ -8,8 +8,10 @@ import ( "encoding/binary" "os" "path/filepath" + "strings" "testing" "time" + "unicode/utf8" "github.com/cockroachdb/pebble" "github.com/maxpert/marmot/hlc" @@ -114,6 +116,52 @@ func TestPebbleMetaStoreTransactionAbort(t *testing.T) { } } +// TestPebbleMetaStoreAbortRefusesCommitted: the local log is +// append-only, so AbortTransaction must refuse a COMMITTED transaction and +// must not touch its seq-index entry or records. +func TestPebbleMetaStoreAbortRefusesCommitted(t *testing.T) { + store, cleanup := createTestPebbleMetaStore(t) + defer cleanup() + + clock := hlc.NewClock(1) + startTS := clock.Now() + txnID := startTS.ToTxnID() + + if err := store.BeginTransaction(txnID, 1, startTS); err != nil { + t.Fatalf("BeginTransaction failed: %v", err) + } + commitTS := clock.Now() + if err := store.CommitTransaction(txnID, commitTS, nil, "testdb", "", 0, 0); err != nil { + t.Fatalf("CommitTransaction failed: %v", err) + } + + if err := store.AbortTransaction(txnID); err != ErrAbortCommitted { + t.Fatalf("AbortTransaction on a committed txn = %v, want ErrAbortCommitted", err) + } + + rec, err := store.GetTransaction(txnID) + if err != nil { + t.Fatalf("GetTransaction failed: %v", err) + } + if rec == nil || rec.Status != TxnStatusCommitted { + t.Fatalf("committed transaction record must survive a refused abort, got %+v", rec) + } + + entries, _, _, err := store.ListCommittedLog(LogPosition{}, 10) + if err != nil { + t.Fatalf("ListCommittedLog failed: %v", err) + } + found := false + for _, e := range entries { + if e.TxnID == txnID { + found = true + } + } + if !found { + t.Fatal("committed txn's seq-index entry must survive a refused abort") + } +} + func TestPebbleMetaStoreWriteIntents(t *testing.T) { store, cleanup := createTestPebbleMetaStore(t) defer cleanup() @@ -204,40 +252,32 @@ func TestPebbleMetaStoreWriteIntentConflict(t *testing.T) { } } -func TestPebbleMetaStoreReplicationState(t *testing.T) { +// TestPebbleMetaStoreWriteIntentConflictBinaryKeyIsValidUTF8 pins that a +// conflict on a row whose intent key is not valid UTF-8 (an encoded binary +// primary key) yields an error message that is: the message is sent as a +// protobuf string field in the PREPARE response, and an invalid one fails +// the whole gRPC response's encoding. +func TestPebbleMetaStoreWriteIntentConflictBinaryKeyIsValidUTF8(t *testing.T) { store, cleanup := createTestPebbleMetaStore(t) defer cleanup() - peerNodeID := uint64(2) - dbName := "testdb" - clock := hlc.NewClock(1) - ts := clock.Now() - - // Initially no state - state, err := store.GetReplicationState(peerNodeID, dbName) - if err != nil { - t.Fatalf("GetReplicationState failed: %v", err) + ts := hlc.NewClock(1).Now() + key := string([]byte{0xff, 0xfe, 0x00, 0x81}) + if err := store.WriteIntent(1, IntentTypeDML, "blobs", key, OpTypeInsert, "", nil, ts, 1); err != nil { + t.Fatalf("first WriteIntent: %v", err) } - if state != nil { - t.Error("Expected no initial state") + if err := store.BeginTransaction(1, 1, ts); err != nil { + t.Fatalf("BeginTransaction: %v", err) } - - // Update state - err = store.UpdateReplicationState(peerNodeID, dbName, 100, ts) - if err != nil { - t.Fatalf("UpdateReplicationState failed: %v", err) - } - - // Verify state - state, err = store.GetReplicationState(peerNodeID, dbName) - if err != nil { - t.Fatalf("GetReplicationState failed: %v", err) + err := store.WriteIntent(2, IntentTypeDML, "blobs", key, OpTypeUpdate, "", nil, ts, 1) + if !isWriteWriteConflict(err) { + t.Fatalf("expected a write-write conflict, got %v", err) } - if state == nil { - t.Fatal("State not found") + if !utf8.ValidString(err.Error()) { + t.Fatalf("conflict message is not valid UTF-8: %q", err.Error()) } - if state.LastAppliedTxnID != 100 { - t.Errorf("Expected txn_id 100, got %d", state.LastAppliedTxnID) + if !strings.Contains(err.Error(), "fffe0081") { + t.Fatalf("conflict message must carry the key hex-encoded: %q", err.Error()) } } @@ -408,23 +448,23 @@ func TestPebbleMetaStoreSequenceNumbers(t *testing.T) { store, cleanup := createTestPebbleMetaStore(t) defer cleanup() - nodeID := uint64(1) - - // Get sequence numbers - should be monotonically increasing - seq1, err := store.GetNextSeqNum(nodeID) + // The store-wide log sequence (formerly per-node GetNextSeqNum) should + // be monotonically increasing regardless of origin. + seq1, err := store.nextLogSeq() if err != nil { - t.Fatalf("GetNextSeqNum failed: %v", err) + t.Fatal(err) } - - seq2, err := store.GetNextSeqNum(nodeID) + defer store.logSeq.markDone(seq1) + seq2, err := store.nextLogSeq() if err != nil { - t.Fatalf("GetNextSeqNum failed: %v", err) + t.Fatal(err) } - - seq3, err := store.GetNextSeqNum(nodeID) + defer store.logSeq.markDone(seq2) + seq3, err := store.nextLogSeq() if err != nil { - t.Fatalf("GetNextSeqNum failed: %v", err) + t.Fatal(err) } + defer store.logSeq.markDone(seq3) if seq2 != seq1+1 { t.Errorf("Expected seq2 (%d) = seq1 (%d) + 1", seq2, seq1) @@ -464,34 +504,21 @@ func TestPebbleMetaStoreCommitCounters(t *testing.T) { } } +// TestPebbleMetaStoreSchemaVersion covers what remains of GetSchemaVersion: +// the schema-version migration read. UpdateSchemaVersion is removed - versions now live +// in each user database's own __marmot_schema_version SQLite table - so there +// is nothing left to write here; an unset key just reads back 0. func TestPebbleMetaStoreSchemaVersion(t *testing.T) { store, cleanup := createTestPebbleMetaStore(t) defer cleanup() - dbName := "testdb" - - // Initial version - version, err := store.GetSchemaVersion(dbName) + version, err := store.GetSchemaVersion("testdb") if err != nil { t.Fatalf("GetSchemaVersion failed: %v", err) } if version != 0 { t.Errorf("Expected initial version 0, got %d", version) } - - // Update version - err = store.UpdateSchemaVersion(dbName, 1, "CREATE TABLE users (id INT)", 100) - if err != nil { - t.Fatalf("UpdateSchemaVersion failed: %v", err) - } - - version, err = store.GetSchemaVersion(dbName) - if err != nil { - t.Fatalf("GetSchemaVersion failed: %v", err) - } - if version != 1 { - t.Errorf("Expected version 1, got %d", version) - } } func TestPebbleMetaStoreDDLLock(t *testing.T) { @@ -582,7 +609,7 @@ func TestPebbleMetaStoreStoreReplayedTransaction(t *testing.T) { nodeID := uint64(2) // Store a replayed transaction (no prior BeginTransaction) - err := store.StoreReplayedTransaction(txnID, nodeID, commitTS, "testdb", 1) + err := store.StoreReplayedTransaction(txnID, nodeID, commitTS, "testdb", 1, 5) if err != nil { t.Fatalf("StoreReplayedTransaction failed: %v", err) } @@ -599,7 +626,10 @@ func TestPebbleMetaStoreStoreReplayedTransaction(t *testing.T) { t.Errorf("Expected COMMITTED, got %s", rec.Status.String()) } if rec.NodeID != nodeID { - t.Errorf("Expected nodeID %d, got %d", nodeID, rec.NodeID) + t.Errorf("Expected origin nodeID %d, got %d", nodeID, rec.NodeID) + } + if rec.RequiredSchemaVersion != 5 { + t.Errorf("Expected RequiredSchemaVersion 5, got %d", rec.RequiredSchemaVersion) } } diff --git a/db/prepared_intent_test.go b/db/prepared_intent_test.go index 3c194a08..16e71205 100644 --- a/db/prepared_intent_test.go +++ b/db/prepared_intent_test.go @@ -248,6 +248,30 @@ func TestApplyClaimsACKsAtMostOneOfTwoOverlappingClaims(t *testing.T) { require.Equal(t, uint64(520), readClaimBase(t, dm, "testdb", "users")) } +// TestCommitOfAClaimAppliedBeforeAFailedCommitIsACKed: a COMMIT applied the +// claim in the system database and then failed before the user-database +// commit (its writer busy, or a crash), leaving the transaction PENDING. The +// transaction was decided COMMITTED, so committing it again - a retried +// COMMIT, or the log puller committing it because a peer's log holds it - +// must succeed rather than be refused for the floor its own claim raised. +// +// Mutation: drop claimAlreadyApplied from applyClaimTx. The second commit is +// refused and "a claim this node already applied was refused" fires. +func TestCommitOfAClaimAppliedBeforeAFailedCommitIsACKed(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 500, 1) + + const a = 9560 + require.True(t, prepareClaim(engine, t, a, 1, 500, 10).Success) + require.NoError(t, autoIncClaimStoreForTest(dm).ApplyClaims("testdb", a, mdb.GetMetaStore())) + + res := commitClaim(engine, a) + require.True(t, res.Success, "a claim this node already applied was refused: %s", res.Error) + require.Equal(t, uint64(510), readClaimBase(t, dm, "testdb", "users")) +} + // TestApplyClaimsIsConditionedOnTheRangeItWrites: the COMMIT's conditional // write guards the range it hands out, newBase+1..newBase+size, so it must not // depend on PREPARE having admitted only newBase == prevBase. Here the intent @@ -287,7 +311,7 @@ func TestApplyClaimsIsConditionedOnTheRangeItWrites(t *testing.T) { // heartbeat_timeout + gc_interval after its PREPARE. The pass is driven // directly here so the test does not race the GC goroutine. // -// Mutation: make MemoryMetaStore.CleanupStaleTransactions never treat a +// Mutation: make MemoryMetaStore.isStale never treat a // pending transaction as stale. "the GC did not abort a prepared transaction // past heartbeat_timeout" fires. (Dropping only its DeleteIntentsByTxn does // not strand the row: once the record is aborted, the next writer's conflict @@ -344,7 +368,7 @@ func TestDeadCoordinatorsPreparedRowIsFreedByTheStaleTransactionGC(t *testing.T) // racing the stale-transaction GC on one node. The COMMIT has loaded the // transaction while it was still pending (engine.Commit -> // TransactionManager.GetTransaction); the GC then deletes the transaction's -// intents and captured rows (MemoryMetaStore.CleanupStaleTransactions, in its +// intents and captured rows (MemoryMetaStore.AbortStaleTransaction, in its // own order), and the COMMIT runs before the GC's abort reaches the record. // The COMMIT used to find no rows, take the statement branch, write a commit // record and ACK a write it never applied. diff --git a/db/replay_apply.go b/db/replay_apply.go new file mode 100644 index 00000000..8a3a5696 --- /dev/null +++ b/db/replay_apply.go @@ -0,0 +1,598 @@ +package db + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/maxpert/marmot/common" + "github.com/maxpert/marmot/hlc" + "github.com/rs/zerolog/log" +) + +// reapplyLocalLogPageSize bounds how many local log entries ReapplyLocalLog +// lists from MetaStore per page. +const reapplyLocalLogPageSize = 256 + +// ReplayTxn is one committed transaction to apply outside 2PC: pulled from a +// peer's local log by anti-entropy, or read back from this node's own local +// log during a restore's re-apply walk (ReapplyLocalLog). Rows are exactly +// as captured - EncodedCapturedRow's Op distinguishes DML from DDL, LOAD DATA +// and vector-index control (db/meta_schema.go's OpType). +type ReplayTxn struct { + TxnID uint64 + OriginNodeID uint64 // the transaction's coordinator; owner of the DDL incarnations it creates, the same on every node that replays it + CommitTS hlc.Timestamp + RequiredSchemaVersion uint64 + Rows []*EncodedCapturedRow +} + +// ErrReplayPending is returned by ApplyReplayedTxn when this node itself +// holds t.TxnID PENDING (a prepared 2PC transaction). Nothing is applied; +// the caller retries once the local transaction resolves. +var ErrReplayPending = errors.New("replay: local transaction is PENDING") + +// ErrReplayIntentConflict is returned by ApplyReplayedTxn when a row t +// touches is locally held by a different transaction's intent or CDC row +// lock. Nothing is applied; the caller retries next round. Every +// actual return uses *ReplayIntentConflictError, which also satisfies +// errors.Is(err, ErrReplayIntentConflict); match on the concrete type to read +// HolderTxnID. +var ErrReplayIntentConflict = errors.New("replay: row held by a different local transaction") + +// ReplayIntentConflictError is ErrReplayIntentConflict's concrete form: it +// additionally names the local transaction (HolderTxnID) whose intent or CDC +// row lock blocked the replay - either the intent holder or the CDC row-lock +// holder - so a caller such as the log puller can act on that specific +// transaction (for example, wait for it to resolve) instead of blindly +// retrying the whole pull round. +type ReplayIntentConflictError struct { + HolderTxnID uint64 +} + +// Error implements error. +func (e *ReplayIntentConflictError) Error() string { + return fmt.Sprintf("replay: row held by local transaction %d", e.HolderTxnID) +} + +// Is reports whether target is ErrReplayIntentConflict, so +// errors.Is(err, ErrReplayIntentConflict) matches every +// *ReplayIntentConflictError regardless of its HolderTxnID. +func (e *ReplayIntentConflictError) Is(target error) bool { + return target == ErrReplayIntentConflict +} + +// isDMLOp reports whether op is a row-data change (as opposed to DDL, LOAD +// DATA or vector-index control). +func isDMLOp(op OpType) bool { + switch op { + case OpTypeInsert, OpTypeReplace, OpTypeUpdate, OpTypeDelete: + return true + default: + return false + } +} + +// markerExists reports whether __marmot_applied_txn already holds txnID, read +// through q (either the database's read pool for a preliminary check, or the +// replay's own tx for the authoritative one - see ApplyReplayedTxn). +func markerExists(q rowQuerier, txnID uint64) (bool, error) { + var one int + err := q.QueryRow("SELECT 1 FROM __marmot_applied_txn WHERE txn_id = ?", txnID).Scan(&one) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, fmt.Errorf("check applied marker: %w", err) + } + return true, nil +} + +// LogReplayedTxn stores t's rows in mdb's local log, sealed, followed by +// StoreReplayedTransaction, unless t.TxnID is already COMMITTED there. This +// is what closes the crash window between logging a pulled txn and applying +// it: logging before the SQLite tx below means a crash between the two leaves +// the txn correctly recorded as committed-elsewhere in this node's own log, +// re-applied on the next pass because no marker exists yet. It is idempotent +// - a second call for an already-logged txn does nothing - so ApplyReplayedTxn +// can call it on every attempt. Returns the log entry's SeqNum. +// +// Held across its whole body is the txn id's stripe of +// TransactionManager.txnIDLocks, the same stripe +// BeginTransactionWithID holds around tm.metaStore.BeginTransaction. That +// makes the two atomic with each other: either this call logs t completely +// before a late local PREPARE's begin for the same id (which then finds a +// COMMITTED record and is refused - see BeginTransactionWithID's caller), or +// the begin runs first and is visible here - in which case the existing +// record is PENDING (never ABORTED: an aborted txn's immutable record is +// deleted, so GetTransaction would report it absent, not a status), and this +// call refuses with ErrReplayPending, writing nothing, exactly like +// ApplyReplayedTxn's own precheck. +func (mdb *ReplicatedDatabase) LogReplayedTxn(t *ReplayTxn) (uint64, error) { + unlock := mdb.txnMgr.lockTxnID(t.TxnID) + defer unlock() + + rec, err := mdb.metaStore.GetTransaction(t.TxnID) + if err != nil { + return 0, fmt.Errorf("check existing log entry: %w", err) + } + if rec != nil { + if rec.Status == TxnStatusCommitted { + return rec.SeqNum, nil + } + return 0, ErrReplayPending + } + + for i, row := range t.Rows { + data, err := EncodeRow(row) + if err != nil { + return 0, fmt.Errorf("encode captured row: %w", err) + } + if err := mdb.metaStore.WriteCapturedRow(t.TxnID, uint64(i+1), data); err != nil { + return 0, fmt.Errorf("write captured row: %w", err) + } + } + if err := mdb.metaStore.SealCapturedRows(t.TxnID); err != nil { + return 0, fmt.Errorf("seal captured rows: %w", err) + } + + origin := t.OriginNodeID + if origin == 0 { + origin = t.CommitTS.NodeID + } + if err := mdb.metaStore.StoreReplayedTransaction(t.TxnID, origin, t.CommitTS, mdb.dbName, uint32(len(t.Rows)), t.RequiredSchemaVersion); err != nil { + return 0, fmt.Errorf("store replayed transaction: %w", err) + } + + rec, err = mdb.metaStore.GetTransaction(t.TxnID) + if err != nil { + return 0, err + } + if rec == nil { + return 0, fmt.Errorf("replayed transaction %d missing immediately after logging it", t.TxnID) + } + return rec.SeqNum, nil +} + +// DiscardAbandonedBegin deletes this node's PendingBegunAbandoned record for +// txnID (MetaStore.DiscardAbandonedBegin), holding the txn id's stripe of +// TransactionManager.txnIDLocks so a PREPARE beginning the same id cannot +// interleave with the discard. +func (mdb *ReplicatedDatabase) DiscardAbandonedBegin(txnID uint64) error { + unlock := mdb.txnMgr.lockTxnID(txnID) + defer unlock() + return mdb.metaStore.DiscardAbandonedBegin(txnID) +} + +// ApplyReplayedTxn applies a committed transaction pulled from a peer's log +// (or, with logFirst=false, read back from this node's own local log by +// ReapplyLocalLog), exactly-once and atomically with the applied-txn +// marker. +// +// Order: +// 1. If this node itself holds t.TxnID PENDING, return ErrReplayPending and +// touch nothing: the local transaction must resolve first. +// 2. For every DML row, refuse (ErrReplayIntentConflict, touching nothing) +// if the row's key holds a local intent or CDC row lock owned by a +// different transaction. This stops replay from writing over a newer +// image a locally in-flight (or about-to-be-applied) transaction holds. +// 3. If logFirst, log t first (LogReplayedTxn) - see its doc comment for the +// crash-window this closes. +// 4. Vector-control rows: gated on the marker (checked read-only, before the +// SQLite tx, since the control is not SQLite and cannot share it) so a +// replay that already committed never re-runs the control. See +// TransactionManager.applyNonDMLIntents's doc comment for why re-running +// it once, on a crash between this step and the marker tx below, is safe. +// 5. One SQLite tx: re-check the marker inside the tx (authoritative - the +// read-only check above can race a concurrent apply); if present, roll +// back and return applied=false. Otherwise apply every remaining row (DML +// via ApplyCDCValues, DDL via ApplyReplayedDDL with owner=t.OriginNodeID, +// LOAD DATA via ApplyLoadDataInTx), finish any schema change, bump the +// schema version iff the txn carries DDL, write the marker, commit. +// 6. After commit: reload the schema cache and refresh the read pool if DDL +// changed it (warn-only, matching handleReplay's existing contract), +// update the cached schema version, then apply committed vector CDC for +// the DML rows (idempotent by txn/seq; errors are logged, not returned - +// the row data itself already committed). +// +// A claim-only txn (zero rows - an AUTO_INCREMENT range claim, which never +// travels in CDC) reaches step 5 with nothing to apply but the marker itself: +// see applyNonDMLIntents's doc comment for why that is safe though not +// exactly-once. +func (mdb *ReplicatedDatabase) ApplyReplayedTxn(ctx context.Context, t *ReplayTxn, logFirst bool) (applied bool, err error) { + rec, err := mdb.replayPrecheck(t) + if err != nil { + return false, err + } + + var seqNum uint64 + if logFirst { + seqNum, err = mdb.LogReplayedTxn(t) + if err != nil { + return false, fmt.Errorf("log replayed transaction: %w", err) + } + } else if rec != nil { + seqNum = rec.SeqNum + } + + if err := mdb.applyReplayVectorControl(ctx, t); err != nil { + return false, err + } + + result, applied, err := mdb.commitReplayedTx(ctx, t, seqNum) + if err != nil || !applied { + return false, err + } + + mdb.afterReplayCommit(ctx, t, seqNum, result.change, result.hasDDL, result.newSchemaVersion, result.dmlEntries) + return true, nil +} + +// replayTxCommitResult bundles what commitReplayedTx's SQLite tx produced, +// for its caller's post-commit refresh (afterReplayCommit). +type replayTxCommitResult struct { + change *SchemaChange + dmlEntries []common.CDCEntry + hasDDL bool + newSchemaVersion uint64 +} + +// commitReplayedTx is ApplyReplayedTxn's step 5: one SQLite tx that +// re-checks the marker (authoritative - the read-only check in +// applyReplayVectorControl can race a concurrent apply), applies every +// remaining row, finishes any schema change, bumps the schema version iff +// the txn carries DDL, writes the marker, and commits. applied is false, +// with no error, when the marker already existed - nothing else is touched. +func (mdb *ReplicatedDatabase) commitReplayedTx(ctx context.Context, t *ReplayTxn, seqNum uint64) (*replayTxCommitResult, bool, error) { + tx, err := mdb.writeDB.BeginTx(ctx, nil) + if err != nil { + return nil, false, fmt.Errorf("begin replay transaction: %w", err) + } + defer tx.Rollback() + + if already, err := markerExists(tx, t.TxnID); err != nil { + return nil, false, err + } else if already { + return nil, false, nil + } + + schemaAdapter := &schemaCacheAdapter{cache: mdb.schemaCache} + var change SchemaChange + dmlEntries, hasDDL, err := mdb.applyReplayedRowsInTx(ctx, tx, t, seqNum, schemaAdapter, &change) + if err != nil { + return nil, false, err + } + + if !change.Empty() { + if err := mdb.FinishReplayedSchemaChange(tx, &change); err != nil { + return nil, false, fmt.Errorf("finish replayed schema change: %w", err) + } + } + + var newSchemaVersion uint64 + if hasDDL { + newSchemaVersion, err = bumpSchemaVersionInTx(tx) + if err != nil { + return nil, false, fmt.Errorf("bump schema version: %w", err) + } + } + + if err := MarkSQLiteTxnApplied(tx, t.TxnID, t.CommitTS); err != nil { + return nil, false, err + } + + if err := tx.Commit(); err != nil { + return nil, false, fmt.Errorf("commit replay transaction: %w", err) + } + + return &replayTxCommitResult{change: &change, dmlEntries: dmlEntries, hasDDL: hasDDL, newSchemaVersion: newSchemaVersion}, true, nil +} + +// replayPrecheck is ApplyReplayedTxn's step 1-2 precheck: refuse to touch +// anything if this node itself holds t.TxnID PENDING (ErrReplayPending), or +// if a DML row t touches is locally held, by intent or CDC row lock, by a +// different transaction (*ReplayIntentConflictError, matching +// errors.Is(err, ErrReplayIntentConflict)). Returns t's existing log +// record, if any, for the caller to reuse its SeqNum. +func (mdb *ReplicatedDatabase) replayPrecheck(t *ReplayTxn) (*TransactionRecord, error) { + rec, err := mdb.metaStore.GetTransaction(t.TxnID) + if err != nil { + return nil, fmt.Errorf("check local transaction status: %w", err) + } + if rec != nil && rec.Status == TxnStatusPending { + return nil, ErrReplayPending + } + + for _, row := range t.Rows { + if !isDMLOp(OpType(row.Op)) || len(row.IntentKey) == 0 { + continue + } + key := string(row.IntentKey) + if intent, ierr := mdb.metaStore.GetIntent(row.Table, key); ierr == nil && intent != nil && intent.TxnID != t.TxnID { + return nil, &ReplayIntentConflictError{HolderTxnID: intent.TxnID} + } + if lockTxnID, lerr := mdb.metaStore.GetCDCRowLock(row.Table, key); lerr == nil && lockTxnID != 0 && lockTxnID != t.TxnID { + return nil, &ReplayIntentConflictError{HolderTxnID: lockTxnID} + } + } + return rec, nil +} + +// applyReplayVectorControl runs t's vector-index control rows, if any, +// outside SQLite (it is not SQLite and cannot share the tx ApplyReplayedTxn +// opens afterward), gated read-only on the applied-txn marker so a replay +// that already committed never re-runs the control. See +// TransactionManager.applyNonDMLIntents's doc comment for why re-running it +// once, on a crash between this step and the marker tx, is safe. +func (mdb *ReplicatedDatabase) applyReplayVectorControl(ctx context.Context, t *ReplayTxn) error { + already, err := markerExists(mdb.readDB, t.TxnID) + if err != nil { + return err + } + if already { + return nil + } + notifier := mdb.txnMgr.VectorCDCNotifier() + for _, row := range t.Rows { + if OpType(row.Op) != OpTypeVectorIndex || row.VectorIndexChange == nil { + continue + } + controlApplier, ok := notifier.(interface { + ApplyVectorControl(context.Context, common.VectorIndexChange) error + }) + if !ok { + return fmt.Errorf("vector index control %s: vector manager not configured", row.VectorIndexChange.IndexName) + } + if err := controlApplier.ApplyVectorControl(ctx, *row.VectorIndexChange); err != nil { + return fmt.Errorf("apply vector index control: %w", err) + } + } + return nil +} + +// applyReplayedRowsInTx applies every row of t inside tx - DML via +// ApplyCDCValues, DDL via ApplyReplayedDDL (accumulating into change), LOAD +// DATA via ApplyLoadDataInTx - and returns the CDC entries to notify vector +// CDC with after commit, and whether any row was DDL. +func (mdb *ReplicatedDatabase) applyReplayedRowsInTx(ctx context.Context, tx *sql.Tx, t *ReplayTxn, seqNum uint64, schemaAdapter *schemaCacheAdapter, change *SchemaChange) ([]common.CDCEntry, bool, error) { + hasDDL := false + dmlEntries := make([]common.CDCEntry, 0, len(t.Rows)) + + for _, row := range t.Rows { + switch OpType(row.Op) { + case OpTypeInsert, OpTypeReplace, OpTypeUpdate, OpTypeDelete: + if err := ApplyCDCValues(tx, schemaAdapter, OpType(row.Op), row.Table, row.OldValues, row.NewValues); err != nil { + return nil, false, fmt.Errorf("apply replayed CDC row for %s: %w", row.Table, err) + } + dmlEntries = append(dmlEntries, common.CDCEntry{ + Table: row.Table, + IntentKey: row.IntentKey, + Operation: row.Op, + OldValues: row.OldValues, + NewValues: row.NewValues, + CommitTxnID: t.TxnID, + CommitSeqNum: seqNum, + }) + case OpTypeDDL: + if err := mdb.ApplyReplayedDDL(ctx, tx, row.DDLSQL, row.Table, t.OriginNodeID, change); err != nil { + return nil, false, fmt.Errorf("apply replayed DDL: %w", err) + } + hasDDL = true + case OpTypeLoadData: + if _, err := ApplyLoadDataInTx(tx, row.LoadSQL, row.LoadData); err != nil { + return nil, false, fmt.Errorf("apply replayed LOAD DATA: %w", err) + } + case OpTypeVectorIndex: + // Applied outside this tx above (or already applied on an earlier + // attempt, per the marker check); nothing left to do here. + default: + return nil, false, fmt.Errorf("unsupported replayed row op %d for table %s", row.Op, row.Table) + } + } + + return dmlEntries, hasDDL, nil +} + +// afterReplayCommit runs ApplyReplayedTxn's step 6 post-commit refresh: +// reload the schema cache and refresh the read pool when DDL changed it, +// advance the cached schema version, and apply committed vector CDC for the +// DML rows. Every failure here is logged, not returned - the row data itself +// already committed in SQLite. +func (mdb *ReplicatedDatabase) afterReplayCommit(ctx context.Context, t *ReplayTxn, seqNum uint64, change *SchemaChange, hasDDL bool, newSchemaVersion uint64, dmlEntries []common.CDCEntry) { + if !change.Empty() { + if err := mdb.ReloadSchema(); err != nil { + log.Warn().Err(err).Uint64("txn_id", t.TxnID).Str("database", mdb.dbName).Msg("ApplyReplayedTxn: failed to reload schema after DDL") + } + } + if hasDDL { + mdb.advanceSchemaVersion(newSchemaVersion) + mdb.RefreshReadPool() + } + + if len(dmlEntries) > 0 { + if notifier := mdb.txnMgr.VectorCDCNotifier(); notifier != nil { + if err := notifier.ApplyCommittedVectorCDC(ctx, mdb.dbName, t.TxnID, seqNum, dmlEntries); err != nil { + log.Error().Err(err).Uint64("txn_id", t.TxnID).Str("database", mdb.dbName). + Msg("ApplyReplayedTxn: vector CDC failed after row commit; local vector index is dirty") + } + } + } +} + +// AppliedTxns returns, for every id in ids, whether __marmot_applied_txn +// holds a marker for it. Looked up through the read pool in chunks to bound +// the IN clause's size. +func (mdb *ReplicatedDatabase) AppliedTxns(ids []uint64) (map[uint64]bool, error) { + result := make(map[uint64]bool, len(ids)) + const chunkSize = 512 + + for start := 0; start < len(ids); start += chunkSize { + end := start + chunkSize + if end > len(ids) { + end = len(ids) + } + chunk := ids[start:end] + + placeholders := make([]byte, 0, len(chunk)*2) + args := make([]interface{}, len(chunk)) + for i, id := range chunk { + if i > 0 { + placeholders = append(placeholders, ',') + } + placeholders = append(placeholders, '?') + args[i] = id + result[id] = false + } + + query := fmt.Sprintf("SELECT txn_id FROM __marmot_applied_txn WHERE txn_id IN (%s)", string(placeholders)) + rows, err := mdb.readDB.Query(query, args...) + if err != nil { + return nil, fmt.Errorf("query applied markers: %w", err) + } + for rows.Next() { + var id uint64 + if err := rows.Scan(&id); err != nil { + rows.Close() + return nil, fmt.Errorf("scan applied marker: %w", err) + } + result[id] = true + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, err + } + rows.Close() + } + + return result, nil +} + +// ReapplyLocalLog walks this database's own local log, in position order, +// and re-applies (through ApplyReplayedTxn with logFirst=false, since the +// entry is already in this node's own log) every COMMITTED entry whose +// marker a restored SQLite file lacks. Used after a snapshot restore: the +// restored file may lack markers for transactions this node's own log +// already holds durably, that the snapshot's source itself lacked. +// +// An entry that cannot be applied - PENDING, an intent conflict, a +// capture/decode error, or a captured row count that does not match the log +// record's own RowCount (never applying a partial transaction) - does +// not stop the walk: every other entry is still re-applied, and the first +// such error is returned with the count of entries newly applied, so the +// caller keeps the re-apply pending and runs it again. A repeated run is +// always safe, since ApplyReplayedTxn applies nothing whose marker exists. +func (mdb *ReplicatedDatabase) ReapplyLocalLog(ctx context.Context) (int, error) { + applied := 0 + var firstErr error + after := LogPosition{} + + for { + if err := ctx.Err(); err != nil { + return applied, err + } + entries, _, more, err := mdb.metaStore.ListCommittedLog(after, reapplyLocalLogPageSize) + if err != nil { + return applied, fmt.Errorf("list local log: %w", err) + } + + ids := make([]uint64, len(entries)) + for i, pos := range entries { + ids[i] = pos.TxnID + } + markers, err := mdb.AppliedTxns(ids) + if err != nil { + return applied, err + } + + for _, pos := range entries { + after = pos + if markers[pos.TxnID] { + continue + } + replayed, err := mdb.reapplyLocalEntry(ctx, pos.TxnID) + if err != nil { + if firstErr == nil { + firstErr = fmt.Errorf("reapply txn %d: %w", pos.TxnID, err) + } + continue + } + if replayed { + applied++ + } + } + + if !more { + return applied, firstErr + } + } +} + +// ReapplyLocalLogIfPending runs ReapplyLocalLog when the durable +// re-apply-pending mark (MetaStore.SetReapplyPending) is set, and clears the +// mark only once a run succeeds. It is a no-op otherwise. +func (mdb *ReplicatedDatabase) ReapplyLocalLogIfPending(ctx context.Context) (int, error) { + pending, err := mdb.metaStore.ReapplyPending() + if err != nil || !pending { + return 0, err + } + applied, err := mdb.ReapplyLocalLog(ctx) + if err != nil { + return applied, err + } + return applied, mdb.metaStore.SetReapplyPending(false) +} + +// reapplyLocalEntry re-applies one committed entry of this node's own log +// from its locally captured rows. +func (mdb *ReplicatedDatabase) reapplyLocalEntry(ctx context.Context, txnID uint64) (bool, error) { + rec, err := mdb.metaStore.GetTransaction(txnID) + if err != nil { + return false, fmt.Errorf("read local log entry: %w", err) + } + if rec == nil || rec.Status != TxnStatusCommitted { + return false, nil + } + rows, err := decodeLocalCapturedRows(mdb.metaStore, txnID, rec.RowCount) + if err != nil { + return false, fmt.Errorf("read captured rows: %w", err) + } + return mdb.ApplyReplayedTxn(ctx, &ReplayTxn{ + TxnID: txnID, + OriginNodeID: rec.NodeID, + CommitTS: hlc.Timestamp{WallTime: rec.CommitTSWall, Logical: rec.CommitTSLogical, NodeID: rec.NodeID}, + RequiredSchemaVersion: rec.RequiredSchemaVersion, + Rows: rows, + }, false) +} + +// decodeLocalCapturedRows reads and decodes every captured row this node's +// own meta store holds for txnID, and refuses a partial result: a capture or +// decode error, or a row count that does not match expected (the log +// record's own RowCount), fails outright rather than silently replaying a +// truncated transaction. +func decodeLocalCapturedRows(metaStore MetaStore, txnID uint64, expected uint32) ([]*EncodedCapturedRow, error) { + cursor, err := metaStore.IterateCapturedRows(txnID) + if err != nil { + return nil, fmt.Errorf("iterate captured rows: %w", err) + } + defer cursor.Close() + + rows := make([]*EncodedCapturedRow, 0, expected) + for cursor.Next() { + _, data := cursor.Row() + row, err := DecodeRow(data) + if err != nil { + return nil, fmt.Errorf("decode captured row: %w", err) + } + rows = append(rows, row) + } + if err := cursor.Err(); err != nil { + return nil, fmt.Errorf("iterate captured rows: %w", err) + } + + if uint32(len(rows)) != expected { + return nil, fmt.Errorf("captured row count mismatch: got %d want %d", len(rows), expected) + } + return rows, nil +} diff --git a/db/replay_apply_test.go b/db/replay_apply_test.go new file mode 100644 index 00000000..ef3cadb4 --- /dev/null +++ b/db/replay_apply_test.go @@ -0,0 +1,496 @@ +package db + +import ( + "context" + "sync" + "testing" + + "github.com/maxpert/marmot/common" + "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" +) + +// fakeVectorNotifier implements both VectorCDCNotifier and the duck-typed +// ApplyVectorControl interface applyNonDMLIntents/ApplyReplayedTxn look for, +// counting calls so a test can pin exactly-once application. +type fakeVectorNotifier struct { + controlCalls int + cdcCalls int +} + +func (f *fakeVectorNotifier) ApplyVectorControl(_ context.Context, _ common.VectorIndexChange) error { + f.controlCalls++ + return nil +} + +func (f *fakeVectorNotifier) ApplyCommittedVectorCDC(_ context.Context, _ string, _, _ uint64, _ []common.CDCEntry) error { + f.cdcCalls++ + return nil +} + +// newReplayTestDatabase creates a real ReplicatedDatabase, under a real +// DatabaseManager, with a "docs" table and a fakeVectorNotifier wired in. +func newReplayTestDatabase(t *testing.T) (*DatabaseManager, *ReplicatedDatabase, *fakeVectorNotifier) { + t.Helper() + dbMgr, err := NewDatabaseManager(t.TempDir(), 1, hlc.NewClock(1)) + require.NoError(t, err) + t.Cleanup(func() { dbMgr.Close() }) + require.NoError(t, dbMgr.CreateDatabase("app")) + mdb, err := dbMgr.GetDatabase("app") + require.NoError(t, err) + + _, err = mdb.GetWriteDB().Exec(`CREATE TABLE docs (id INTEGER PRIMARY KEY, title TEXT)`) + require.NoError(t, err) + require.NoError(t, mdb.ReloadSchema()) + + fake := &fakeVectorNotifier{} + mdb.GetTransactionManager().SetVectorCDCNotifier(fake) + + return dbMgr, mdb, fake +} + +func insertRow(id int64, title string) *EncodedCapturedRow { + return &EncodedCapturedRow{ + Table: "docs", + Op: uint8(OpTypeInsert), + IntentKey: []byte("docs:" + title), + NewValues: encodeTestValues(map[string]interface{}{"id": id, "title": title}), + } +} + +func rowExists(t *testing.T, mdb *ReplicatedDatabase, id int64) bool { + t.Helper() + var got int64 + err := mdb.GetWriteDB().QueryRow("SELECT id FROM docs WHERE id = ?", id).Scan(&got) + if err != nil { + return false + } + return got == id +} + +// TestApplyReplayedTxn_IdempotentSecondCallAppliesNothing pins the core +// exactly-once contract: replaying the same committed txn twice applies the +// row once, and the second call is a no-op (applied=false, nil error), with +// the row's value unchanged. +func TestApplyReplayedTxn_IdempotentSecondCallAppliesNothing(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + + txn := &ReplayTxn{TxnID: 100, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: 10, NodeID: 1}, Rows: []*EncodedCapturedRow{insertRow(1, "hello")}} + + applied, err := mdb.ApplyReplayedTxn(context.Background(), txn, true) + require.NoError(t, err) + require.True(t, applied) + require.True(t, rowExists(t, mdb, 1)) + + applied, err = mdb.ApplyReplayedTxn(context.Background(), txn, true) + require.NoError(t, err) + require.False(t, applied, "a second replay of an already-applied txn must be a no-op") +} + +// TestApplyReplayedTxn_PendingRefusedNothingWritten pins that a txn this +// node itself holds PENDING (a prepared 2PC transaction it has not yet +// resolved) must be refused, untouched, rather than overwritten by replay. +func TestApplyReplayedTxn_PendingRefusedNothingWritten(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + + require.NoError(t, mdb.GetMetaStore().BeginTransaction(200, 1, hlc.Timestamp{WallTime: 1, NodeID: 1})) + + txn := &ReplayTxn{TxnID: 200, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: 10, NodeID: 1}, Rows: []*EncodedCapturedRow{insertRow(2, "pending")}} + + applied, err := mdb.ApplyReplayedTxn(context.Background(), txn, true) + require.ErrorIs(t, err, ErrReplayPending) + require.False(t, applied) + require.False(t, rowExists(t, mdb, 2), "nothing must be written when the local txn is PENDING") + + rec, err := mdb.GetMetaStore().GetTransaction(200) + require.NoError(t, err) + require.NotNil(t, rec) + require.Equal(t, TxnStatusPending, rec.Status, "the local PENDING record must be untouched") +} + +// countLogEntries returns how many of store's committed log entries name txnID. +func countLogEntries(t *testing.T, store MetaStore, txnID uint64) int { + t.Helper() + entries, _, _, err := store.ListCommittedLog(LogPosition{}, 100) + require.NoError(t, err) + count := 0 + for _, e := range entries { + if e.TxnID == txnID { + count++ + } + } + return count +} + +// beginPreparedDMLTxn begins a txn via tm.BeginTransaction and manually +// writes it one captured DML row (mirroring what PREPARE's hook capture +// leaves behind before COMMIT), so CommitTransaction's DML path +// (GetIntentEntries) finds it, without depending on the preupdate-hook +// pipeline elsewhere in this package's test scope. +func beginPreparedDMLTxn(t *testing.T, tm *TransactionManager, metaStore MetaStore, row *EncodedCapturedRow) *Transaction { + t.Helper() + txn, err := tm.BeginTransaction(1) + require.NoError(t, err) + data, err := EncodeRow(row) + require.NoError(t, err) + require.NoError(t, metaStore.WriteCapturedRow(txn.ID, 1, data)) + return txn +} + +// TestCommitTransaction_SecondCommitOfSameTxnIsRefused is the +// concurrent-commit guard's sequential case. A second *Transaction object +// for the same txn id - reconstructed via TransactionManager.GetTransaction, +// exactly as a late COMMIT RPC's handler would, before the first commit +// resolves the id - must be refused once the first commit has already +// finished. ApplyCDCInsert is INSERT OR REPLACE, so a second, unguarded +// apply of the very same captured row would succeed at the SQL level and +// reach finalizeCommit, writing a second log entry for the same txn id: the +// guard, not a coincidental SQL error, is what must stop it. +func TestCommitTransaction_SecondCommitOfSameTxnIsRefused(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + tm := mdb.GetTransactionManager() + metaStore := mdb.GetMetaStore() + + txnA := beginPreparedDMLTxn(t, tm, metaStore, insertRow(42, "guarded")) + txnB := tm.GetTransaction(txnA.ID) + require.NotNil(t, txnB, "a second Transaction object for the same id must reconstruct while PENDING") + + require.NoError(t, tm.CommitTransaction(txnA)) + require.True(t, rowExists(t, mdb, 42)) + + err := tm.CommitTransaction(txnB) + require.Error(t, err, "a second commit of an already-committed txn id must be refused") + + require.Equal(t, 1, countLogEntries(t, metaStore, txnA.ID), "the local log must hold exactly one entry") +} + +// TestCommitTransaction_ConcurrentCommitsOfSameTxnSerialize is the +// concurrent-commit guard's concurrent case: two goroutines committing +// distinct *Transaction objects for the same txn id at the same time (the +// log puller's local-PREPARE commit racing a late COMMIT RPC) must resolve +// to exactly one success, with exactly one log entry, never both goroutines +// applying the DML. +func TestCommitTransaction_ConcurrentCommitsOfSameTxnSerialize(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + tm := mdb.GetTransactionManager() + metaStore := mdb.GetMetaStore() + + txnA := beginPreparedDMLTxn(t, tm, metaStore, insertRow(43, "guarded-concurrent")) + txnB := tm.GetTransaction(txnA.ID) + require.NotNil(t, txnB) + + var wg sync.WaitGroup + errs := make([]error, 2) + wg.Add(2) + go func() { defer wg.Done(); errs[0] = tm.CommitTransaction(txnA) }() + go func() { defer wg.Done(); errs[1] = tm.CommitTransaction(txnB) }() + wg.Wait() + + successes := 0 + for _, e := range errs { + if e == nil { + successes++ + } + } + require.Equal(t, 1, successes, "exactly one of two concurrent commits of the same txn id must succeed") + require.Equal(t, 1, countLogEntries(t, metaStore, txnA.ID), "the local log must hold exactly one entry even under a concurrent race") +} + +// TestLogReplayedTxn_LocalBeginPendingRefusesAndWritesNothing: a txn id begun through TransactionManager.BeginTransactionWithID (the actual +// entry point a local PREPARE uses, exercising the txnIDLocks stripe the fix +// adds) makes both LogReplayedTxn and ApplyReplayedTxn refuse with +// ErrReplayPending, writing no captured row and no log entry - not merely +// leaving the immutable-only record it started with untouched. +func TestLogReplayedTxn_LocalBeginPendingRefusesAndWritesNothing(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + tm := mdb.GetTransactionManager() + + const txnID = uint64(500) + _, err := tm.BeginTransactionWithID(txnID, 1, hlc.Timestamp{WallTime: 1, NodeID: 1}) + require.NoError(t, err) + + rows := []*EncodedCapturedRow{insertRow(5, "gated")} + + _, err = mdb.LogReplayedTxn(&ReplayTxn{TxnID: txnID, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: 10, NodeID: 1}, Rows: rows}) + require.ErrorIs(t, err, ErrReplayPending) + + cursor, err := mdb.GetMetaStore().IterateCapturedRows(txnID) + require.NoError(t, err) + require.False(t, cursor.Next(), "no captured row must be written while the local begin is pending") + require.NoError(t, cursor.Err()) + require.NoError(t, cursor.Close()) + + entries, _, _, err := mdb.GetMetaStore().ListCommittedLog(LogPosition{}, 10) + require.NoError(t, err) + for _, e := range entries { + require.NotEqual(t, txnID, e.TxnID, "no log entry must be written while the local begin is pending") + } + + applied, err := mdb.ApplyReplayedTxn(context.Background(), &ReplayTxn{TxnID: txnID, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: 10, NodeID: 1}, Rows: rows}, true) + require.False(t, applied) + require.ErrorIs(t, err, ErrReplayPending) +} + +// TestApplyReplayedTxn_ForeignIntentConflictRefusedNothingWritten: a row locally held by another transaction's write intent must refuse +// replay (ErrReplayIntentConflict) rather than overwrite it with an older or +// unrelated image. +func TestApplyReplayedTxn_ForeignIntentConflictRefusedNothingWritten(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + + otherTxnID := uint64(9) + require.NoError(t, mdb.GetMetaStore().WriteIntent(otherTxnID, IntentTypeDML, "docs", "docs:conflict", + OpTypeInsert, "", nil, hlc.Timestamp{WallTime: 1, NodeID: 1}, 1)) + + row := insertRow(3, "conflict") + row.IntentKey = []byte("docs:conflict") + txn := &ReplayTxn{TxnID: 300, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: 10, NodeID: 1}, Rows: []*EncodedCapturedRow{row}} + + applied, err := mdb.ApplyReplayedTxn(context.Background(), txn, true) + require.ErrorIs(t, err, ErrReplayIntentConflict) + require.False(t, applied) + require.False(t, rowExists(t, mdb, 3), "nothing must be written on an intent conflict") + + var conflictErr *ReplayIntentConflictError + require.ErrorAs(t, err, &conflictErr, "the concrete error must carry the intent holder's txn id") + require.Equal(t, otherTxnID, conflictErr.HolderTxnID) +} + +// TestApplyReplayedTxn_CDCRowLockConflictCarriesHolderTxnID: a row +// held by another transaction's CDC row lock (rather than a write intent) +// must also surface *ReplayIntentConflictError with that lock's holder id. +func TestApplyReplayedTxn_CDCRowLockConflictCarriesHolderTxnID(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + + holderTxnID := uint64(11) + require.NoError(t, mdb.GetMetaStore().AcquireCDCRowLock(holderTxnID, "docs", "docs:lockconflict")) + + row := insertRow(4, "lockconflict") + row.IntentKey = []byte("docs:lockconflict") + txn := &ReplayTxn{TxnID: 301, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: 10, NodeID: 1}, Rows: []*EncodedCapturedRow{row}} + + applied, err := mdb.ApplyReplayedTxn(context.Background(), txn, true) + require.ErrorIs(t, err, ErrReplayIntentConflict) + require.False(t, applied) + + var conflictErr *ReplayIntentConflictError + require.ErrorAs(t, err, &conflictErr) + require.Equal(t, holderTxnID, conflictErr.HolderTxnID) +} + +// TestApplyReplayedTxn_ClaimOnlyMarksOnly pins that a zero-row replayed txn +// (the shape a claim-only commit's replay takes: claim payloads never travel +// in CDC) only writes the marker. +func TestApplyReplayedTxn_ClaimOnlyMarksOnly(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + + txn := &ReplayTxn{TxnID: 400, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: 10, NodeID: 1}} + + applied, err := mdb.ApplyReplayedTxn(context.Background(), txn, true) + require.NoError(t, err) + require.True(t, applied) + + var one int + require.NoError(t, mdb.GetWriteDB().QueryRow("SELECT 1 FROM __marmot_applied_txn WHERE txn_id = ?", 400).Scan(&one)) +} + +// TestApplyReplayedTxn_DDLUsesOriginAsOwnerAndBumpsVersionOnce pins that the +// DDL incarnation owner is the transaction's origin, not the replaying node, +// together with the version-bump contract already covered end to end +// by TestApplyNonDMLIntents_2PCDDLCommitBumpsVersionExactlyOnceAndReplayIsNoOp, +// exercised here from a cold ApplyReplayedTxn call instead. +func TestApplyReplayedTxn_DDLUsesOriginAsOwnerAndBumpsVersionOnce(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + + row := &EncodedCapturedRow{Table: "docs", Op: uint8(OpTypeDDL), DDLSQL: "ALTER TABLE docs ADD COLUMN rank INTEGER"} + const origin = uint64(42) + txn := &ReplayTxn{TxnID: 500, OriginNodeID: origin, CommitTS: hlc.Timestamp{WallTime: 10, NodeID: origin}, Rows: []*EncodedCapturedRow{row}} + + applied, err := mdb.ApplyReplayedTxn(context.Background(), txn, true) + require.NoError(t, err) + require.True(t, applied) + require.Equal(t, uint64(1), mdb.SchemaVersion()) + + var name string + require.NoError(t, mdb.GetWriteDB().QueryRow( + "SELECT name FROM pragma_table_info('docs') WHERE name = 'rank'").Scan(&name)) + require.Equal(t, "rank", name) +} + +// TestApplyReplayedTxn_VectorControlNotReappliedWhenMarked: once +// a replayed vector-control txn's marker is present, a later replay attempt +// (the shape a retried anti-entropy round takes) must not re-run the +// control. +func TestApplyReplayedTxn_VectorControlNotReappliedWhenMarked(t *testing.T) { + _, mdb, fake := newReplayTestDatabase(t) + + change := common.VectorIndexChange{ + Action: common.VectorIndexActionCreate, Database: "app", + IndexName: "docs_idx", TableName: "docs", ColumnName: "title", + } + row := &EncodedCapturedRow{Table: "docs", Op: uint8(OpTypeVectorIndex), VectorIndexChange: &change} + txn := &ReplayTxn{TxnID: 600, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: 10, NodeID: 1}, Rows: []*EncodedCapturedRow{row}} + + applied, err := mdb.ApplyReplayedTxn(context.Background(), txn, true) + require.NoError(t, err) + require.True(t, applied) + require.Equal(t, 1, fake.controlCalls) + + applied, err = mdb.ApplyReplayedTxn(context.Background(), txn, true) + require.NoError(t, err) + require.False(t, applied) + require.Equal(t, 1, fake.controlCalls, "a marked vector-control txn must not re-run the control") +} + +// TestApplyReplayedTxn_LogFirstLeavesCommittedLogEntryWithNoMarkerOnFailure +// pins the log-first crash-window contract: when the SQLite apply +// fails (here, a DML row into a table that does not exist), the txn is still +// COMMITTED in the local log because it was logged first, but carries no +// marker. ReapplyLocalLog then finds and re-applies it once the underlying +// problem is fixed, and leaves an already-marked entry alone. +func TestApplyReplayedTxn_LogFirstLeavesCommittedLogEntryWithNoMarkerOnFailure(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + + badRow := &EncodedCapturedRow{ + Table: "missing_table", + Op: uint8(OpTypeInsert), + IntentKey: []byte("missing_table:1"), + NewValues: encodeTestValues(map[string]interface{}{"id": int64(1)}), + } + txn := &ReplayTxn{TxnID: 700, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: 10, NodeID: 1}, Rows: []*EncodedCapturedRow{badRow}} + + applied, err := mdb.ApplyReplayedTxn(context.Background(), txn, true) + require.Error(t, err) + require.False(t, applied) + + rec, err := mdb.GetMetaStore().GetTransaction(700) + require.NoError(t, err) + require.NotNil(t, rec) + require.Equal(t, TxnStatusCommitted, rec.Status, "log-first must have recorded the txn as committed despite the apply failure") + + has, err := markerExists(mdb.GetWriteDB(), 700) + require.NoError(t, err) + require.False(t, has, "no marker must exist after a failed apply") +} + +// TestAppliedTxns_ReportsMarkerPresence pins AppliedTxns' contract: true for +// every id that carries a marker, false for the rest, including an id this +// database has never heard of. +func TestAppliedTxns_ReportsMarkerPresence(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + + applied, err := mdb.ApplyReplayedTxn(context.Background(), &ReplayTxn{ + TxnID: 1000, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: 10, NodeID: 1}, + Rows: []*EncodedCapturedRow{insertRow(1, "marked")}, + }, true) + require.NoError(t, err) + require.True(t, applied) + + result, err := mdb.AppliedTxns([]uint64{1000, 1001, 9999999}) + require.NoError(t, err) + require.Equal(t, map[uint64]bool{1000: true, 1001: false, 9999999: false}, result) +} + +// TestReapplyLocalLog_ReappliesMissingSkipsPresent is the restore +// re-apply regression: ReapplyLocalLog re-applies a local log entry whose +// marker is absent (as a restored SQLite file would be, or as the failed +// apply above leaves it once the missing table is created) and leaves an +// already-marked entry alone. +func TestReapplyLocalLog_ReappliesMissingSkipsPresent(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + + // Entry 1: applied normally, so it already carries a marker. + applied, err := mdb.ApplyReplayedTxn(context.Background(), &ReplayTxn{ + TxnID: 800, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: 10, NodeID: 1}, + Rows: []*EncodedCapturedRow{insertRow(10, "already-applied")}, + }, true) + require.NoError(t, err) + require.True(t, applied) + + // Entry 2: logged (COMMITTED) but its apply failed - "missing2" doesn't + // exist yet - leaving no marker, exactly like a restored file's gap. + badRow := &EncodedCapturedRow{ + Table: "missing2", Op: uint8(OpTypeInsert), IntentKey: []byte("missing2:1"), + NewValues: encodeTestValues(map[string]interface{}{"id": int64(1)}), + } + _, err = mdb.ApplyReplayedTxn(context.Background(), &ReplayTxn{ + TxnID: 900, OriginNodeID: 1, CommitTS: hlc.Timestamp{WallTime: 20, NodeID: 1}, + Rows: []*EncodedCapturedRow{badRow}, + }, true) + require.Error(t, err) + + has800, err := markerExists(mdb.GetWriteDB(), 800) + require.NoError(t, err) + require.True(t, has800) + has900, err := markerExists(mdb.GetWriteDB(), 900) + require.NoError(t, err) + require.False(t, has900) + + // Fix the underlying problem, then walk the local log: entry 800 must be + // left alone (already marked), entry 900 re-applied for real. + _, err = mdb.GetWriteDB().Exec("CREATE TABLE missing2 (id INTEGER PRIMARY KEY)") + require.NoError(t, err) + require.NoError(t, mdb.ReloadSchema()) + + count, err := mdb.ReapplyLocalLog(context.Background()) + require.NoError(t, err) + require.Equal(t, 1, count, "only the unmarked entry must be reapplied") + + has900, err = markerExists(mdb.GetWriteDB(), 900) + require.NoError(t, err) + require.True(t, has900, "the previously-failed entry must now carry a marker") + + var got int64 + require.NoError(t, mdb.GetWriteDB().QueryRow("SELECT id FROM missing2 WHERE id = 1").Scan(&got)) + require.Equal(t, int64(1), got) + + // A second walk finds nothing left to do. + count, err = mdb.ReapplyLocalLog(context.Background()) + require.NoError(t, err) + require.Equal(t, 0, count) +} + +// TestBeginTransactionWithID_LatePrepareAfterReplayIsRefused pins that a +// PREPARE arriving after its txn was already pulled and replayed is refused +// (ErrTxnAlreadyCommitted) and leaves the committed log entry untouched, +// instead of overwriting it with a PENDING record whose abort or commit +// would corrupt it. +func TestBeginTransactionWithID_LatePrepareAfterReplayIsRefused(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + tm := mdb.GetTransactionManager() + + const txnID = uint64(600) + applied, err := mdb.ApplyReplayedTxn(context.Background(), &ReplayTxn{ + TxnID: txnID, OriginNodeID: 2, CommitTS: hlc.Timestamp{WallTime: 10, NodeID: 2}, + Rows: []*EncodedCapturedRow{insertRow(6, "replayed")}, + }, true) + require.NoError(t, err) + require.True(t, applied) + + _, err = tm.BeginTransactionWithID(txnID, 2, hlc.Timestamp{WallTime: 5, NodeID: 2}) + require.ErrorIs(t, err, ErrTxnAlreadyCommitted) + + rec, err := mdb.GetMetaStore().GetTransaction(txnID) + require.NoError(t, err) + require.NotNil(t, rec) + require.Equal(t, TxnStatusCommitted, rec.Status, "the refused begin must leave the committed record as it was") + require.Equal(t, 1, countLogEntries(t, mdb.GetMetaStore(), txnID)) +} + +// TestBeginTransactionWithID_RefusedWhenOnlyTheMarkerExists pins the marker +// half of the refusal: a txn applied in this database's SQLite file (a +// restored snapshot's marker, say) with no local record is refused too. +func TestBeginTransactionWithID_RefusedWhenOnlyTheMarkerExists(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + + const txnID = uint64(700) + tx, err := mdb.GetWriteDB().Begin() + require.NoError(t, err) + require.NoError(t, MarkSQLiteTxnApplied(tx, txnID, hlc.Timestamp{WallTime: 10, NodeID: 2})) + require.NoError(t, tx.Commit()) + + _, err = mdb.GetTransactionManager().BeginTransactionWithID(txnID, 2, hlc.Timestamp{WallTime: 5, NodeID: 2}) + require.ErrorIs(t, err, ErrTxnAlreadyCommitted) + rec, err := mdb.GetMetaStore().GetTransaction(txnID) + require.NoError(t, err) + require.Nil(t, rec, "a refused begin must write no record") +} diff --git a/db/replication_engine.go b/db/replication_engine.go index 59f506a9..47f49320 100644 --- a/db/replication_engine.go +++ b/db/replication_engine.go @@ -67,7 +67,6 @@ type CommitRequest struct { type CommitResult struct { Success bool Error string - DDLSQL string // SQL for DDL statements (if any) // ClaimNotApplicable reports a COMMIT refused because this node could not // apply the AUTO_INCREMENT claim it carries (ErrAutoIncClaimNotApplicable). ClaimNotApplicable bool @@ -145,6 +144,19 @@ func statementsCarryAutoIDClaim(statements []protocol.Statement) bool { func (re *ReplicationEngine) prepareDatabaseOperation(req *PrepareRequest) *PrepareResult { stmt := req.Statements[0] + dbOp := DatabaseOpCreate + if stmt.Type == protocol.StatementDropDatabase { + dbOp = DatabaseOpDrop + } + + // Fail fast: resolve and fence the op's registry key before touching the + // system database's transaction manager, so a stale coordinator's PREPARE + // is refused without leaving a transaction behind to abort. + opKey, rejected := re.resolveDatabaseOpKey(stmt, dbOp) + if rejected != nil { + return rejected + } + systemDB, err := re.dbMgr.GetDatabase(SystemDatabaseName) if err != nil { return &PrepareResult{Success: false, Error: fmt.Sprintf("system database not found: %v", err)} @@ -162,16 +174,13 @@ func (re *ReplicationEngine) prepareDatabaseOperation(req *PrepareRequest) *Prep } dbIntentKey := filter.EncodeDBOpIntentKey(stmt.Database) - dbOp := DatabaseOpCreate - if stmt.Type == protocol.StatementDropDatabase { - dbOp = DatabaseOpDrop - } snapshotData := DatabaseOperationSnapshot{ Type: int(stmt.Type), Timestamp: req.StartTS.WallTime, DatabaseName: stmt.Database, Operation: dbOp, + Generation: opKey.Generation, } dataSnapshot, err := SerializeData(snapshotData) if err != nil { @@ -187,6 +196,7 @@ func (re *ReplicationEngine) prepareDatabaseOperation(req *PrepareRequest) *Prep log.Info(). Str("database", stmt.Database). Str("operation", dbOp.String()). + Uint64("generation", opKey.Generation). Uint64("node_id", re.nodeID). Uint64("txn_id", req.TxnID). Msg("Database operation prepared (intent created)") @@ -199,6 +209,55 @@ func (re *ReplicationEngine) prepareDatabaseOperation(req *PrepareRequest) *Prep return &PrepareResult{Success: true} } +// resolveDatabaseOpKey computes the DatabaseRegistryKey a CREATE/DROP +// DATABASE statement's PREPARE resolves to for stmt.Database, and gates a +// coordinator's stamp against this participant's local key. +// +// stmt.DatabaseGeneration == 0 means "unstamped": a coordinator that predates +// Statement.DatabaseGeneration (rolling upgrade). The key is then computed +// locally, the same way a stamping coordinator would from its own registry +// (CREATE: the current key if live, else one generation above it; DROP: the +// current key), and the gate below is skipped - this participant simply +// trusts the legacy request, as it always has. +// +// Otherwise the stamped key is compared with the local key: below local means +// the coordinator's view of the database's history is stale, so the PREPARE +// is refused with ErrStaleDatabaseOpCoordinator (see +// DatabaseManager.ApplyDatabaseOp's proof for why this is required for +// convergence). Equal or above is accepted unchanged. +func (re *ReplicationEngine) resolveDatabaseOpKey(stmt protocol.Statement, dbOp DatabaseOpType) (DatabaseRegistryKey, *PrepareResult) { + dbMgr, ok := re.dbMgr.(*DatabaseManager) + if !ok { + return DatabaseRegistryKey{}, &PrepareResult{Success: false, Error: "database manager does not support database operations"} + } + + local, err := dbMgr.RegistryKey(stmt.Database) + if err != nil { + return DatabaseRegistryKey{}, &PrepareResult{Success: false, Error: fmt.Sprintf("failed to read database registry: %v", err)} + } + + if stmt.DatabaseGeneration == 0 { + if dbOp == DatabaseOpDrop { + return local, nil + } + if !local.Dropped { + return local, nil // already live: CREATE is a no-op at the current generation + } + return DatabaseRegistryKey{Generation: local.Generation + 1, Dropped: false}, nil + } + + opKey := DatabaseRegistryKey{Generation: stmt.DatabaseGeneration, Dropped: dbOp == DatabaseOpDrop} + if opKey.Compare(local) < 0 { + return DatabaseRegistryKey{}, &PrepareResult{ + Success: false, + Rejected: true, + Error: fmt.Sprintf("%v: database %s op key %+v is below local key %+v", + ErrStaleDatabaseOpCoordinator, stmt.Database, opKey, local), + } + } + return opKey, nil +} + // prepareRegularTransaction handles regular transaction preparation func (re *ReplicationEngine) prepareRegularTransaction(ctx context.Context, req *PrepareRequest) *PrepareResult { replicatedDB, err := re.dbMgr.GetDatabase(req.Database) @@ -278,7 +337,13 @@ func (re *ReplicationEngine) prepareRegularTransaction(ctx context.Context, req // own BEGIN. Callers must reject the PREPARE / refuse the COMMIT on error // rather than silently skip the claim. func (re *ReplicationEngine) autoIncClaimStore() (*AutoIncClaimStore, error) { - systemDB, err := re.dbMgr.GetDatabase(SystemDatabaseName) + return autoIncClaimStoreOf(re.dbMgr) +} + +// autoIncClaimStoreOf resolves dbMgr's AUTO_INCREMENT claim store (see +// ReplicationEngine.autoIncClaimStore). +func autoIncClaimStoreOf(dbMgr DatabaseProvider) (*AutoIncClaimStore, error) { + systemDB, err := dbMgr.GetDatabase(SystemDatabaseName) if err != nil { return nil, fmt.Errorf("system database unavailable: %w", err) } @@ -705,28 +770,28 @@ func (re *ReplicationEngine) Commit(ctx context.Context, req *CommitRequest) *Co dbOp := snapshotData.Operation dbName := snapshotData.DatabaseName + if dbOp != DatabaseOpCreate && dbOp != DatabaseOpDrop { + log.Error().Str("operation", dbOp.String()).Uint64("txn_id", req.TxnID).Msg("Unknown database operation") + continue + } - // Execute the database operation BEFORE committing the transaction + // Execute the database operation BEFORE committing the transaction, + // through the same key-fenced merge ApplyDatabaseOp gives + // anti-entropy reconciliation. dbMgr, ok := re.dbMgr.(*DatabaseManager) if !ok { _ = systemTxnMgr.AbortTransaction(systemTxn) return &CommitResult{Success: false, Error: "database manager does not support database operations"} } - var dbOpErr error - switch dbOp { - case DatabaseOpCreate: - log.Info().Str("database", dbName).Uint64("node_id", re.nodeID).Msg("Executing CREATE DATABASE in commit phase") - dbOpErr = dbMgr.CreateDatabase(dbName) - case DatabaseOpDrop: - log.Info().Str("database", dbName).Uint64("node_id", re.nodeID).Msg("Executing DROP DATABASE in commit phase") - dbOpErr = dbMgr.DropDatabase(dbName) - default: - log.Error().Str("operation", dbOp.String()).Uint64("txn_id", req.TxnID).Msg("Unknown database operation") - continue - } - - if dbOpErr != nil { + opKey := DatabaseRegistryKey{Generation: snapshotData.Generation, Dropped: dbOp == DatabaseOpDrop} + log.Info(). + Str("database", dbName). + Str("operation", dbOp.String()). + Uint64("generation", opKey.Generation). + Uint64("node_id", re.nodeID). + Msg("Applying database operation in commit phase") + if _, dbOpErr := dbMgr.ApplyDatabaseOp(dbName, opKey); dbOpErr != nil { log.Error().Err(dbOpErr).Str("database", dbName).Str("operation", dbOp.String()).Msg("Database operation failed in commit phase") _ = systemTxnMgr.AbortTransaction(systemTxn) return &CommitResult{Success: false, Error: fmt.Sprintf("database operation failed: %v", dbOpErr)} @@ -745,15 +810,7 @@ func (re *ReplicationEngine) Commit(ctx context.Context, req *CommitRequest) *Co Uint64("node_id", re.nodeID). Msg("Database operation committed successfully") - // Return DDLSQL so replication handler can increment schema version - var ddlSQL string - switch dbOp { - case DatabaseOpCreate: - ddlSQL = fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", dbName) - case DatabaseOpDrop: - ddlSQL = fmt.Sprintf("DROP DATABASE IF EXISTS %s", dbName) - } - return &CommitResult{Success: true, DDLSQL: ddlSQL} + return &CommitResult{Success: true} } } } @@ -766,10 +823,12 @@ func (re *ReplicationEngine) Commit(ctx context.Context, req *CommitRequest) *Co return &CommitResult{Success: false, Error: fmt.Sprintf("database not found: %s", req.Database)} } - txnMgr := replicatedDB.GetTransactionManager() - - txn := txnMgr.GetTransaction(req.TxnID) + txn := replicatedDB.GetTransactionManager().GetTransaction(req.TxnID) if txn == nil { + if committedLocally(replicatedDB, req.TxnID) { + // The log puller already committed it here (CommitLocallyPrepared). + return &CommitResult{Success: true} + } log.Error(). Uint64("txn_id", req.TxnID). Uint64("node_id", re.nodeID). @@ -779,6 +838,60 @@ func (re *ReplicationEngine) Commit(ctx context.Context, req *CommitRequest) *Co } txn.Statements = req.Statements + // The gate is the statement flag, not a store lookup: reading intents on + // every commit would put a Pebble scan on the write path. The flag decides + // only whether to look; every value written comes from the intent. + result, _ := commitPreparedTxn(re.dbMgr, replicatedDB, txn, req.Database, statementsCarryAutoIDClaim(req.Statements)) + return result +} + +// CommitLocallyPrepared commits database's durably prepared transaction +// txnID through the same local commit path a COMMIT RPC takes, for a +// transaction some peer's committed log proves was decided COMMITTED while +// this node never received (or failed to apply) its COMMIT. With no +// coordinator statements to read the claim flag from, whether the +// transaction carries an AUTO_INCREMENT claim is read from its intents. +func (dm *DatabaseManager) CommitLocallyPrepared(database string, txnID uint64) error { + replicatedDB, err := dm.GetDatabase(database) + if err != nil { + return fmt.Errorf("database %s: %w", database, err) + } + txn := replicatedDB.GetTransactionManager().GetTransaction(txnID) + if txn == nil { + if committedLocally(replicatedDB, txnID) { + return nil + } + return fmt.Errorf("transaction %d is not pending in %s", txnID, database) + } + intents, err := replicatedDB.GetMetaStore().GetIntentsByTxn(txnID) + if err != nil { + return fmt.Errorf("read intents for txn %d: %w", txnID, err) + } + carriesClaim := false + for _, intent := range intents { + if intent.IntentType == IntentTypeAutoIDClaim { + carriesClaim = true + break + } + } + _, err = commitPreparedTxn(dm, replicatedDB, txn, database, carriesClaim) + return err +} + +// committedLocally reports whether replicatedDB's local record for txnID is +// COMMITTED. +func committedLocally(replicatedDB *ReplicatedDatabase, txnID uint64) bool { + rec, err := replicatedDB.GetMetaStore().GetTransaction(txnID) + return err == nil && rec != nil && rec.Status == TxnStatusCommitted +} + +// commitPreparedTxn is the user-database half of the COMMIT phase, shared by +// ReplicationEngine.Commit and DatabaseManager.CommitLocallyPrepared: apply +// txn's AUTO_INCREMENT claim first when applyClaims is set, then commit txn +// and refresh the read pool after DDL. It returns the result to answer a +// COMMIT with and the underlying error (nil when that result is a success, +// including for a transaction another caller already committed). +func commitPreparedTxn(dbMgr DatabaseProvider, replicatedDB *ReplicatedDatabase, txn *Transaction, database string, applyClaims bool) (*CommitResult, error) { // Apply any AUTO_INCREMENT range claim BEFORE committing, and refuse the // commit if it cannot be applied (the claim's first invariant: a // participant must never ACK COMMIT unless the claim row is durably in its @@ -795,61 +908,59 @@ func (re *ReplicationEngine) Commit(ctx context.Context, req *CommitRequest) *Co // transaction, instead of unilaterally discarding writes the rest of the // cluster may be committing. // - // The gate is the statement flag, not a store lookup: reading intents on - // every commit would put a Pebble scan on the write path. The flag decides - // only whether to look; every value written comes from the intent. - if statementsCarryAutoIDClaim(req.Statements) { - claimStore, csErr := re.autoIncClaimStore() - if csErr != nil { - log.Error(). - Err(csErr). - Uint64("txn_id", req.TxnID). - Uint64("node_id", re.nodeID). - Str("database", req.Database). - Msg("COMMIT REFUSED: system database unavailable for auto-increment claim apply") - return &CommitResult{Success: false, Error: fmt.Sprintf("auto-increment claim apply failed: %v", csErr)} - } - if err := claimStore.ApplyClaims(req.Database, req.TxnID, replicatedDB.GetMetaStore()); err != nil { - // A claim this node cannot apply is the protocol refusing an ACK - // it must not give; anything else is a failed write. - notApplicable := errors.Is(err, ErrAutoIncClaimNotApplicable) - ev := log.Error() - if notApplicable { - ev = log.Warn() - } - ev.Err(err). - Uint64("txn_id", req.TxnID). - Uint64("node_id", re.nodeID). - Str("database", req.Database). - Msg("COMMIT REFUSED: auto-increment claim could not be applied") - return &CommitResult{Success: false, ClaimNotApplicable: notApplicable, - Error: fmt.Sprintf("auto-increment claim apply failed: %v", err)} + // The claim is applied under CommitTransactionAfter's concurrent-commit + // guard: it is conditional on the claim row's floors, so applying it a + // second time, for a transaction the log puller (or an earlier COMMIT) + // already committed, would be refused as not applicable. + var claimErr error + applyClaim := func() error { + if applyClaims { + claimErr = applyCommitClaims(dbMgr, replicatedDB, txn.ID, database) } + return claimErr } - - if err := txnMgr.CommitTransaction(txn); err != nil { - return &CommitResult{Success: false, Error: err.Error()} + err := replicatedDB.GetTransactionManager().CommitTransactionAfter(txn, applyClaim) + switch { + case claimErr != nil: + return &CommitResult{Success: false, ClaimNotApplicable: errors.Is(claimErr, ErrAutoIncClaimNotApplicable), + Error: fmt.Sprintf("auto-increment claim apply failed: %v", claimErr)}, claimErr + case errors.Is(err, ErrTxnAlreadyCommitted): + return &CommitResult{Success: true}, nil + case err != nil: + return &CommitResult{Success: false, Error: err.Error()}, err } - // Extract DDL SQL from committed statements (populated during CommitTransaction) - var ddlSQL string + // If DDL was committed, refresh read pool to pick up schema changes. + // SQLite caches schema per-connection; read connections need refresh. for _, stmt := range txn.Statements { if stmt.Type == protocol.StatementDDL { - ddlSQL = stmt.SQL + replicatedDB.RefreshReadPool() break } } + return &CommitResult{Success: true}, nil +} - // If DDL was committed, refresh read pool to pick up schema changes - // SQLite caches schema per-connection; read connections need refresh - if ddlSQL != "" { - replicatedDB.RefreshReadPool() - } - - return &CommitResult{ - Success: true, - DDLSQL: ddlSQL, +// applyCommitClaims applies txnID's AUTO_INCREMENT claim intents. An error +// wrapping ErrAutoIncClaimNotApplicable is the protocol refusing an ACK this +// node must not give; any other is a failed write. +func applyCommitClaims(dbMgr DatabaseProvider, replicatedDB *ReplicatedDatabase, txnID uint64, database string) error { + claimStore, err := autoIncClaimStoreOf(dbMgr) + if err != nil { + log.Error().Err(err).Uint64("txn_id", txnID).Str("database", database). + Msg("COMMIT REFUSED: system database unavailable for auto-increment claim apply") + return err + } + if err := claimStore.ApplyClaims(database, txnID, replicatedDB.GetMetaStore()); err != nil { + ev := log.Error() + if errors.Is(err, ErrAutoIncClaimNotApplicable) { + ev = log.Warn() + } + ev.Err(err).Uint64("txn_id", txnID).Str("database", database). + Msg("COMMIT REFUSED: auto-increment claim could not be applied") + return err } + return nil } // Abort handles the abort phase of 2PC replication diff --git a/db/replication_engine_test.go b/db/replication_engine_test.go index 1b4c1aaf..bb0cfcec 100644 --- a/db/replication_engine_test.go +++ b/db/replication_engine_test.go @@ -647,6 +647,103 @@ func TestReplicationEngine_CommitDatabaseOp(t *testing.T) { assert.Equal(t, TxnStatusCommitted, txnRec.Status) } +// TestReplicationEngine_CommitDatabaseOp_UnstampedComputesGenerationLocally +// pins the rolling-upgrade fallback: a +// CREATE/DROP DATABASE statement with DatabaseGeneration left at its zero +// value (an old coordinator that predates the field) is not gated, and +// PREPARE computes its registry key the same way a stamping coordinator +// would - here, generation 1, live, for a name never seen before. +func TestReplicationEngine_CommitDatabaseOp_UnstampedComputesGenerationLocally(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + ctx := context.Background() + + prepReq := &PrepareRequest{ + TxnID: 2004, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: 9003, Logical: 1}, + Statements: []protocol.Statement{ + {Type: protocol.StatementCreateDatabase, Database: "legacydb"}, // DatabaseGeneration left unset (0) + }, + } + prepResult := engine.Prepare(ctx, prepReq) + require.True(t, prepResult.Success) + + commitResult := engine.Commit(ctx, &CommitRequest{TxnID: 2004}) + require.True(t, commitResult.Success) + + key, err := dm.RegistryKey("legacydb") + require.NoError(t, err) + require.Equal(t, DatabaseRegistryKey{Generation: 1, Dropped: false}, key) +} + +// TestReplicationEngine_PrepareDatabaseOp_RefusesStaleCoordinatorStamp +// verifies PREPARE refuses a CREATE/DROP DATABASE whose stamped +// DatabaseGeneration is below the participant's local registry key: a +// coordinator whose view of the database's history predates a drop and +// re-create the participant already knows about. +func TestReplicationEngine_PrepareDatabaseOp_RefusesStaleCoordinatorStamp(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + ctx := context.Background() + + // Drive the database to generation 2, live (create, drop, re-create), so + // a stamp of 1 is stale. + require.NoError(t, dm.CreateDatabase("staledb")) + require.NoError(t, dm.DropDatabase("staledb")) + require.NoError(t, dm.CreateDatabase("staledb")) + local, err := dm.RegistryKey("staledb") + require.NoError(t, err) + require.Equal(t, DatabaseRegistryKey{Generation: 2, Dropped: false}, local) + + result := engine.Prepare(ctx, &PrepareRequest{ + TxnID: 2005, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: 9004, Logical: 1}, + Statements: []protocol.Statement{ + {Type: protocol.StatementCreateDatabase, Database: "staledb", DatabaseGeneration: 1}, + }, + }) + require.False(t, result.Success) + require.True(t, result.Rejected, "a stale coordinator stamp must be a deterministic refusal") +} + +// TestReplicationEngine_PrepareDatabaseOp_AcceptsStampAtOrAboveLocalKey +// verifies PREPARE accepts a stamp equal to, or above, the participant's +// local registry key. +func TestReplicationEngine_PrepareDatabaseOp_AcceptsStampAtOrAboveLocalKey(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + ctx := context.Background() + + // Equal to local: eqdb is live at generation 1; a DROP stamped at + // generation 1 matches it exactly. + require.NoError(t, dm.CreateDatabase("eqdb")) + result := engine.Prepare(ctx, &PrepareRequest{ + TxnID: 2006, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: 9005, Logical: 1}, + Statements: []protocol.Statement{ + {Type: protocol.StatementDropDatabase, Database: "eqdb", DatabaseGeneration: 1}, + }, + }) + require.True(t, result.Success, "a stamp equal to the local key must be accepted: %s", result.Error) + + // Above local: hidb is tombstoned at generation 1; a CREATE stamped at + // generation 5 is well above the tombstone and must still be accepted. + require.NoError(t, dm.CreateDatabase("hidb")) + require.NoError(t, dm.DropDatabase("hidb")) + result = engine.Prepare(ctx, &PrepareRequest{ + TxnID: 2007, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: 9006, Logical: 1}, + Statements: []protocol.Statement{ + {Type: protocol.StatementCreateDatabase, Database: "hidb", DatabaseGeneration: 5}, + }, + }) + require.True(t, result.Success, "a stamp above the local key must be accepted: %s", result.Error) +} + // TestReplicationEngine_CommitNotFound verifies error when transaction doesn't exist func TestReplicationEngine_CommitNotFound(t *testing.T) { engine, dm, cleanup := setupTestReplicationEngine(t) diff --git a/db/schema_version_manager.go b/db/schema_version_manager.go index 30e82326..be2a32fd 100644 --- a/db/schema_version_manager.go +++ b/db/schema_version_manager.go @@ -1,163 +1,52 @@ package db -import ( - "fmt" - "sync" - "time" - - "github.com/rs/zerolog/log" -) - -// SchemaVersionManager tracks and manages schema versions per database -// Enables QUORUM-based DDL replication with automatic catch-up +import "fmt" + +// SchemaVersionManager reads the schema version of every user database from +// its own source of truth: the __marmot_schema_version table inside that +// database's SQLite file, cached on its *ReplicatedDatabase. It +// replaces the old pebble-backed counter, which could diverge from the DDL +// it counted across a crash. +// +// There is no increment/set method: a DDL's tx bumps the version itself +// (db/schema_version_table.go's bumpSchemaVersionInTx, called from the DDL's +// own commit or replay path), so every writer of the version is also the +// writer of the DDL it counts. This manager only reads. type SchemaVersionManager struct { - metaStore MetaStore - mu sync.RWMutex - // Cache of schema versions per database - versions map[string]uint64 + dbMgr *DatabaseManager } -// NewSchemaVersionManager creates a new schema version manager -func NewSchemaVersionManager(metaStore MetaStore) *SchemaVersionManager { - return &SchemaVersionManager{ - metaStore: metaStore, - versions: make(map[string]uint64), - } +// NewSchemaVersionManager creates a schema version manager reading through +// dbMgr's open databases. +func NewSchemaVersionManager(dbMgr *DatabaseManager) *SchemaVersionManager { + return &SchemaVersionManager{dbMgr: dbMgr} } -// GetSchemaVersion returns the current schema version for a database +// GetSchemaVersion returns database's current schema version. The system +// database has no version and always returns 0. An unknown database is an +// error: unlike the old cache, there is no version to fall back to. func (svm *SchemaVersionManager) GetSchemaVersion(database string) (uint64, error) { - svm.mu.RLock() - if version, ok := svm.versions[database]; ok { - svm.mu.RUnlock() - return version, nil - } - svm.mu.RUnlock() - - // Not in cache, query MetaStore - svm.mu.Lock() - defer svm.mu.Unlock() - - // Double-check after acquiring write lock - if version, ok := svm.versions[database]; ok { - return version, nil + if database == "" || database == SystemDatabaseName { + return 0, nil } - - version, err := svm.metaStore.GetSchemaVersion(database) + mdb, err := svm.dbMgr.GetDatabase(database) if err != nil { return 0, fmt.Errorf("failed to get schema version: %w", err) } - - svm.versions[database] = uint64(version) - return uint64(version), nil -} - -// IncrementSchemaVersion increments the schema version after a DDL operation -// Returns the NEW schema version -func (svm *SchemaVersionManager) IncrementSchemaVersion(database string, ddlSQL string, txnID uint64) (uint64, error) { - svm.mu.Lock() - defer svm.mu.Unlock() - - // Get current version from MetaStore - currentVersion, err := svm.metaStore.GetSchemaVersion(database) - if err != nil { - return 0, fmt.Errorf("failed to get current schema version: %w", err) - } - - newVersion := currentVersion + 1 - - // Update schema version in MetaStore - err = svm.metaStore.UpdateSchemaVersion(database, newVersion, ddlSQL, txnID) - if err != nil { - return 0, fmt.Errorf("failed to increment schema version: %w", err) - } - - // Update cache - svm.versions[database] = uint64(newVersion) - - log.Info(). - Str("database", database). - Int64("old_version", currentVersion). - Int64("new_version", newVersion). - Uint64("txn_id", txnID). - Msg("Schema version incremented") - - return uint64(newVersion), nil -} - -// SetSchemaVersion explicitly sets the schema version (used during catch-up) -func (svm *SchemaVersionManager) SetSchemaVersion(database string, version uint64, ddlSQL string, txnID uint64) error { - svm.mu.Lock() - defer svm.mu.Unlock() - - err := svm.metaStore.UpdateSchemaVersion(database, int64(version), ddlSQL, txnID) - if err != nil { - return fmt.Errorf("failed to set schema version: %w", err) - } - - // Update cache - svm.versions[database] = version - - log.Info(). - Str("database", database). - Uint64("version", version). - Uint64("txn_id", txnID). - Msg("Schema version set") - - return nil + return mdb.SchemaVersion(), nil } -// WaitForSchemaVersion waits until the local schema version reaches the target -// Returns error if timeout is exceeded -// Used by delta sync to ensure DDL is applied before DML -func (svm *SchemaVersionManager) WaitForSchemaVersion(database string, targetVersion uint64, timeout time.Duration) error { - deadline := time.Now().Add(timeout) - ticker := time.NewTicker(100 * time.Millisecond) - defer ticker.Stop() - - for { - currentVersion, err := svm.GetSchemaVersion(database) +// GetAllSchemaVersions returns the current schema version of every user +// database (never the system database, which ListDatabases already excludes). +func (svm *SchemaVersionManager) GetAllSchemaVersions() (map[string]uint64, error) { + names := svm.dbMgr.ListDatabases() + result := make(map[string]uint64, len(names)) + for _, name := range names { + mdb, err := svm.dbMgr.GetDatabase(name) if err != nil { - return fmt.Errorf("failed to get schema version: %w", err) - } - - if currentVersion >= targetVersion { - return nil + continue // dropped between ListDatabases and GetDatabase; skip it } - - if time.Now().After(deadline) { - return fmt.Errorf("timeout waiting for schema version %d (current: %d)", targetVersion, currentVersion) - } - - <-ticker.C + result[name] = mdb.SchemaVersion() } -} - -// GetAllSchemaVersions returns a map of all database schema versions -// Used by gossip protocol to exchange schema version info -func (svm *SchemaVersionManager) GetAllSchemaVersions() (map[string]uint64, error) { - svm.mu.Lock() - defer svm.mu.Unlock() - - versions, err := svm.metaStore.GetAllSchemaVersions() - if err != nil { - return nil, fmt.Errorf("failed to query schema versions: %w", err) - } - - result := make(map[string]uint64) - for dbName, version := range versions { - result[dbName] = uint64(version) - // Update cache - svm.versions[dbName] = uint64(version) - } - return result, nil } - -// InvalidateCache clears the cached schema versions -// Useful after database deletion or during testing -func (svm *SchemaVersionManager) InvalidateCache(database string) { - svm.mu.Lock() - defer svm.mu.Unlock() - delete(svm.versions, database) -} diff --git a/db/schema_version_restore.go b/db/schema_version_restore.go deleted file mode 100644 index e820e732..00000000 --- a/db/schema_version_restore.go +++ /dev/null @@ -1,50 +0,0 @@ -package db - -import ( - "fmt" - - "github.com/rs/zerolog/log" -) - -// RestoreSchemaVersions records schema versions carried by a snapshot into the -// system MetaStore, where schema versions are tracked. -// -// Snapshots transfer SQLite files only, so a receiver that does not restore these -// keeps reporting version 0 and then refuses every transaction stamped with a -// higher required schema version - including plain DML. That state never heals on -// its own, because the only way to advance the version is committing a DDL -// transaction the node can no longer take part in. -// -// A database is never moved backwards: the node may have applied DDL after the -// snapshot was taken. -func RestoreSchemaVersions(metaStore MetaStore, versions map[string]uint64) error { - if metaStore == nil || len(versions) == 0 { - return nil - } - - for database, version := range versions { - if database == "" || version == 0 { - continue - } - - current, err := metaStore.GetSchemaVersion(database) - if err != nil { - return fmt.Errorf("failed to read schema version for %s: %w", database, err) - } - if current >= int64(version) { - continue - } - - if err := metaStore.UpdateSchemaVersion(database, int64(version), "", 0); err != nil { - return fmt.Errorf("failed to restore schema version for %s: %w", database, err) - } - - log.Info(). - Str("database", database). - Int64("previous_version", current). - Uint64("restored_version", version). - Msg("Restored schema version from snapshot") - } - - return nil -} diff --git a/db/schema_version_restore_test.go b/db/schema_version_restore_test.go deleted file mode 100644 index 9eca5832..00000000 --- a/db/schema_version_restore_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package db - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func newRestoreTestMetaStore(t *testing.T) MetaStore { - t.Helper() - - metaStore, err := NewPebbleMetaStore(t.TempDir(), PebbleMetaStoreOptions{ - CacheSizeMB: 16, - MemTableSizeMB: 16, - MemTableCount: 2, - L0CompactionThreshold: 4, - L0StopWrites: 12, - }) - require.NoError(t, err) - t.Cleanup(func() { metaStore.Close() }) - return metaStore -} - -func TestRestoreSchemaVersions(t *testing.T) { - t.Parallel() - metaStore := newRestoreTestMetaStore(t) - - require.NoError(t, RestoreSchemaVersions(metaStore, map[string]uint64{"appdb": 17, "otherdb": 3})) - - version, err := metaStore.GetSchemaVersion("appdb") - require.NoError(t, err) - require.Equal(t, int64(17), version) - - version, err = metaStore.GetSchemaVersion("otherdb") - require.NoError(t, err) - require.Equal(t, int64(3), version) -} - -// A node that applied DDL after the snapshot was taken must not be rewound: that -// would make it refuse transactions it is actually able to serve. -func TestRestoreSchemaVersionsNeverGoesBackwards(t *testing.T) { - t.Parallel() - metaStore := newRestoreTestMetaStore(t) - - require.NoError(t, metaStore.UpdateSchemaVersion("appdb", 20, "", 0)) - require.NoError(t, RestoreSchemaVersions(metaStore, map[string]uint64{"appdb": 17})) - - version, err := metaStore.GetSchemaVersion("appdb") - require.NoError(t, err) - require.Equal(t, int64(20), version) -} - -func TestRestoreSchemaVersionsIgnoresEmptyInput(t *testing.T) { - t.Parallel() - metaStore := newRestoreTestMetaStore(t) - - require.NoError(t, RestoreSchemaVersions(metaStore, nil)) - require.NoError(t, RestoreSchemaVersions(nil, map[string]uint64{"appdb": 4})) - require.NoError(t, RestoreSchemaVersions(metaStore, map[string]uint64{"": 4, "appdb": 0})) - - versions, err := metaStore.GetAllSchemaVersions() - require.NoError(t, err) - require.Empty(t, versions) -} - -// The restored version must be what a subsequent PREPARE compares against, which -// is what SchemaVersionManager reads. -func TestRestoreSchemaVersionsVisibleToSchemaVersionManager(t *testing.T) { - t.Parallel() - metaStore := newRestoreTestMetaStore(t) - - require.NoError(t, RestoreSchemaVersions(metaStore, map[string]uint64{"appdb": 17})) - - version, err := NewSchemaVersionManager(metaStore).GetSchemaVersion("appdb") - require.NoError(t, err) - require.Equal(t, uint64(17), version, "a restored node must not report itself behind the cluster") -} diff --git a/db/schema_version_table.go b/db/schema_version_table.go new file mode 100644 index 00000000..a33acc54 --- /dev/null +++ b/db/schema_version_table.go @@ -0,0 +1,154 @@ +package db + +import ( + "database/sql" + "fmt" +) + +// schemaVersionTableDDL creates __marmot_schema_version, the durable home of +// a user database's schema version. The +// single row (id=1) is bumped inside the same SQLite transaction as the DDL +// that changed the schema, so the version and the DDL it counts can never +// diverge across a crash - unlike the old pebble-stored counter, which lived +// in a different file from the DDL it counted. +const schemaVersionTableDDL = ` +CREATE TABLE IF NOT EXISTS __marmot_schema_version ( + id INTEGER PRIMARY KEY CHECK(id = 1), + version INTEGER NOT NULL +)` + +// sqliteTableExists reports whether table exists in db's schema. +func sqliteTableExists(db interface { + QueryRow(query string, args ...any) *sql.Row +}, table string) (bool, error) { + var name string + err := db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name = ?", table).Scan(&name) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, fmt.Errorf("check table %s: %w", table, err) + } + return true, nil +} + +// readSchemaVersionRow reads the single seed row of an existing +// __marmot_schema_version table, reporting ok=false (no error) when the table +// exists but the row does not - the row-less state a crash between CREATE and +// the seeding INSERT can leave behind (see ensureSchemaVersionTable). +func readSchemaVersionRow(q rowQuerier) (uint64, bool, error) { + var v uint64 + err := q.QueryRow("SELECT version FROM __marmot_schema_version WHERE id = 1").Scan(&v) + if err == sql.ErrNoRows { + return 0, false, nil + } + if err != nil { + return 0, false, fmt.Errorf("read schema version: %w", err) + } + return v, true, nil +} + +// computeInitialSchemaVersion returns the value to seed +// __marmot_schema_version's single row with, either on first creation or when +// repairing a row-less table: +// max(0, legacyRead(dbName)) from the retiring pebble-stored counter, but +// ONLY when filePreexisted is true - i.e. this SQLite file already carried +// __marmot_applied_txn before this open, so it predates this feature and may +// carry DDL the pebble store already counted. A database file that did not +// exist before this open (a fresh CREATE DATABASE) always starts at 0, even +// if a stale pebble entry exists under the same name from a prior, dropped +// incarnation. +func computeInitialSchemaVersion(dbName string, filePreexisted bool, legacyRead func(dbName string) (int64, error)) (uint64, error) { + if !filePreexisted || legacyRead == nil || dbName == "" { + return 0, nil + } + legacy, err := legacyRead(dbName) + if err != nil { + return 0, fmt.Errorf("read legacy schema version for %s: %w", dbName, err) + } + if legacy > 0 { + return uint64(legacy), nil + } + return 0, nil +} + +// ensureSchemaVersionTable creates __marmot_schema_version in a database file +// if it is not already there, repairs it if a prior open left it without its +// seed row, and returns the version to cache. +// +// Atomicity: the CREATE TABLE and the seeding INSERT run +// together inside one SQLite transaction, and the INSERT is OR IGNORE, so +// this function is idempotent whether the table is missing entirely, already +// exists with its row, or - the state a crash between the two statements used +// to leave a pre-fix binary in - exists without a row. Every open path +// (NewReplicatedDatabase) calls this before any query that reads the cached +// version, so a row-less table can never make a later open fail with +// "read schema version: sql: no rows in result set". +// +// Migration rule: see computeInitialSchemaVersion. Note for a rolling +// upgrade: restoring a database from a snapshot taken on an older release +// leaves the +// restored file with no __marmot_schema_version table (the source predates +// it), so this function's migration reads THIS node's own legacy pebble +// value, never the source's - the source node is a different pebble store. +// The documented upgrade procedure is to upgrade every node (so every node's +// legacy value is populated and consistent) before issuing the DDL that would +// make the versions matter; that keeps every node's migrated value equal. +func ensureSchemaVersionTable(db *sql.DB, dbName string, filePreexisted bool, legacyRead func(dbName string) (int64, error)) (uint64, error) { + existed, err := sqliteTableExists(db, "__marmot_schema_version") + if err != nil { + return 0, err + } + if existed { + if v, ok, err := readSchemaVersionRow(db); err != nil { + return 0, err + } else if ok { + return v, nil + } + // Row-less table: repair it below with the same seed rule as a fresh + // creation. + } + + initial, err := computeInitialSchemaVersion(dbName, filePreexisted, legacyRead) + if err != nil { + return 0, err + } + + tx, err := db.Begin() + if err != nil { + return 0, fmt.Errorf("begin schema version setup: %w", err) + } + defer tx.Rollback() + + if _, err := tx.Exec(schemaVersionTableDDL); err != nil { + return 0, fmt.Errorf("create schema version table: %w", err) + } + if _, err := tx.Exec("INSERT OR IGNORE INTO __marmot_schema_version (id, version) VALUES (1, ?)", initial); err != nil { + return 0, fmt.Errorf("seed schema version: %w", err) + } + v, ok, err := readSchemaVersionRow(tx) + if err != nil { + return 0, err + } + if !ok { + return 0, fmt.Errorf("seed schema version: row missing immediately after insert") + } + if err := tx.Commit(); err != nil { + return 0, fmt.Errorf("commit schema version setup: %w", err) + } + return v, nil +} + +// bumpSchemaVersionInTx increments the user database's schema version by one +// inside tx and returns the new value. The caller commits tx together with +// the DDL that earned the bump, so the two can never diverge. +func bumpSchemaVersionInTx(tx *sql.Tx) (uint64, error) { + if _, err := tx.Exec("UPDATE __marmot_schema_version SET version = version + 1 WHERE id = 1"); err != nil { + return 0, fmt.Errorf("bump schema version: %w", err) + } + var v uint64 + if err := tx.QueryRow("SELECT version FROM __marmot_schema_version WHERE id = 1").Scan(&v); err != nil { + return 0, fmt.Errorf("read bumped schema version: %w", err) + } + return v, nil +} diff --git a/db/schema_version_table_test.go b/db/schema_version_table_test.go new file mode 100644 index 00000000..b457f1da --- /dev/null +++ b/db/schema_version_table_test.go @@ -0,0 +1,215 @@ +package db + +import ( + "database/sql" + "path/filepath" + "testing" + "time" + + "github.com/maxpert/marmot/encoding" + "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" +) + +// seedLegacySchemaVersionForTest writes directly into store's pebble +// representation, standing in for what an older release's now-removed +// UpdateSchemaVersion write path would have left behind. There is no +// production writer of this key any more, so a test simulating a +// pre-upgrade database must poke it directly. +func seedLegacySchemaVersionForTest(t *testing.T, store *PebbleMetaStore, dbName string, version int64) { + t.Helper() + rec := pebbleSchemaVersionRecord{Version: version, UpdatedAt: time.Now().UnixNano()} + data, err := encoding.Marshal(rec) + require.NoError(t, err) + require.NoError(t, store.db.Set(pebbleSchemaKey(dbName), data, store.syncWrite)) +} + +// TestSchemaVersionTable_PersistsAcrossCloseReopen pins that +// __marmot_schema_version, not any in-memory cache, is the durable source: +// a bumped version must survive a full close and reopen of the database. +func TestSchemaVersionTable_PersistsAcrossCloseReopen(t *testing.T) { + tmpDir := t.TempDir() + dbPath := filepath.Join(tmpDir, "appdb.db") + + metaStore, err := NewMetaStore(dbPath) + require.NoError(t, err) + mdb, err := NewReplicatedDatabase(dbPath, 1, hlc.NewClock(1), metaStore, WithDatabaseName("appdb")) + require.NoError(t, err) + require.Equal(t, uint64(0), mdb.SchemaVersion()) + + tx, err := mdb.GetWriteDB().Begin() + require.NoError(t, err) + v, err := bumpSchemaVersionInTx(tx) + require.NoError(t, err) + require.Equal(t, uint64(1), v) + require.NoError(t, tx.Commit()) + mdb.schemaVersion.Store(v) + require.NoError(t, mdb.Close()) + + metaStore2, err := NewMetaStore(dbPath) + require.NoError(t, err) + mdb2, err := NewReplicatedDatabase(dbPath, 1, hlc.NewClock(1), metaStore2, WithDatabaseName("appdb")) + require.NoError(t, err) + defer mdb2.Close() + require.Equal(t, uint64(1), mdb2.SchemaVersion(), "schema version must persist across close/reopen") +} + +// TestSchemaVersionTable_MigratesLegacyOnlyForPreexistingFile pins the +// migration rule precisely: a SQLite file that already carried +// __marmot_applied_txn before this open (it predates the table) seeds its new +// __marmot_schema_version from the retiring pebble-stored counter. +func TestSchemaVersionTable_MigratesLegacyOnlyForPreexistingFile(t *testing.T) { + tmpDir := t.TempDir() + legacy, err := NewPebbleMetaStore(filepath.Join(tmpDir, "legacy_meta.pebble"), DefaultPebbleOptions()) + require.NoError(t, err) + defer legacy.Close() + seedLegacySchemaVersionForTest(t, legacy, "appdb", 7) + + dbPath := filepath.Join(tmpDir, "appdb.db") + raw, err := sql.Open(SQLiteDriverName, dbPath) + require.NoError(t, err) + require.NoError(t, ensureAppliedTxnTable(raw)) + require.NoError(t, raw.Close()) + + metaStore, err := NewMetaStore(dbPath) + require.NoError(t, err) + mdb, err := NewReplicatedDatabase(dbPath, 1, hlc.NewClock(1), metaStore, + WithDatabaseName("appdb"), WithLegacySchemaVersionSource(legacy.GetSchemaVersion)) + require.NoError(t, err) + defer mdb.Close() + + require.Equal(t, uint64(7), mdb.SchemaVersion(), "a pre-existing file must migrate the legacy pebble version") +} + +// TestSchemaVersionTable_FreshDatabaseIgnoresStaleLegacyEntry pins the other +// half of the migration rule: a database file that did not exist before this +// open (a fresh CREATE DATABASE) starts at 0 even when a legacy pebble entry +// exists under the same name, left over from an earlier, dropped incarnation. +func TestSchemaVersionTable_FreshDatabaseIgnoresStaleLegacyEntry(t *testing.T) { + tmpDir := t.TempDir() + legacy, err := NewPebbleMetaStore(filepath.Join(tmpDir, "legacy_meta.pebble"), DefaultPebbleOptions()) + require.NoError(t, err) + defer legacy.Close() + seedLegacySchemaVersionForTest(t, legacy, "appdb", 7) + + dbPath := filepath.Join(tmpDir, "appdb.db") // never opened before: genuinely fresh + + metaStore, err := NewMetaStore(dbPath) + require.NoError(t, err) + mdb, err := NewReplicatedDatabase(dbPath, 1, hlc.NewClock(1), metaStore, + WithDatabaseName("appdb"), WithLegacySchemaVersionSource(legacy.GetSchemaVersion)) + require.NoError(t, err) + defer mdb.Close() + + require.Equal(t, uint64(0), mdb.SchemaVersion(), "a fresh database must not inherit a stale legacy value") +} + +// TestSchemaVersionTable_EveryOpenPathCreatesTheTable: a database file +// opened without a name - the system database, or any caller of +// NewReplicatedDatabase that passes no options - still gets the table, at +// version 0, so a DDL commit on it bumps the version instead of failing +// after its peers committed (a partial commit). +// +// Mutation: create the table only for a named database. "a database opened +// without a name has no schema version table" fires. +func TestSchemaVersionTable_EveryOpenPathCreatesTheTable(t *testing.T) { + tmpDir := t.TempDir() + dbPath := filepath.Join(tmpDir, "unnamed.db") + + metaStore, err := NewMetaStore(dbPath) + require.NoError(t, err) + mdb, err := NewReplicatedDatabase(dbPath, 1, hlc.NewClock(1), metaStore) + require.NoError(t, err) + defer mdb.Close() + + existed, err := sqliteTableExists(mdb.GetWriteDB(), "__marmot_schema_version") + require.NoError(t, err) + require.True(t, existed, "a database opened without a name has no schema version table") + require.Equal(t, uint64(0), mdb.SchemaVersion()) + + tx, err := mdb.GetWriteDB().Begin() + require.NoError(t, err) + v, err := bumpSchemaVersionInTx(tx) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + require.Equal(t, uint64(1), v) +} + +// TestSchemaVersionTable_ManagerPathsCreateTheTable: every DatabaseManager +// path that opens a user database - CreateDatabase, a restart's registry +// load, and ImportExistingDatabases - leaves the file with the table. +func TestSchemaVersionTable_ManagerPathsCreateTheTable(t *testing.T) { + dir := t.TempDir() + dm, err := NewDatabaseManager(dir, 1, hlc.NewClock(1)) + require.NoError(t, err) + require.NoError(t, dm.CreateDatabase("created")) + requireSchemaVersionTable(t, dm, "created", "CreateDatabase") + sys, err := dm.GetDatabase(SystemDatabaseName) + require.NoError(t, err) + existed, err := sqliteTableExists(sys.GetWriteDB(), "__marmot_schema_version") + require.NoError(t, err) + require.True(t, existed, "the system database has no schema version table") + require.NoError(t, dm.Close()) + + dm, err = NewDatabaseManager(dir, 1, hlc.NewClock(1)) + require.NoError(t, err) + defer dm.Close() + requireSchemaVersionTable(t, dm, "created", "registry load on restart") + requireSchemaVersionTable(t, dm, DefaultDatabaseName, "default database") +} + +// TestSchemaVersionTable_RowlessTableIsRepairedAtOpen: a crash between the CREATE TABLE and the seeding INSERT left +// __marmot_schema_version present but empty. Before the fix, opening such a +// file failed every time with "read schema version: sql: no rows in result +// set"; ensureSchemaVersionTable must instead repair the row and open +// normally. +func TestSchemaVersionTable_RowlessTableIsRepairedAtOpen(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "x.db") + sqlDB, err := sql.Open(SQLiteDriverName, dbPath) + require.NoError(t, err) + defer sqlDB.Close() + + // State after a crash right after CREATE TABLE, before the INSERT. + _, err = sqlDB.Exec(schemaVersionTableDDL) + require.NoError(t, err) + + v, err := ensureSchemaVersionTable(sqlDB, "x", true, nil) + require.NoError(t, err, "open must survive a crash between CREATE TABLE and the seeding INSERT") + require.Equal(t, uint64(0), v) + + // A second open must also see the repaired row, not repair it again. + v, err = ensureSchemaVersionTable(sqlDB, "x", true, nil) + require.NoError(t, err) + require.Equal(t, uint64(0), v) +} + +// TestAdvanceSchemaVersion_NeverLowersCache pins that two post-commit +// stores of the cached schema version finishing out of order (replay versus +// the 2PC non-DML commit callback, both outside the writer lock) must never +// regress the cache below a value it already holds. +func TestAdvanceSchemaVersion_NeverLowersCache(t *testing.T) { + tmpDir := t.TempDir() + metaStore, err := NewMetaStore(filepath.Join(tmpDir, "appdb.db")) + require.NoError(t, err) + mdb, err := NewReplicatedDatabase(filepath.Join(tmpDir, "appdb.db"), 1, hlc.NewClock(1), metaStore, WithDatabaseName("appdb")) + require.NoError(t, err) + defer mdb.Close() + + mdb.advanceSchemaVersion(5) + require.Equal(t, uint64(5), mdb.SchemaVersion()) + + mdb.advanceSchemaVersion(3) // out-of-order, lower: must not regress + require.Equal(t, uint64(5), mdb.SchemaVersion()) + + mdb.advanceSchemaVersion(7) + require.Equal(t, uint64(7), mdb.SchemaVersion()) +} + +func requireSchemaVersionTable(t *testing.T, dm *DatabaseManager, name, path string) { + t.Helper() + mdb, err := dm.GetDatabase(name) + require.NoError(t, err) + existed, err := sqliteTableExists(mdb.GetWriteDB(), "__marmot_schema_version") + require.NoError(t, err) + require.True(t, existed, "%s: %s has no schema version table", path, name) +} diff --git a/db/snapshot/grpc_adapter.go b/db/snapshot/grpc_adapter.go index 8e626cb9..3b7547ae 100644 --- a/db/snapshot/grpc_adapter.go +++ b/db/snapshot/grpc_adapter.go @@ -12,6 +12,8 @@ type GRPCChunk interface { GetData() []byte GetChecksum() string GetIsLastForFile() bool + GetFileSha256() string + GetFileSizeBytes() int64 } // GRPCStream represents a gRPC streaming client that can receive chunks @@ -43,6 +45,8 @@ func (a *GRPCChunkAdapter) Recv() (*Chunk, error) { Data: grpcChunk.GetData(), MD5Checksum: grpcChunk.GetChecksum(), IsLastForFile: grpcChunk.GetIsLastForFile(), + FileSHA256: grpcChunk.GetFileSha256(), + FileSizeBytes: grpcChunk.GetFileSizeBytes(), }, nil } @@ -76,6 +80,8 @@ func (a *GenericStreamAdapter[T]) Recv() (*Chunk, error) { Data: grpcChunk.GetData(), MD5Checksum: grpcChunk.GetChecksum(), IsLastForFile: grpcChunk.GetIsLastForFile(), + FileSHA256: grpcChunk.GetFileSha256(), + FileSizeBytes: grpcChunk.GetFileSizeBytes(), }, nil } diff --git a/db/snapshot/restorer.go b/db/snapshot/restorer.go index b5a71c72..99e4cc01 100644 --- a/db/snapshot/restorer.go +++ b/db/snapshot/restorer.go @@ -118,13 +118,16 @@ func (r *Restorer) RestoreFromStream( } // STEP 2: Download all chunks to temp directory - if err := r.downloadToTemp(stream, tempDir); err != nil { + streamed, err := r.downloadToTemp(stream, tempDir) + if err != nil { r.setError(err) return err } - // STEP 3: Verify integrity + // STEP 3: Verify integrity, against what the stream itself reported for + // each file when it reported it (withStreamedManifest). r.setPhase(PhaseVerifying) + files = withStreamedManifest(files, streamed) if err := VerifyFileList(tempDir, files); err != nil { r.setError(err) return fmt.Errorf("snapshot integrity verification failed: %w", err) @@ -142,8 +145,37 @@ func (r *Restorer) RestoreFromStream( return nil } -// downloadToTemp downloads all chunks from stream to temp directory -func (r *Restorer) downloadToTemp(stream ChunkReceiver, tempDir string) error { +// streamedFile is what a sender reported, on a file's last chunk, about the +// whole file it streamed. +type streamedFile struct { + sha256 string + sizeBytes int64 +} + +// withStreamedManifest returns files with each entry's checksum and size +// replaced by what the stream reported for that file, where it reported +// them. The stream's values describe exactly the bytes it sent; the list a +// caller got from GetSnapshotInfo describes a separate, earlier snapshot of +// a database that may have been written since, so under write load it can +// never match. An entry the stream reported nothing for (an older sender) +// keeps the caller's values. +func withStreamedManifest(files []DatabaseFileInfo, streamed map[string]streamedFile) []DatabaseFileInfo { + out := make([]DatabaseFileInfo, len(files)) + for i, f := range files { + if sf, ok := streamed[f.Filename]; ok { + f.SHA256Checksum = sf.sha256 + f.SizeBytes = sf.sizeBytes + } + out[i] = f + } + return out +} + +// downloadToTemp downloads all chunks from stream to temp directory and +// returns, per filename, the whole-file checksum and size the stream +// reported on that file's last chunk. +func (r *Restorer) downloadToTemp(stream ChunkReceiver, tempDir string) (map[string]streamedFile, error) { + streamed := make(map[string]streamedFile) openFiles := make(map[string]*os.File) defer func() { for _, f := range openFiles { @@ -160,13 +192,16 @@ func (r *Restorer) downloadToTemp(stream ChunkReceiver, tempDir string) error { break } if err != nil { - return fmt.Errorf("snapshot stream error: %w", err) + return nil, fmt.Errorf("snapshot stream error: %w", err) } // Process single chunk bytesWritten, fileCompleted, err := r.processChunk(chunk, tempDir, openFiles) if err != nil { - return err + return nil, err + } + if chunk.IsLastForFile && chunk.FileSHA256 != "" { + streamed[chunk.Filename] = streamedFile{sha256: chunk.FileSHA256, sizeBytes: chunk.FileSizeBytes} } // Update progress @@ -185,7 +220,7 @@ func (r *Restorer) downloadToTemp(stream ChunkReceiver, tempDir string) error { Str("temp_dir", tempDir). Msg("Snapshot downloaded to temp directory") - return nil + return streamed, nil } // processChunk handles a single chunk: verify, write, and optionally close file diff --git a/db/snapshot/restorer_test.go b/db/snapshot/restorer_test.go index 09eff25a..a716d222 100644 --- a/db/snapshot/restorer_test.go +++ b/db/snapshot/restorer_test.go @@ -572,3 +572,35 @@ func TestRestorer_SystemDBMergeRunsBeforeTheSwap(t *testing.T) { t.Fatalf("the system database was installed without the merge: %q", got) } } + +// TestRestorer_VerifiesAgainstTheStreamedManifest: GetSnapshotInfo's list +// describes an earlier snapshot than the one StreamSnapshot sends, so under +// write load its checksum and size are stale. A restore verifies each file +// against the checksum and size the stream reported with the file itself, +// and still refuses bytes that do not match those. +// +// Mutation: ignore the streamed manifest (withStreamedManifest returns files +// unchanged). "a file written after the info call was refused" fires. +func TestRestorer_VerifiesAgainstTheStreamedManifest(t *testing.T) { + content := []byte("SQLite format 3\x00written after the info call") + stale := []DatabaseFileInfo{{ + Name: "test", + Filename: "databases/test.db", + SizeBytes: 7, + SHA256Checksum: calculateSHA256([]byte("earlier")), + }} + + chunks := createTestChunksWithMD5("databases/test.db", content) + chunks[0].FileSHA256 = calculateSHA256(content) + chunks[0].FileSizeBytes = int64(len(content)) + if err := NewRestorer(t.TempDir(), nil).RestoreFromStream(&mockChunkStream{chunks: chunks}, stale); err != nil { + t.Fatalf("a file written after the info call was refused: %v", err) + } + + corrupt := createTestChunksWithMD5("databases/test.db", content) + corrupt[0].FileSHA256 = calculateSHA256([]byte("other bytes")) + corrupt[0].FileSizeBytes = int64(len(content)) + if err := NewRestorer(t.TempDir(), nil).RestoreFromStream(&mockChunkStream{chunks: corrupt}, stale); err == nil { + t.Fatal("bytes that do not match the streamed checksum were accepted") + } +} diff --git a/db/snapshot/snapshot.go b/db/snapshot/snapshot.go index 71114234..1e7cf7a8 100644 --- a/db/snapshot/snapshot.go +++ b/db/snapshot/snapshot.go @@ -92,6 +92,10 @@ type Chunk struct { Data []byte MD5Checksum string IsLastForFile bool + // FileSHA256 and FileSizeBytes, set on a file's last chunk by a sender + // that reports them, describe the whole file exactly as streamed. + FileSHA256 string + FileSizeBytes int64 } // ChunkReceiver abstracts the gRPC stream for testability diff --git a/db/stale_gc_commit_race_test.go b/db/stale_gc_commit_race_test.go new file mode 100644 index 00000000..1f82703f --- /dev/null +++ b/db/stale_gc_commit_race_test.go @@ -0,0 +1,129 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "context" + "database/sql" + "fmt" + "sync" + "testing" + + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// staleGCRaceTxns is how many prepared transactions the concurrent race test +// sets the stale-transaction GC and the local commit path loose on. +const staleGCRaceTxns = 200 + +// TestStaleGCAndLocalCommitNeverCommitAnEmptyTransaction runs the +// stale-transaction GC and the log puller's local commit path +// (DatabaseManager.CommitLocallyPrepared) concurrently over many prepared +// transactions, every one of them stale. Each transaction must end either +// COMMITTED with its row applied, or not committed with its row absent - +// never COMMITTED without its row, which the cursor would then pass. +// +// Mutation (both halves of the fix): take the commit guard out of +// TransactionManager.cleanupStaleTransactions and make verifyPreparedPayload +// return nil. "committed without its row" fires. +func TestStaleGCAndLocalCommitNeverCommitAnEmptyTransaction(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE log (id INTEGER PRIMARY KEY, v TEXT)") + tm := mdb.GetTransactionManager() + tm.StopGarbageCollection() + tm.heartbeatTimeout = 0 + + const firstTxn = 20000 + for i := 0; i < staleGCRaceTxns; i++ { + txnID := uint64(firstTxn + i) + prep := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, + NodeID: 2, + StartTS: hlc.Timestamp{WallTime: int64(txnID), NodeID: 2}, + Database: "testdb", + Statements: []protocol.Statement{dmlLogInsertStatement("testdb", fmt.Sprintf("log:%d", txnID), int64(txnID), "race")}, + }) + require.True(t, prep.Success, "prepare %d: %s", txnID, prep.Error) + } + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; i < staleGCRaceTxns; i++ { + _, _ = tm.cleanupStaleTransactions() + } + }() + go func() { + defer wg.Done() + for i := 0; i < staleGCRaceTxns; i++ { + _ = dm.CommitLocallyPrepared("testdb", uint64(firstTxn+i)) + } + }() + wg.Wait() + + for i := 0; i < staleGCRaceTxns; i++ { + txnID := uint64(firstTxn + i) + rec, err := mdb.GetMetaStore().GetTransaction(txnID) + require.NoError(t, err) + var v string + rowErr := mdb.GetReadDB().QueryRow("SELECT v FROM log WHERE id = ?", int64(txnID)).Scan(&v) + committed := rec != nil && rec.Status == TxnStatusCommitted + if committed { + require.NoError(t, rowErr, "txn %d committed without its row", txnID) + } else { + require.ErrorIs(t, rowErr, sql.ErrNoRows, "txn %d not committed but its row was applied", txnID) + } + } +} + +// TestCommitRPCRefusesATransactionWhosePreparedPayloadIsGone pins the COMMIT +// RPC half of the prepared-payload check: a COMMIT carrying no statements +// (as the log puller's local commit, or a coordinator's minimal COMMIT, +// does) for a transaction whose captured rows, or DDL intent, were deleted +// after its durable prepare must be refused, not ACKed with nothing applied. +// +// Mutation: make verifyPreparedPayload return nil. Both COMMITs are ACKed. +func TestCommitRPCRefusesATransactionWhosePreparedPayloadIsGone(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE log (id INTEGER PRIMARY KEY, v TEXT)") + mdb.GetTransactionManager().StopGarbageCollection() + ms := mdb.GetMetaStore() + + const dmlTxn, ddlTxn = 21000, 21001 + prep := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: dmlTxn, NodeID: 2, StartTS: hlc.Timestamp{WallTime: dmlTxn, NodeID: 2}, Database: "testdb", + Statements: []protocol.Statement{dmlLogInsertStatement("testdb", "log:1", 1, "gone")}, + }) + require.True(t, prep.Success, prep.Error) + require.NoError(t, ms.DeleteIntentsByTxn(dmlTxn)) + require.NoError(t, ms.DeleteIntentEntries(dmlTxn)) + + res := engine.Commit(context.Background(), &CommitRequest{TxnID: dmlTxn, Database: "testdb"}) + require.False(t, res.Success, "a COMMIT whose prepared rows are gone was ACKed") + require.Contains(t, res.Error, ErrPreparedRowsMissing.Error()) + + prep = engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: ddlTxn, NodeID: 2, StartTS: hlc.Timestamp{WallTime: ddlTxn, NodeID: 2}, Database: "testdb", + Statements: []protocol.Statement{{Type: protocol.StatementDDL, Database: "testdb", TableName: "extra", + SQL: "CREATE TABLE extra (id INTEGER PRIMARY KEY)"}}, + }) + require.True(t, prep.Success, prep.Error) + require.NoError(t, ms.DeleteIntentsByTxn(ddlTxn)) + + res = engine.Commit(context.Background(), &CommitRequest{TxnID: ddlTxn, Database: "testdb"}) + require.False(t, res.Success, "a COMMIT whose prepared DDL intent is gone was ACKed") + require.Contains(t, res.Error, ErrPreparedRowsMissing.Error()) + + for _, txnID := range []uint64{dmlTxn, ddlTxn} { + rec, err := ms.GetTransaction(txnID) + require.NoError(t, err) + require.NotNil(t, rec) + require.Equal(t, TxnStatusPending, rec.Status, "a refused COMMIT must leave txn %d PENDING", txnID) + } +} diff --git a/db/test_helpers.go b/db/test_helpers.go index ed4f540b..1fef8a06 100644 --- a/db/test_helpers.go +++ b/db/test_helpers.go @@ -68,6 +68,13 @@ func openTestDBWithMeta(t *testing.T, dbPath string) *testDBWithMetaStore { db.Close() t.Fatalf("Failed to create applied transaction marker table: %v", err) } + // applyNonDMLIntents bumps __marmot_schema_version for any real DDL + // intent inside its one SQLite tx, same as NewReplicatedDatabase would + // have created it; this raw test *sql.DB needs it too. + if _, err := ensureSchemaVersionTable(db, "", false, nil); err != nil { + db.Close() + t.Fatalf("Failed to create schema version table: %v", err) + } metaPath := strings.TrimSuffix(dbPath, ".db") + "_meta.pebble" pebbleStore, err := NewPebbleMetaStore(metaPath, PebbleMetaStoreOptions{ diff --git a/db/transaction.go b/db/transaction.go index 608128c3..1873f4f6 100644 --- a/db/transaction.go +++ b/db/transaction.go @@ -33,17 +33,12 @@ type Transaction struct { mu sync.RWMutex } -// MinAppliedTxnIDFunc returns the minimum last_applied_txn_id across all peers for a database -// Used for GC coordination to prevent deleting logs needed by lagging peers -type MinAppliedTxnIDFunc func(database string) (uint64, error) - -// ClusterMinWatermarkFunc returns the minimum applied seq_num across all alive nodes -// Used for GC coordination via gossip protocol (no extra RPC calls needed) -type ClusterMinWatermarkFunc func() uint64 - -// RefreshReplicationStatesFunc refreshes peer replication states before GC decisions -// This queries all alive peers and updates local state, ensuring fresh watermarks -type RefreshReplicationStatesFunc func(ctx context.Context) error +// GCSafePositionFunc returns the highest LogPosition below which GC may +// delete committed log entries (the min of every current member's +// consumed position, R[self,m,d], in this database's log), and whether +// that position is known yet. Absent, or returning false, GC treats +// nothing as safe except entries past max retention. +type GCSafePositionFunc func() (LogPosition, bool) type VectorCDCNotifier interface { ApplyCommittedVectorCDC(ctx context.Context, database string, txnID, seqNum uint64, entries []common.CDCEntry) error @@ -52,29 +47,51 @@ type VectorCDCNotifier interface { // TransactionManager manages distributed transactions // All transaction state is stored in MetaStore (PebbleDB) - no in-memory caching type TransactionManager struct { - db *sql.DB // User database for data operations - metaStore MetaStore // MetaStore for transaction metadata - clock *hlc.Clock - schemaCache *SchemaCache // Schema cache for table metadata - mu sync.RWMutex - gcInterval time.Duration - gcThreshold time.Duration - gcMinRetention time.Duration // Minimum retention for replication - gcMaxRetention time.Duration // Force GC after this duration - heartbeatTimeout time.Duration - stopGC chan struct{} - gcDone chan struct{} - gcRunning bool - databaseName string // Name of database this manager manages - getMinAppliedTxnID MinAppliedTxnIDFunc // Callback for GC coordination - getClusterMinWatermark ClusterMinWatermarkFunc // Callback for GC via gossip watermark - refreshReplicationStates RefreshReplicationStatesFunc // Callback to refresh watermarks before GC - batchCommitter *SQLiteBatchCommitter // SQLite write batcher (nil if disabled) - notifier CDCNotifier // Injected, can be nil - vectorCDCNotifier VectorCDCNotifier // Injected, can be nil - autoIncClaimStore *AutoIncClaimStore // AUTO_INCREMENT claim store; always backed by the system database + db *sql.DB // User database for data operations + metaStore MetaStore // MetaStore for transaction metadata + clock *hlc.Clock + schemaCache *SchemaCache // Schema cache for table metadata + mu sync.RWMutex + gcInterval time.Duration + gcThreshold time.Duration + gcMinRetention time.Duration // Minimum retention for replication + gcMaxRetention time.Duration // Force GC after this duration + heartbeatTimeout time.Duration + stopGC chan struct{} + gcDone chan struct{} + gcRunning bool + databaseName string // Name of database this manager manages + gcSafePosition GCSafePositionFunc // Callback for GC's safe deletion position + batchCommitter *SQLiteBatchCommitter // SQLite write batcher (nil if disabled) + notifier CDCNotifier // Injected, can be nil + vectorCDCNotifier VectorCDCNotifier // Injected, can be nil + autoIncClaimStore *AutoIncClaimStore // AUTO_INCREMENT claim store; always backed by the system database + schemaVersionBumped func(uint64) // Injected; told the new value whenever a non-DML commit bumps __marmot_schema_version + appliedMarker func(uint64) (bool, error) // Injected; reports whether __marmot_applied_txn holds a txn id (SetAppliedMarkerCheck) + + // commitInProgress serializes CommitTransaction across *Transaction + // objects that name the same txn id: txn id -> a channel closed when + // that id's commit attempt ends. A locally PREPARED transaction the log + // puller commits through the normal 2PC path can then never race a COMMIT + // RPC for the same id into a double apply. See CommitTransaction. + commitInProgress sync.Map + + // txnIDLocks gates BeginTransactionWithID and + // ReplicatedDatabase.LogReplayedTxn against each other for the same txn + // id, so a local PREPARE's begin and a replay of the same txn id + // can never interleave. See lockTxnID's doc comment. + txnIDLocks [txnIDLockStripes]sync.Mutex + + // failBeforeFinalizeForTest stops CommitTransaction after the data and + // marker are committed and before the commit record: the crash window a + // test drives. Only tests set it. + failBeforeFinalizeForTest bool } +// errInjectedCrashBeforeFinalize is what CommitTransaction returns when a +// test injects a crash before the commit record (failBeforeFinalizeForTest). +var errInjectedCrashBeforeFinalize = errors.New("injected crash before the commit record") + // NewTransactionManager creates a new transaction manager func NewTransactionManager(db *sql.DB, metaStore MetaStore, clock *hlc.Clock, schemaCache *SchemaCache) *TransactionManager { // Import config values (with fallback to defaults if config not loaded) @@ -121,28 +138,15 @@ func (tm *TransactionManager) SetDatabaseName(name string) { tm.databaseName = name } -// SetMinAppliedTxnIDFunc sets the callback for querying minimum applied txn_id across peers -// Used for GC safe point calculation -func (tm *TransactionManager) SetMinAppliedTxnIDFunc(fn MinAppliedTxnIDFunc) { - tm.mu.Lock() - defer tm.mu.Unlock() - tm.getMinAppliedTxnID = fn -} - -// SetClusterMinWatermarkFunc sets the callback for getting cluster minimum watermark -// This is obtained from the gossip protocol without extra RPC calls -func (tm *TransactionManager) SetClusterMinWatermarkFunc(fn ClusterMinWatermarkFunc) { - tm.mu.Lock() - defer tm.mu.Unlock() - tm.getClusterMinWatermark = fn -} - -// SetRefreshReplicationStatesFunc sets the callback for refreshing peer replication states -// This is called before GC Phase 2 to ensure watermarks are fresh before deletion decisions -func (tm *TransactionManager) SetRefreshReplicationStatesFunc(fn RefreshReplicationStatesFunc) { +// SetGCSafePositionFunc sets the callback GC uses to learn the safe +// deletion position. It replaces SetMinAppliedTxnIDFunc, +// SetClusterMinWatermarkFunc and SetRefreshReplicationStatesFunc, whose +// txn-id/seq-num watermarks could not tell GC which log positions every +// member had pulled (see CleanupOldTransactionRecords). +func (tm *TransactionManager) SetGCSafePositionFunc(fn GCSafePositionFunc) { tm.mu.Lock() defer tm.mu.Unlock() - tm.refreshReplicationStates = fn + tm.gcSafePosition = fn } // SetNotifier sets the CDC notifier for signaling after commits @@ -171,6 +175,25 @@ func (tm *TransactionManager) SetVectorCDCNotifier(notifier VectorCDCNotifier) { tm.vectorCDCNotifier = notifier } +// VectorCDCNotifier returns the injected vector CDC notifier, or nil if none +// is configured. Used by db/replay_apply.go to apply vector-control rows and +// committed vector CDC for a replayed transaction. +func (tm *TransactionManager) VectorCDCNotifier() VectorCDCNotifier { + tm.mu.RLock() + defer tm.mu.RUnlock() + return tm.vectorCDCNotifier +} + +// SetSchemaVersionBumped injects the callback applyNonDMLIntents notifies, +// with the new value, whenever it bumps this database's +// __marmot_schema_version. Wired by NewReplicatedDatabase to update +// its own cached atomic; nil is fine and just means nothing is told. +func (tm *TransactionManager) SetSchemaVersionBumped(fn func(uint64)) { + tm.mu.Lock() + defer tm.mu.Unlock() + tm.schemaVersionBumped = fn +} + // SetAutoIncClaimStore injects the AUTO_INCREMENT claim store used by // seedAutoIncBasesForDDL (db/autoinc_seed.go). The store always backs onto // the system database, never onto tm's own db; DatabaseManager.wireGCCoordination @@ -211,13 +234,88 @@ func (tm *TransactionManager) BeginTransaction(nodeID uint64) (*Transaction, err return tm.BeginTransactionWithID(txnID, nodeID, startTS) } +// txnIDLockStripes is the stripe count for TransactionManager.txnIDLocks: a +// power of two so a stripe index is a cheap mask, sized well above the +// expected number of transactions racing this window concurrently. +const txnIDLockStripes = 64 + +// lockTxnID acquires the stripe of txnIDLocks that guards txnID and returns +// a function that releases it. Held by BeginTransactionWithID around +// tm.metaStore.BeginTransaction, and by ReplicatedDatabase.LogReplayedTxn +// across its whole body, so a local PREPARE's begin can never interleave +// with a replay of the same txn id: either the replay logs the txn +// completely first - LogReplayedTxn's own COMMITTED check then makes the +// later begin observe it - or the begin runs first and the replay's status +// check (or its caller's PENDING refusal) resolves the race instead. +func (tm *TransactionManager) lockTxnID(txnID uint64) (unlock func()) { + stripe := &tm.txnIDLocks[txnID%txnIDLockStripes] + stripe.Lock() + return stripe.Unlock +} + +// ErrTxnAlreadyCommitted refuses to begin a transaction id that this node +// already holds COMMITTED or already applied (see BeginTransactionWithID). +var ErrTxnAlreadyCommitted = errors.New("transaction already committed on this node") + +// SetAppliedMarkerCheck installs the read-only lookup of the database's +// applied-txn marker that BeginTransactionWithID consults. The owning +// ReplicatedDatabase installs one reading through its read pool, so the check +// never waits on the single SQLite writer. +func (tm *TransactionManager) SetAppliedMarkerCheck(check func(txnID uint64) (bool, error)) { + tm.mu.Lock() + defer tm.mu.Unlock() + tm.appliedMarker = check +} + +// refuseCommittedTxnID returns ErrTxnAlreadyCommitted when txnID's local +// record is COMMITTED or, lacking a record, its applied-txn marker exists. +func (tm *TransactionManager) refuseCommittedTxnID(txnID uint64) error { + rec, err := tm.metaStore.GetTransaction(txnID) + if err != nil { + return fmt.Errorf("check transaction %d status: %w", txnID, err) + } + if rec != nil { + if rec.Status == TxnStatusCommitted { + return fmt.Errorf("transaction %d: %w", txnID, ErrTxnAlreadyCommitted) + } + return nil + } + tm.mu.RLock() + check := tm.appliedMarker + tm.mu.RUnlock() + if check == nil { + return nil + } + applied, err := check(txnID) + if err != nil { + return fmt.Errorf("check transaction %d applied marker: %w", txnID, err) + } + if applied { + return fmt.Errorf("transaction %d: %w", txnID, ErrTxnAlreadyCommitted) + } + return nil +} + // BeginTransactionWithID starts a distributed transaction with a specific ID // Used by coordinator replication to ensure consistent txn_id across cluster // Transaction state is persisted to MetaStore only - no in-memory caching +// +// A txn id this node already holds COMMITTED, or whose applied-txn marker its +// database already carries, is refused with ErrTxnAlreadyCommitted: it was +// pulled from a peer's log (or arrived in a restored snapshot) before this +// late PREPARE, and beginning it again would overwrite the committed record +// with a PENDING one whose abort or commit would corrupt the log entry. func (tm *TransactionManager) BeginTransactionWithID(txnID, nodeID uint64, startTS hlc.Timestamp) (*Transaction, error) { - // Persist transaction record to MetaStore - if err := tm.metaStore.BeginTransaction(txnID, nodeID, startTS); err != nil { - return nil, fmt.Errorf("failed to create transaction record: %w", err) + unlock := tm.lockTxnID(txnID) + err := tm.refuseCommittedTxnID(txnID) + if err == nil { + if err = tm.metaStore.BeginTransaction(txnID, nodeID, startTS); err != nil { + err = fmt.Errorf("failed to create transaction record: %w", err) + } + } + unlock() + if err != nil { + return nil, err } // Return a transient object for use during this request @@ -288,7 +386,29 @@ func (tm *TransactionManager) WriteIntent(txn *Transaction, intentType IntentTyp // CommitTransaction commits the transaction. // DML: Get CDC entries → apply via batch committer → cleanup // DDL: Flush pending DML → get intents → apply DDL → cleanup +// +// Concurrent-commit guard: commitInProgress serializes commits of the same +// txn id across distinct *Transaction objects (txn.mu only guards one +// object, not the id) - the log puller commits a locally prepared +// transaction through this path (DatabaseManager.CommitLocallyPrepared), +// which can race the COMMIT RPC, or the coordinator's own local commit, for +// the same id. A second caller waits for the first to finish, then re-reads +// the authoritative status from MetaStore - not just txn.Status, which an +// independently reconstructed *Transaction object still shows as PENDING - +// and gets ErrTxnAlreadyCommitted, without redoing any of the apply, if the +// first committed it (MarkSQLiteTxnApplied is INSERT OR IGNORE, so its own +// idempotence would otherwise mask a duplicate finalizeCommit writing a +// second log entry). func (tm *TransactionManager) CommitTransaction(txn *Transaction) error { + return tm.CommitTransactionAfter(txn, nil) +} + +// CommitTransactionAfter is CommitTransaction with before run first, under +// the concurrent-commit guard and after the status check, so work that must +// happen exactly once before the commit (applying an AUTO_INCREMENT claim) +// never runs for a transaction another caller already committed. An error +// from before aborts the commit and is returned as is. +func (tm *TransactionManager) CommitTransactionAfter(txn *Transaction, before func() error) error { txn.mu.Lock() defer txn.mu.Unlock() @@ -296,11 +416,26 @@ func (tm *TransactionManager) CommitTransaction(txn *Transaction) error { return fmt.Errorf("transaction %d is not pending", txn.ID) } + release, err := tm.acquireCommit(txn.ID) + if err != nil { + return err + } + defer release() + + if before != nil { + if err := before(); err != nil { + return err + } + } + // Get CDC entries - determines DML vs DDL path cdcEntries, err := tm.metaStore.GetIntentEntries(txn.ID) if err != nil { return fmt.Errorf("failed to load CDC entries: %w", err) } + if err := tm.verifyPreparedPayload(txn.ID, len(cdcEntries)); err != nil { + return err + } // Calculate commit timestamp txn.CommitTS = tm.calculateCommitTS(txn.StartTS) @@ -337,19 +472,26 @@ func (tm *TransactionManager) CommitTransaction(txn *Transaction) error { return fmt.Errorf("failed to fetch write intents: %w", err) } - if err := tm.applyNonDMLIntents(txn.ID, intents); err != nil { - return err - } - // Write non-DML statements to CDC storage for streaming replication + // The captured rows are written before the SQLite commit that writes + // the marker: a crash after that commit then leaves rows for + // repairAppliedTxnMetadata to finish the commit record from at the + // next open, so the applied DDL still enters this node's log. if err := tm.writeNonDMLToCDC(txn.ID, intents); err != nil { return err } if err := tm.metaStore.SealCapturedRows(txn.ID); err != nil { return fmt.Errorf("failed to seal CDC rows: %w", err) } + if err := tm.applyNonDMLIntents(txn.ID, txn.CommitTS, intents); err != nil { + return err + } txn.Statements = tm.rebuildStatementsFromCDC(nil, intents) } + if tm.failBeforeFinalizeForTest { + return errInjectedCrashBeforeFinalize + } + // Finalize commit in MetaStore if err := tm.finalizeCommit(txn); err != nil { return err @@ -370,14 +512,87 @@ func (tm *TransactionManager) CommitTransaction(txn *Transaction) error { return nil } -// ErrPreparedRowsMissing refuses the COMMIT of a transaction that names DML -// rows whose captured images are gone. Every such row was captured at PREPARE -// (createDMLIntent refuses a DML row without its image), so finding none means -// something discarded them - the stale-transaction GC aborting the transaction -// while this COMMIT was under way, or a recovery that dropped them - and -// committing would ACK a write this node never applied. +// acquireCommit takes txnID's commitInProgress entry, waiting for any +// commit of the same id already under way to finish first, and re-reads its +// authoritative status (see CommitTransaction's concurrent-commit guard): +// ErrTxnAlreadyCommitted if it is COMMITTED, an error if it is otherwise not +// PENDING. The caller runs release once its commit attempt ends. +func (tm *TransactionManager) acquireCommit(txnID uint64) (release func(), err error) { + done := make(chan struct{}) + for { + prev, busy := tm.commitInProgress.LoadOrStore(txnID, done) + if !busy { + break + } + <-prev.(chan struct{}) + } + release = func() { + tm.commitInProgress.Delete(txnID) + close(done) + } + rec, err := tm.metaStore.GetTransaction(txnID) + switch { + case err != nil: + err = fmt.Errorf("read transaction %d status: %w", txnID, err) + case rec != nil && rec.Status == TxnStatusCommitted: + err = fmt.Errorf("transaction %d: %w", txnID, ErrTxnAlreadyCommitted) + case rec == nil || rec.Status != TxnStatusPending: + err = fmt.Errorf("transaction %d is not pending", txnID) + } + if err != nil { + release() + return nil, err + } + return release, nil +} + +// ErrPreparedRowsMissing refuses the COMMIT of a transaction whose prepared +// payload is gone: DML rows its statements name, or fewer captured rows or +// persisted intents than its durable prepare recorded (PreparedPayload). +// Every such row and intent was written at PREPARE, so finding fewer means +// something discarded them - a stale-transaction abort, or a recovery that +// dropped them - and committing would ACK a write this node never applied. var ErrPreparedRowsMissing = errors.New("prepared DML rows are gone; refusing to commit nothing") +// verifyPreparedPayload refuses, with ErrPreparedRowsMissing, a commit of +// txnID when what it finds - rows captured rows and its persisted intents - +// differs from what its durable prepare recorded. A transaction with no such +// record (never durably prepared) is not checked. The check reads the +// durable record, not the Statements of the *Transaction being committed, +// which a commit that reconstructed the transaction (the log puller, a +// COMMIT RPC) never has. +func (tm *TransactionManager) verifyPreparedPayload(txnID uint64, rows int) error { + payload, found, err := tm.metaStore.PreparedPayload(txnID) + if err != nil { + return fmt.Errorf("read prepared payload of transaction %d: %w", txnID, err) + } + if !found { + return nil + } + if uint64(rows) != payload.Rows { + return fmt.Errorf("transaction %d: %w: prepared %d captured rows, found %d", + txnID, ErrPreparedRowsMissing, payload.Rows, rows) + } + if payload.Intents == 0 { + return nil + } + intents, err := tm.metaStore.GetIntentsByTxn(txnID) + if err != nil { + return fmt.Errorf("failed to fetch write intents: %w", err) + } + var persisted uint64 + for _, intent := range intents { + if intent.IntentType != IntentTypeDML { + persisted++ + } + } + if persisted != payload.Intents { + return fmt.Errorf("transaction %d: %w: prepared %d intents, found %d", + txnID, ErrPreparedRowsMissing, payload.Intents, persisted) + } + return nil +} + // statementsNamePreparedRows reports whether statements carry a DML row that // PREPARE captured: a row change with an intent key. An INSERT without one // (an auto-increment key the CDC hook assigns) and an AUTO_INCREMENT claim @@ -525,92 +740,222 @@ func (tm *TransactionManager) applyCDCEntries(txnID uint64, commitTS hlc.Timesta return nil } -// applyNonDMLIntents executes DDL and LOAD DATA statements from write intents. -func (tm *TransactionManager) applyNonDMLIntents(txnID uint64, intents []*WriteIntentRecord) error { - nonDMLIntents := make([]*WriteIntentRecord, 0, len(intents)) +// nonDMLIntentKind classifies one non-DML write intent by what its data +// snapshot decodes as: vector-control metadata, a LOAD DATA payload, or (the +// default) a raw DDL statement. +type nonDMLIntentKind struct { + intent *WriteIntentRecord + vector *common.VectorIndexChange + load *LoadDataSnapshot +} + +// classifyNonDMLIntents filters intents down to the DDL-carrying ones (the +// same filter applyNonDMLIntents has always used) and decodes each one's +// kind, in CreatedAt order. +func classifyNonDMLIntents(intents []*WriteIntentRecord) []nonDMLIntentKind { + filtered := make([]*WriteIntentRecord, 0, len(intents)) for _, intent := range intents { if intent.IntentType == IntentTypeDDL && (intent.SQLStatement != "" || len(intent.DataSnapshot) > 0) { - nonDMLIntents = append(nonDMLIntents, intent) + filtered = append(filtered, intent) } } - - sort.Slice(nonDMLIntents, func(i, j int) bool { - return nonDMLIntents[i].CreatedAt < nonDMLIntents[j].CreatedAt + sort.Slice(filtered, func(i, j int) bool { + return filtered[i].CreatedAt < filtered[j].CreatedAt }) - var change SchemaChange - for _, intent := range nonDMLIntents { + kinds := make([]nonDMLIntentKind, 0, len(filtered)) + for _, intent := range filtered { var vectorChange common.VectorIndexChange if err := DeserializeData(intent.DataSnapshot, &vectorChange); err == nil && vectorChange.Action != 0 { - controlApplier, ok := tm.vectorCDCNotifier.(interface { - ApplyVectorControl(context.Context, common.VectorIndexChange) error - }) - if !ok { - return fmt.Errorf("vector index control %s: vector manager not configured", vectorChange.IndexName) - } - if err := controlApplier.ApplyVectorControl(context.Background(), vectorChange); err != nil { - return fmt.Errorf("failed to apply vector index control: %w", err) - } + kinds = append(kinds, nonDMLIntentKind{intent: intent, vector: &vectorChange}) continue } var loadSnap LoadDataSnapshot if err := DeserializeData(intent.DataSnapshot, &loadSnap); err == nil && loadSnap.Type == int(protocol.StatementLoadData) { - if _, err := ApplyLoadData(tm.db, loadSnap.SQL, loadSnap.Data); err != nil { - return fmt.Errorf("failed to execute LOAD DATA statement: %w", err) - } - log.Debug().Uint64("txn_id", txnID).Msg("LOAD DATA statement executed") + kinds = append(kinds, nonDMLIntentKind{intent: intent, load: &loadSnap}) continue } - if err := tm.execDDL(&change, intent); err != nil { - return err - } - log.Debug().Uint64("txn_id", txnID).Str("sql", intent.SQLStatement).Msg("DDL statement executed") + kinds = append(kinds, nonDMLIntentKind{intent: intent}) } - if change.Empty() { + return kinds +} + +// applyNonDMLIntents applies a non-DML transaction's DDL, LOAD DATA and +// vector-control intents, atomically with the applied-txn marker and (when +// the txn carries real DDL) the schema-version bump. +// +// Exactly-once / idempotence, per effect: +// - DDL, LOAD DATA, the schema-version bump and the marker all commit in one +// SQLite transaction: a crash anywhere before commit leaves none of them +// applied, and replay (db/replay_apply.go) sees no marker and reapplies +// the whole txn from scratch. A crash after commit is a normal completed +// commit. +// - Vector control (CREATE/DROP/CHECKPOINT/REINDEX on __marmot_vector_indexes) +// is not SQLite and cannot share that transaction, so it is applied first, +// outside it. CREATE, DROP and CHECKPOINT are no-ops on repeat and REINDEX +// is a repeatable rebuild (VectorIndexManager.ApplyVectorControl), so a +// crash between applying the control and committing the marker tx just +// re-runs it once on the next apply - replay is gated on the marker +// before the control runs (db/replay_apply.go). +// - A claim-only txn (an AUTO_INCREMENT range claim, which has no DDL/LOAD +// DATA/vector intents at all) writes NO marker here and opens no SQLite +// tx at all, as explained below. +// +// A claim-only commit writes no marker because it cannot take the user +// database's writer: ReplicationEngine.Commit calls this after +// AutoIncClaimStore.ApplyClaims for every claim-carrying txn, including the +// LLDAP shape where a pinned session already holds the user database's one +// SQLite writer (_txlock=immediate) for the whole surrounding transaction. +// __marmot_applied_txn lives in that same user database file, so writing it +// needs that same writer - which would self-deadlock the claim behind the +// pinned session that is waiting on it, defeating the entire reason +// AutoIncClaimStore lives in the system database (a separate file with its +// own writer) rather than the user database +// (TestClaimCompletesWhilePinnedSessionOpen pins this). +// +// Skipping it is safe: a claim-only txn still gets a local log entry +// (finalizeCommit writes its commit record), so a peer's anti-entropy, and +// this node's own restore re-apply, will find it without a marker and replay +// it. That replay is a zero-row ReplayTxn: ApplyReplayedTxn only writes the +// marker, off this commit path, and never re-applies the claim - claim +// payloads never travel in CDC, and a bystander's floor comes from the claim +// protocol's merged base (AutoIncHoldTable, grpc.RunAutoIncBaseMerge), not +// from replay. The marker is therefore bookkeeping for the pull diff only, +// and writing it later, once, is equivalent to writing it here. +func (tm *TransactionManager) applyNonDMLIntents(txnID uint64, commitTS hlc.Timestamp, intents []*WriteIntentRecord) error { + kinds := classifyNonDMLIntents(intents) + if len(kinds) == 0 { + // Claim-only (or otherwise empty): no marker, no SQLite tx. See the + // doc comment for why. return nil } - // Reported before the schema cache reload below makes the new definitions - // visible to queries, so no insert into a new incarnation can take an id - // from a range claimed for an old one. - tm.endIncarnations(&change) + if err := tm.applyNonDMLVectorControl(kinds); err != nil { + return err + } - // Reload schema cache after DDL operations. A failed reload leaves the - // cache stale, so subsequent preupdate hooks would silently drop CDC data - // for any column the DDL added/changed - fail the DDL apply instead of - // swallowing the error. - if tm.schemaCache != nil { - if err := tm.reloadSchemaCache(); err != nil { - return fmt.Errorf("failed to reload schema cache after DDL (txn %d): %w", txnID, err) + ctx := context.Background() + tx, err := tm.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to begin non-DML transaction: %w", err) + } + defer tx.Rollback() + + change, hasDDL, err := tm.execNonDMLIntentsInTx(ctx, tx, txnID, kinds) + if err != nil { + return err + } + + var newSchemaVersion uint64 + if hasDDL { + newSchemaVersion, err = bumpSchemaVersionInTx(tx) + if err != nil { + return fmt.Errorf("failed to bump schema version (txn %d): %w", txnID, err) } } - // Seed __marmot__autoinc for every table this DDL tagged AUTO_INCREMENT, - // after the DDL has executed and the schema cache reflects it. The claim - // table lives in the system database - a separate SQLite file from tm.db - // (db/autoinc_claim.go) - so the seed cannot share the DDL's own - // transaction: it is written in its own system-database transaction, - // still before this COMMIT is ACKed. A seeding failure therefore still - // fails the DDL apply and the COMMIT with it, the same durability boundary - // AutoIncClaimStore.ApplyClaims relies on for the claim itself: a table - // left untagged here would let a subsequent range claim be evaluated - // against a missing base row. - if err := tm.seedSchemaChange(tm.db, &change); err != nil { - return fmt.Errorf("failed to seed auto-increment base after DDL (txn %d): %w", txnID, err) + if err := MarkSQLiteTxnApplied(tx, txnID, commitTS); err != nil { + return err + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("failed to commit non-DML transaction: %w", err) + } + + return tm.afterNonDMLCommit(txnID, hasDDL, newSchemaVersion, &change) +} + +// applyNonDMLVectorControl applies every vector-index control intent in +// kinds. It runs first, outside SQLite - vector control is not SQLite and +// cannot share the tx applyNonDMLIntents opens afterward - matching the order +// applyNonDMLIntents's doc comment describes and depends on. +func (tm *TransactionManager) applyNonDMLVectorControl(kinds []nonDMLIntentKind) error { + for _, k := range kinds { + if k.vector == nil { + continue + } + controlApplier, ok := tm.vectorCDCNotifier.(interface { + ApplyVectorControl(context.Context, common.VectorIndexChange) error + }) + if !ok { + return fmt.Errorf("vector index control %s: vector manager not configured", k.vector.IndexName) + } + if err := controlApplier.ApplyVectorControl(context.Background(), *k.vector); err != nil { + return fmt.Errorf("failed to apply vector index control: %w", err) + } } return nil } -// execDDL executes one DDL intent on one connection, so the schema reads -// SchemaChange.Exec takes around it bracket exactly this statement. -func (tm *TransactionManager) execDDL(change *SchemaChange, intent *WriteIntentRecord) error { - ctx := context.Background() - conn, err := tm.db.Conn(ctx) - if err != nil { - return fmt.Errorf("failed to execute DDL statement: %w", err) +// execNonDMLIntentsInTx runs every non-vector intent in kinds inside tx +// (LOAD DATA and DDL, vector control having already run outside it), seeds +// AUTO_INCREMENT bases for any DDL this txn ran, and reports the accumulated +// schema change and whether any DDL ran. +func (tm *TransactionManager) execNonDMLIntentsInTx(ctx context.Context, tx *sql.Tx, txnID uint64, kinds []nonDMLIntentKind) (SchemaChange, bool, error) { + var change SchemaChange + hasDDL := false + for _, k := range kinds { + switch { + case k.vector != nil: + continue // already applied above + case k.load != nil: + if _, err := ApplyLoadDataInTx(tx, k.load.SQL, k.load.Data); err != nil { + return change, false, fmt.Errorf("failed to execute LOAD DATA statement: %w", err) + } + log.Debug().Uint64("txn_id", txnID).Msg("LOAD DATA statement executed") + default: + if err := change.Exec(ctx, tx, k.intent.SQLStatement, k.intent.TableName, k.intent.NodeID); err != nil { + return change, false, fmt.Errorf("failed to execute DDL statement: %w", err) + } + hasDDL = true + log.Debug().Uint64("txn_id", txnID).Str("sql", k.intent.SQLStatement).Msg("DDL statement executed") + } + } + + if !change.Empty() { + // Reported before the schema cache reload below makes the new + // definitions visible to queries, so no insert into a new incarnation + // can take an id from a range claimed for an old one. + tm.endIncarnations(&change) + + // Seed __marmot__autoinc for every table this DDL tagged + // AUTO_INCREMENT, reading floors through tx (same as + // FinishReplayedSchemaChange does for replay) so the seed sees + // exactly the rows this transaction's own DDL left behind. + if err := tm.seedSchemaChange(tx, &change); err != nil { + return change, false, fmt.Errorf("failed to seed auto-increment base after DDL (txn %d): %w", txnID, err) + } } - defer conn.Close() - return change.Exec(ctx, conn, intent.SQLStatement, intent.TableName, intent.NodeID) + + return change, hasDDL, nil +} + +// afterNonDMLCommit runs applyNonDMLIntents's post-commit refresh: notifying +// the injected schema-version-bumped callback (which advances the owning +// ReplicatedDatabase's cached __marmot_schema_version, monotonically) and +// reloading the schema cache when DDL ran. A failed reload leaves the cache +// stale, so subsequent preupdate hooks would silently drop CDC data for any +// column the DDL added/changed - fail the commit instead of swallowing the +// error, even though the DDL itself already committed (the same shape as +// handleReplay's post-commit ReloadSchema, but propagated instead of only +// logged, matching this path's existing contract). +func (tm *TransactionManager) afterNonDMLCommit(txnID uint64, hasDDL bool, newSchemaVersion uint64, change *SchemaChange) error { + if hasDDL { + tm.mu.RLock() + bumped := tm.schemaVersionBumped + tm.mu.RUnlock() + if bumped != nil { + bumped(newSchemaVersion) + } + } + + if !change.Empty() && tm.schemaCache != nil { + if err := tm.reloadSchemaCache(); err != nil { + return fmt.Errorf("failed to reload schema cache after DDL (txn %d): %w", txnID, err) + } + } + + return nil } // writeNonDMLToCDC writes DDL/LOAD DATA statements to CDC storage for streaming replication. @@ -711,7 +1056,18 @@ func (tm *TransactionManager) cleanupAfterCommit(txn *Transaction) { } } -// AbortTransaction aborts the transaction and cleans up write intents +// AbortTransaction aborts the transaction and cleans up write intents. +// +// A late PREPARE's abort can hit a txn id that was already replayed and +// COMMITTED in the log: MetaStore.AbortTransaction then refuses with +// ErrAbortCommitted instead of deleting the commit record. This still +// releases the PREPARE's own write intents (DeleteIntentsByTxn) - they are +// this abort's to clean up regardless - but it must NOT delete the CDC +// intent entries or captured rows (DeleteIntentEntries, DeleteCapturedRows): +// those now belong to the committed log entry, and deleting them would +// destroy rows a peer's next pull, or this node's own restore re-apply, +// still needs to read back. The error is returned so the caller knows the +// abort did not take effect. func (tm *TransactionManager) AbortTransaction(txn *Transaction) error { txn.mu.Lock() defer txn.mu.Unlock() @@ -720,16 +1076,23 @@ func (tm *TransactionManager) AbortTransaction(txn *Transaction) error { return fmt.Errorf("transaction %d is not pending", txn.ID) } - // Abort the transaction in MetaStore (deletes record) - if err := tm.metaStore.AbortTransaction(txn.ID); err != nil { - return fmt.Errorf("failed to abort transaction: %w", err) + abortErr := tm.metaStore.AbortTransaction(txn.ID) + if abortErr != nil && !errors.Is(abortErr, ErrAbortCommitted) { + return fmt.Errorf("failed to abort transaction: %w", abortErr) } - txn.Status = TxnStatusAborted - - // Clean up all write intents via MetaStore + // Clean up this PREPARE's own write intents unconditionally: they are + // never the committed log entry's data. _ = tm.metaStore.DeleteIntentsByTxn(txn.ID) + if abortErr != nil { + // ErrAbortCommitted: leave the CDC intent entries and captured rows + // alone, they belong to the committed log entry now. + return abortErr + } + + txn.Status = TxnStatusAborted + // Clean up CDC intent entries _ = tm.metaStore.DeleteIntentEntries(txn.ID) @@ -885,56 +1248,59 @@ func (tm *TransactionManager) runGarbageCollection() { } // cleanupStaleTransactions aborts transactions that haven't had a heartbeat within the timeout +// +// Each abort runs under the transaction's commit guard (acquireCommit), so it +// never interleaves with a commit of the same transaction - the log puller's +// CommitLocallyPrepared or a COMMIT RPC: a commit that wins commits the +// transaction's whole prepared payload, and one that loses finds it no longer +// PENDING. A transaction a commit ended while this pass waited is skipped. func (tm *TransactionManager) cleanupStaleTransactions() (int, error) { - return tm.metaStore.CleanupStaleTransactions(tm.heartbeatTimeout) + stale, err := tm.metaStore.StaleTransactionIDs(tm.heartbeatTimeout) + if err != nil { + return 0, err + } + cleaned := 0 + for _, txnID := range stale { + release, err := tm.acquireCommit(txnID) + if err != nil { + continue // committed, aborted or gone since it was listed + } + aborted, err := tm.metaStore.AbortStaleTransaction(txnID, tm.heartbeatTimeout) + release() + if err != nil { + log.Warn().Err(err).Uint64("txn_id", txnID).Msg("GC: failed to abort stale transaction") + continue + } + if aborted { + cleaned++ + } + } + if cleaned > 0 { + log.Info().Int("stale_txns", cleaned).Msg("GC: aborted stale transactions") + } + return cleaned, nil } -// cleanupOldTransactionRecords removes old COMMITTED/ABORTED transaction records -// with GC safe point coordination to prevent deleting logs needed by lagging peers -// Uses belt-and-suspenders: both txn_id from anti-entropy and seq_num from gossip watermark +// cleanupOldTransactionRecords deletes committed log entries at or below +// the GC-safe position (see GCSafePositionFunc) once they are older than +// gcMinRetention, or unconditionally once they are older than gcMaxRetention. func (tm *TransactionManager) cleanupOldTransactionRecords() (int, error) { - // Get callbacks under lock + // Get callback under lock tm.mu.RLock() - refreshFn := tm.refreshReplicationStates - getMinAppliedFn := tm.getMinAppliedTxnID - getWatermarkFn := tm.getClusterMinWatermark + safeFn := tm.gcSafePosition dbName := tm.databaseName tm.mu.RUnlock() - var minAppliedTxnID uint64 = 0 - var minAppliedSeqNum uint64 = 0 + var safe LogPosition // Skip replication tracking for system database (it's not replicated) - if dbName != "" && dbName != SystemDatabaseName { - // CRITICAL: Refresh peer replication states BEFORE getting watermarks - // This ensures we have fresh data before making GC deletion decisions - if refreshFn != nil { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - if err := refreshFn(ctx); err != nil { - // Log warning but continue with potentially stale data - // The retention windows provide safety margin - log.Warn().Err(err).Str("database", dbName).Msg("GC: Failed to refresh replication states, using cached watermarks") - } - cancel() - } - - // Get min applied txn_id from anti-entropy (now refreshed) - if getMinAppliedFn != nil { - minTxnID, err := getMinAppliedFn(dbName) - if err == nil { - minAppliedTxnID = minTxnID - } else { - log.Warn().Err(err).Str("database", dbName).Msg("GC: Failed to get min applied txn_id") - } - } - - // Get cluster min watermark from gossip (if available) - if getWatermarkFn != nil { - minAppliedSeqNum = getWatermarkFn() + if dbName != "" && dbName != SystemDatabaseName && safeFn != nil { + if pos, ok := safeFn(); ok { + safe = pos } } - count, err := tm.metaStore.CleanupOldTransactionRecords(tm.gcMinRetention, tm.gcMaxRetention, minAppliedTxnID, minAppliedSeqNum) + count, err := tm.metaStore.CleanupOldTransactionRecords(tm.gcMinRetention, tm.gcMaxRetention, safe) if err != nil { return 0, err } @@ -943,8 +1309,8 @@ func (tm *TransactionManager) cleanupOldTransactionRecords() (int, error) { log.Info(). Str("database", dbName). Int("deleted_records", count). - Uint64("min_txn_id", minAppliedTxnID). - Uint64("min_seq_num", minAppliedSeqNum). + Uint64("safe_seq", safe.Seq). + Uint64("safe_txn_id", safe.TxnID). Msg("GC: Cleaned up old transaction records") } diff --git a/db/transaction_nondml_atomic_test.go b/db/transaction_nondml_atomic_test.go new file mode 100644 index 00000000..55e2edb2 --- /dev/null +++ b/db/transaction_nondml_atomic_test.go @@ -0,0 +1,246 @@ +package db + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// TestApplyNonDMLIntents_AtomicAcrossDDLFailure is the atomicity +// regression: a failure on the second statement of a non-DML commit must +// roll back the whole SQLite transaction, so the first (otherwise +// successful) DDL statement, the schema-version bump and the applied-txn +// marker are all-or-nothing together. +// +// Before this fix, DDL ran under autocommit per statement +// (TransactionManager.execDDL opened its own *sql.Conn per intent), so the +// first CREATE TABLE would have persisted even though the whole apply +// failed on the second statement. +func TestApplyNonDMLIntents_AtomicAcrossDDLFailure(t *testing.T) { + testDB := setupTestDBWithMeta(t) + clock := hlc.NewClock(1) + tm := NewTransactionManager(testDB.DB, testDB.MetaStore, clock, NewSchemaCache()) + tm.SetDatabaseName("testdb") + _, err := ensureSchemaVersionTable(testDB.DB, "testdb", false, nil) + require.NoError(t, err) + + intents := []*WriteIntentRecord{ + {IntentType: IntentTypeDDL, TableName: "t", NodeID: 1, SQLStatement: "CREATE TABLE t (id INTEGER PRIMARY KEY)", CreatedAt: 1}, + // Second statement fails: no such table "nope". + {IntentType: IntentTypeDDL, TableName: "nope", NodeID: 1, SQLStatement: "ALTER TABLE nope ADD COLUMN x INTEGER", CreatedAt: 2}, + } + + err = tm.applyNonDMLIntents(99, hlc.Timestamp{WallTime: 1, NodeID: 1}, intents) + require.Error(t, err, "the second DDL statement must fail") + + var name string + err = testDB.DB.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='t'").Scan(&name) + require.ErrorIs(t, err, sql.ErrNoRows, "the first DDL must be rolled back together with the failed second one") + + var one int + err = testDB.DB.QueryRow("SELECT 1 FROM __marmot_applied_txn WHERE txn_id = ?", 99).Scan(&one) + require.ErrorIs(t, err, sql.ErrNoRows, "no marker must be left behind by a failed apply") + + var version uint64 + require.NoError(t, testDB.DB.QueryRow("SELECT version FROM __marmot_schema_version WHERE id = 1").Scan(&version)) + require.Equal(t, uint64(0), version, "the schema version must not have bumped") +} + +// TestApplyNonDMLIntents_ClaimOnlyWritesNoMarkerAndNoSQLiteTx pins the +// deliberate deviation documented on applyNonDMLIntents: a non-DML commit +// with no DDL, LOAD DATA or vector-control intent at all (the shape of an +// AUTO_INCREMENT claim-only transaction, whose IntentTypeAutoIDClaim intent +// is filtered out entirely) must NOT open a SQLite tx or write a marker. +// Writing one here would need the user database's single writer +// (_txlock=immediate) even when a concurrent pinned session already holds +// it for the whole surrounding LLDAP-shaped transaction - see +// TestClaimCompletesWhilePinnedSessionOpen (db/autoinc_hookdb_test.go), +// which this behavior exists to keep passing. +func TestApplyNonDMLIntents_ClaimOnlyWritesNoMarkerAndNoSQLiteTx(t *testing.T) { + testDB := setupTestDBWithMeta(t) + clock := hlc.NewClock(1) + tm := NewTransactionManager(testDB.DB, testDB.MetaStore, clock, NewSchemaCache()) + tm.SetDatabaseName("testdb") + _, err := ensureSchemaVersionTable(testDB.DB, "testdb", false, nil) + require.NoError(t, err) + + require.NoError(t, tm.applyNonDMLIntents(7, hlc.Timestamp{WallTime: 5, NodeID: 1}, nil)) + + err = testDB.DB.QueryRow("SELECT 1 FROM __marmot_applied_txn WHERE txn_id = ?", 7).Scan(new(int)) + require.ErrorIs(t, err, sql.ErrNoRows, "a claim-only commit must not write a marker") + + var version uint64 + require.NoError(t, testDB.DB.QueryRow("SELECT version FROM __marmot_schema_version WHERE id = 1").Scan(&version)) + require.Equal(t, uint64(0), version, "a claim-only commit must not bump the schema version") +} + +// TestApplyNonDMLIntents_2PCDDLCommitBumpsVersionExactlyOnceAndReplayIsNoOp +// is the end-to-end regression for exactly-once DDL: after a multi-statement +// 2PC DDL transaction (ALTER TABLE ADD COLUMN, then CREATE INDEX) commits +// through TransactionManager.CommitTransaction, the marker exists and the +// schema version is exactly +1 (one bump per committed txn, not per +// statement); replaying the very same transaction through ApplyReplayedTxn +// then applies nothing at all, leaving the marker and the version unchanged, +// where idempotent-rewritten DDL would otherwise silently re-apply and bump +// the version twice. +func TestApplyNonDMLIntents_2PCDDLCommitBumpsVersionExactlyOnceAndReplayIsNoOp(t *testing.T) { + tmpDir := t.TempDir() + dbMgr, err := NewDatabaseManager(tmpDir, 1, hlc.NewClock(1)) + require.NoError(t, err) + defer dbMgr.Close() + require.NoError(t, dbMgr.CreateDatabase("app")) + mdb, err := dbMgr.GetDatabase("app") + require.NoError(t, err) + + _, err = mdb.GetWriteDB().Exec("CREATE TABLE t (id INTEGER PRIMARY KEY)") + require.NoError(t, err) + require.NoError(t, mdb.ReloadSchema()) + + txnMgr := mdb.GetTransactionManager() + txn, err := txnMgr.BeginTransaction(1) + require.NoError(t, err) + + stmts := []struct{ sql, table string }{ + {"ALTER TABLE t ADD COLUMN name TEXT", "t"}, + {"CREATE INDEX idx_name ON t(name)", "t"}, + } + for i, s := range stmts { + stmt := protocol.Statement{Type: protocol.StatementDDL, SQL: s.sql, TableName: s.table, Database: "app"} + snap, err := SerializeData(DDLSnapshot{Type: int(stmt.Type), SQL: s.sql, TableName: s.table}) + require.NoError(t, err) + require.NoError(t, txnMgr.WriteIntent(txn, IntentTypeDDL, s.table, fmt.Sprintf("ddl:%d", i), stmt, snap)) + } + require.NoError(t, txnMgr.CommitTransaction(txn)) + + require.Equal(t, uint64(1), mdb.SchemaVersion(), "a two-statement DDL txn must bump the version exactly once") + + var one int + require.NoError(t, mdb.GetWriteDB().QueryRow("SELECT 1 FROM __marmot_applied_txn WHERE txn_id = ?", txn.ID).Scan(&one)) + + // Reconstruct the replayed form of the same transaction from what + // writeNonDMLToCDC captured during the commit above, exactly as a peer + // pulling this txn from this node's log would receive it. + cursor, err := mdb.GetMetaStore().IterateCapturedRows(txn.ID) + require.NoError(t, err) + var rows []*EncodedCapturedRow + for cursor.Next() { + _, data := cursor.Row() + row, err := DecodeRow(data) + require.NoError(t, err) + rows = append(rows, row) + } + require.NoError(t, cursor.Err()) + require.NoError(t, cursor.Close()) + require.Len(t, rows, 2) + + applied, err := mdb.ApplyReplayedTxn(context.Background(), &ReplayTxn{ + TxnID: txn.ID, + OriginNodeID: 1, + CommitTS: txn.CommitTS, + Rows: rows, + }, false) + require.NoError(t, err) + require.False(t, applied, "an already-committed txn must not be replayed again") + require.Equal(t, uint64(1), mdb.SchemaVersion(), "replaying an already-applied DDL must not bump the version again") +} + +// TestAbortTransaction_RefusesCommittedReplayAndKeepsCapturedRows: a late +// local PREPARE's abort can hit a txn +// id a peer's replay already logged as COMMITTED (MetaStore.AbortTransaction +// then refuses with ErrAbortCommitted). TransactionManager.AbortTransaction +// must still release the PREPARE's own write intents but must NOT delete the +// committed entry's CDC intent entries or captured rows - a peer's next pull, +// or this node's own restore re-apply, still needs to read them back. +func TestAbortTransaction_RefusesCommittedReplayAndKeepsCapturedRows(t *testing.T) { + testDB := setupTestDBWithMeta(t) + tm := NewTransactionManager(testDB.DB, testDB.MetaStore, hlc.NewClock(1), NewSchemaCache()) + tm.SetDatabaseName("testdb") + + const txnID = uint64(9001) + require.NoError(t, testDB.MetaStore.WriteCapturedRow(txnID, 1, []byte("row-data"))) + require.NoError(t, testDB.MetaStore.SealCapturedRows(txnID)) + require.NoError(t, testDB.MetaStore.StoreReplayedTransaction(txnID, 2, hlc.Timestamp{WallTime: 1, NodeID: 2}, "testdb", 1, 0)) + + rec, err := testDB.MetaStore.GetTransaction(txnID) + require.NoError(t, err) + require.NotNil(t, rec) + require.Equal(t, TxnStatusCommitted, rec.Status) + + // The *Transaction object a late local PREPARE's handler would still be + // holding, believing txnID PENDING (tm.GetTransaction would already + // refuse to reconstruct one for a COMMITTED id, so this is built the way + // BeginTransactionWithID's own caller would have kept it). + lateTxn := &Transaction{ID: txnID, Status: TxnStatusPending} + + err = tm.AbortTransaction(lateTxn) + require.ErrorIs(t, err, ErrAbortCommitted) + + cursor, err := testDB.MetaStore.IterateCapturedRows(txnID) + require.NoError(t, err) + count := 0 + for cursor.Next() { + count++ + } + require.NoError(t, cursor.Err()) + require.NoError(t, cursor.Close()) + require.Equal(t, 1, count, "the committed log entry's captured rows must survive the refused abort") + + entries, _, _, err := testDB.MetaStore.ListCommittedLog(LogPosition{}, 10) + require.NoError(t, err) + found := false + for _, e := range entries { + found = found || e.TxnID == txnID + } + require.True(t, found, "the committed log entry must still be listed") +} + +// TestDDLCommitCrashAfterMarkerIsRepairedIntoTheLog: a crash after a 2PC +// DDL's SQLite commit (DDL + marker) and before its commit record leaves the +// DDL applied but outside this node's log. The captured rows are written +// before that SQLite commit, so the next open repairs the commit record from +// them and the DDL enters the log peers pull from. +// +// Mutation: write the captured rows after applyNonDMLIntents again. "the +// applied DDL never entered this node's log" fires. +func TestDDLCommitCrashAfterMarkerIsRepairedIntoTheLog(t *testing.T) { + tmpDir := t.TempDir() + dbMgr, err := NewDatabaseManager(tmpDir, 1, hlc.NewClock(1)) + require.NoError(t, err) + require.NoError(t, dbMgr.CreateDatabase("app")) + mdb, err := dbMgr.GetDatabase("app") + require.NoError(t, err) + txnMgr := mdb.GetTransactionManager() + + txn, err := txnMgr.BeginTransaction(1) + require.NoError(t, err) + ddl := "CREATE TABLE crashed (id INTEGER PRIMARY KEY)" + stmt := protocol.Statement{Type: protocol.StatementDDL, SQL: ddl, TableName: "crashed", Database: "app"} + snap, err := SerializeData(DDLSnapshot{Type: int(stmt.Type), SQL: ddl, TableName: "crashed"}) + require.NoError(t, err) + require.NoError(t, txnMgr.WriteIntent(txn, IntentTypeDDL, "crashed", "ddl:0", stmt, snap)) + require.NoError(t, mdb.GetMetaStore().DurablyPrepareTransaction(txn.ID), "PREPARE's durable fence") + txnMgr.failBeforeFinalizeForTest = true + require.Error(t, txnMgr.CommitTransaction(txn), "the injected crash must stop the commit") + require.NoError(t, dbMgr.Close()) + + dbMgr, err = NewDatabaseManager(tmpDir, 1, hlc.NewClock(1)) + require.NoError(t, err) + defer dbMgr.Close() + mdb, err = dbMgr.GetDatabase("app") + require.NoError(t, err) + rec, err := mdb.GetMetaStore().GetTransaction(txn.ID) + require.NoError(t, err) + require.True(t, rec != nil && rec.Status == TxnStatusCommitted, "the applied DDL never entered this node's log") + entries, _, _, err := mdb.GetMetaStore().ListCommittedLog(LogPosition{}, 10) + require.NoError(t, err) + found := false + for _, e := range entries { + found = found || e.TxnID == txn.ID + } + require.True(t, found, "the repaired DDL is not listed in this node's log") +} diff --git a/db/transaction_rowcount_contract_test.go b/db/transaction_rowcount_contract_test.go new file mode 100644 index 00000000..b243c5c6 --- /dev/null +++ b/db/transaction_rowcount_contract_test.go @@ -0,0 +1,188 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package db + +import ( + "context" + "testing" + + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// TestFinalizeCommitRowCountMatchesCapturedRows is the RowCount contract +// FetchTransactions depends on: a receiver refuses to apply a transaction +// whose captured-row count does not match the commit record's RowCount, so +// every commit path must record exactly the number of rows +// IterateCapturedRows will actually yield for that transaction. This drives +// each commit kind through the real db.ReplicationEngine (Prepare+Commit) +// and compares TransactionRecord.RowCount against a live count from +// IterateCapturedRows - never weakening the comparison, per instructions. +func TestFinalizeCommitRowCountMatchesCapturedRows(t *testing.T) { + t.Run("multi-row DML, non-batch applyCDCEntries path", func(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + require.NoError(t, dm.CreateDatabase("testdb")) + mdb, err := dm.GetDatabase("testdb") + require.NoError(t, err) + _, err = mdb.GetDB().Exec("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)") + require.NoError(t, err) + + ctx := context.Background() + const txnID = 9001 + prep := engine.Prepare(ctx, &PrepareRequest{ + TxnID: txnID, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: 1, Logical: 1}, + Database: "testdb", + Statements: []protocol.Statement{ + testProtocolDMLStatement(protocol.StatementInsert, "testdb", "users", []byte("users:1"), nil, map[string][]byte{ + "id": []byte("1"), "name": []byte("a"), + }), + testProtocolDMLStatement(protocol.StatementInsert, "testdb", "users", []byte("users:2"), nil, map[string][]byte{ + "id": []byte("2"), "name": []byte("b"), + }), + testProtocolDMLStatement(protocol.StatementUpdate, "testdb", "users", []byte("users:1"), + map[string][]byte{"id": []byte("1"), "name": []byte("a")}, + map[string][]byte{"id": []byte("1"), "name": []byte("a2")}), + }, + }) + require.True(t, prep.Success, prep.Error) + commit := engine.Commit(ctx, &CommitRequest{TxnID: txnID, Database: "testdb"}) + require.True(t, commit.Success, commit.Error) + + assertRowCountMatchesCaptured(t, mdb.GetMetaStore(), txnID) + }) + + t.Run("2PC DDL, two statements", func(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + require.NoError(t, dm.CreateDatabase("testdb")) + mdb, err := dm.GetDatabase("testdb") + require.NoError(t, err) + + ctx := context.Background() + const txnID = 9002 + prep := engine.Prepare(ctx, &PrepareRequest{ + TxnID: txnID, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: 1, Logical: 1}, + Database: "testdb", + Statements: []protocol.Statement{ + {Type: protocol.StatementDDL, TableName: "a", SQL: "CREATE TABLE a (id INTEGER PRIMARY KEY)"}, + {Type: protocol.StatementDDL, TableName: "b", SQL: "CREATE TABLE b (id INTEGER PRIMARY KEY)"}, + }, + }) + require.True(t, prep.Success, prep.Error) + commit := engine.Commit(ctx, &CommitRequest{TxnID: txnID, Database: "testdb"}) + require.True(t, commit.Success, commit.Error) + + assertRowCountMatchesCaptured(t, mdb.GetMetaStore(), txnID) + }) + + t.Run("LOAD DATA", func(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + require.NoError(t, dm.CreateDatabase("testdb")) + mdb, err := dm.GetDatabase("testdb") + require.NoError(t, err) + _, err = mdb.GetDB().Exec("CREATE TABLE bulk (id INTEGER PRIMARY KEY, v TEXT)") + require.NoError(t, err) + + ctx := context.Background() + const txnID = 9003 + prep := engine.Prepare(ctx, &PrepareRequest{ + TxnID: txnID, + NodeID: 1, + StartTS: hlc.Timestamp{WallTime: 1, Logical: 1}, + Database: "testdb", + Statements: []protocol.Statement{ + { + Type: protocol.StatementLoadData, + TableName: "bulk", + SQL: "LOAD DATA LOCAL INFILE 'x' INTO TABLE bulk FIELDS TERMINATED BY ','", + LoadDataPayload: []byte("1,a\n2,b\n"), + }, + }, + }) + require.True(t, prep.Success, prep.Error) + commit := engine.Commit(ctx, &CommitRequest{TxnID: txnID, Database: "testdb"}) + require.True(t, commit.Success, commit.Error) + + assertRowCountMatchesCaptured(t, mdb.GetMetaStore(), txnID) + }) + + t.Run("AUTO_INCREMENT claim, zero rows", func(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + seedClaimBase(t, dm, "testdb", "users", 1000, 1) + mdb, err := dm.GetDatabase("testdb") + require.NoError(t, err) + + const txnID = 9004 + prep := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, + NodeID: 7, + StartTS: hlc.Timestamp{WallTime: 1}, + Database: "testdb", + Statements: []protocol.Statement{claimStatement(t, "testdb", "users", 1000, 1000, 64)}, + }) + require.True(t, prep.Success, prep.Error) + commit := engine.Commit(context.Background(), &CommitRequest{ + TxnID: txnID, + Database: "testdb", + Statements: []protocol.Statement{commitStatement("testdb", "users")}, + }) + require.True(t, commit.Success, commit.Error) + + rec, err := mdb.GetMetaStore().GetTransaction(txnID) + require.NoError(t, err) + require.NotNil(t, rec) + require.Equal(t, uint32(0), rec.RowCount, "a claim-only commit must record zero rows - claim payloads never travel in CDC") + assertRowCountMatchesCaptured(t, mdb.GetMetaStore(), txnID) + }) + + // Not covered: the batch-committer DML path, and vector-index control. + // + // Batch-committer path: CommitTransaction sets txn.Statements via the + // same tm.rebuildStatementsFromCDC(cdcEntries, nil) call whether or not + // tm.batchCommitEnabled() (db/transaction.go:316,328), and RowCount is + // always len(txn.Statements) (finalizeCommit). Captured rows are written + // during PREPARE either way (createDMLIntent), not at COMMIT, so the + // count this test checks is structurally identical on both paths; no + // test harness in this package wires a batch-commit-enabled database, so + // exercising it here would only re-run the same shared line. + // + // Vector-index control: driving StatementCreateVectorIndex through + // ReplicationEngine.Prepare+Commit needs a VectorIndexManager wired into + // the DatabaseManager and its own schema; no existing test in this + // package sets that up for a real 2PC commit, and building that harness + // is out of scope here (instructions allow skipping it "if testable + // without heavy setup"). +} + +// assertRowCountMatchesCaptured is the RowCount contract check itself: it fails +// with file:line detail the moment a commit kind's recorded RowCount and its +// actual captured-row count diverge, rather than weakening the comparison. +func assertRowCountMatchesCaptured(t *testing.T, metaStore MetaStore, txnID uint64) { + t.Helper() + rec, err := metaStore.GetTransaction(txnID) + require.NoError(t, err) + require.NotNil(t, rec) + + cursor, err := metaStore.IterateCapturedRows(txnID) + require.NoError(t, err) + defer cursor.Close() + var n uint32 + for cursor.Next() { + n++ + } + require.NoError(t, cursor.Err()) + + require.Equal(t, rec.RowCount, n, + "txn %d: commit record RowCount=%d but IterateCapturedRows yields %d rows - FetchTransactions would refuse this transaction", + txnID, rec.RowCount, n) +} diff --git a/db/transaction_schema_reload_test.go b/db/transaction_schema_reload_test.go index fe0b048c..b85e88ea 100644 --- a/db/transaction_schema_reload_test.go +++ b/db/transaction_schema_reload_test.go @@ -54,6 +54,19 @@ func newSchemaReloadFaultDB(t *testing.T) (faultyDB *sql.DB, dbPath string) { // propagate the error and fail the DDL apply instead. func TestApplyNonDMLIntents_PropagatesSchemaReloadFailure(t *testing.T) { db, dbPath := newSchemaReloadFaultDB(t) + + // applyNonDMLIntents needs __marmot_applied_txn and + // __marmot_schema_version to already exist, as NewReplicatedDatabase + // would create them. Create them through a separate, unfaulty connection + // to the same file so this doesn't consume one of the two connections the + // faulty driver below deliberately limits this test to. + setupDB, err := sql.Open(SQLiteDriverName, dbPath) + require.NoError(t, err) + require.NoError(t, ensureAppliedTxnTable(setupDB)) + _, err = ensureSchemaVersionTable(setupDB, "testdb", false, nil) + require.NoError(t, err) + require.NoError(t, setupDB.Close()) + schemaCache := NewSchemaCache() tm := NewTransactionManager(db, nil, hlc.NewClock(1), schemaCache) @@ -64,11 +77,12 @@ func TestApplyNonDMLIntents_PropagatesSchemaReloadFailure(t *testing.T) { }, } - // The DDL exec itself is the first connection (succeeds); the schema - // reload that follows needs a second, fresh connection (fails). Every - // connection through this faulty driver after the first fails, so this - // is the last operation this test can perform against `db` itself. - err := tm.applyNonDMLIntents(1, intents) + // The whole non-DML tx (DDL exec, marker, schema-version bump, commit) is + // the first connection (succeeds); the schema reload that follows needs a + // second, fresh connection (fails). Every connection through this faulty + // driver after the first fails, so this is the last operation this test + // can perform against `db` itself. + err = tm.applyNonDMLIntents(1, hlc.Timestamp{}, intents) require.Error(t, err, "a failed post-DDL schema reload must fail the DDL apply, not be silently logged and swallowed") assert.Contains(t, err.Error(), "reload schema cache") @@ -90,12 +104,23 @@ func TestApplyNonDMLIntents_PropagatesSchemaReloadFailure(t *testing.T) { // list (e.g. only LOAD DATA) does not attempt a schema reload at all, so it // is unaffected by this fix. func TestApplyNonDMLIntents_NoDDLNeverReloads(t *testing.T) { - db, _ := newSchemaReloadFaultDB(t) + db, dbPath := newSchemaReloadFaultDB(t) + + // The marker table must exist for the marker-only tx below to succeed; see + // TestApplyNonDMLIntents_PropagatesSchemaReloadFailure for why this uses a + // separate, unfaulty connection instead of consuming the one the faulty + // driver allows. + setupDB, err := sql.Open(SQLiteDriverName, dbPath) + require.NoError(t, err) + require.NoError(t, ensureAppliedTxnTable(setupDB)) + require.NoError(t, setupDB.Close()) + schemaCache := NewSchemaCache() tm := NewTransactionManager(db, nil, hlc.NewClock(1), schemaCache) // No intents at all: hasDDL stays false, so reloadSchemaCache must never - // be called, and the second (failing) connection must never be needed. - err := tm.applyNonDMLIntents(1, nil) + // be called, and the second (failing) connection must never be needed - + // only the marker-only tx's own (first, successful) connection is used. + err = tm.applyNonDMLIntents(1, hlc.Timestamp{}, nil) require.NoError(t, err) } diff --git a/db/transaction_schema_version_test.go b/db/transaction_schema_version_test.go index e20b654c..09269a53 100644 --- a/db/transaction_schema_version_test.go +++ b/db/transaction_schema_version_test.go @@ -23,6 +23,7 @@ func TestTransactionRecord_SchemaVersionPersistence(t *testing.T) { db, err := sql.Open("sqlite3", dbPath) require.NoError(t, err) defer db.Close() + require.NoError(t, ensureAppliedTxnTable(db)) // Create test table _, err = db.Exec("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)") @@ -90,6 +91,7 @@ func TestTransactionRecord_SchemaVersionZeroDefault(t *testing.T) { db, err := sql.Open("sqlite3", dbPath) require.NoError(t, err) defer db.Close() + require.NoError(t, ensureAppliedTxnTable(db)) // Create test table _, err = db.Exec("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)") @@ -153,6 +155,7 @@ func TestStreamCommittedTransactions_IncludesSchemaVersion(t *testing.T) { db, err := sql.Open("sqlite3", dbPath) require.NoError(t, err) defer db.Close() + require.NoError(t, ensureAppliedTxnTable(db)) // Create test table _, err = db.Exec("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)") diff --git a/docs/src/pages/operations.mdx b/docs/src/pages/operations.mdx index d4bd835f..fb5d1ef2 100644 --- a/docs/src/pages/operations.mdx +++ b/docs/src/pages/operations.mdx @@ -178,6 +178,58 @@ curl -X POST -H "X-Marmot-Secret: $SECRET" http://node1:8080/admin/cluster/remov # 5-node cluster → 3-node cluster, quorum: 3 → 2 ``` +## Rolling Upgrades + +This release replaces how nodes catch up with each other: every node now +pulls the commit log of every peer, and it keeps a per-database schema version +inside each database file. Releases before it count a database's DDL history +differently (their counter also counts CREATE DATABASE and DROP DATABASE), so +the same history yields a different schema version on the two kinds of node. +Because of this, DDL and CREATE/DROP DATABASE are **refused cluster-wide** — +with a retryable error (MySQL 1213) naming the node ids still to upgrade — for +as long as any cluster member runs an earlier release. Ordinary DML keeps +replicating normally throughout. + +**Procedure:** + +1. Upgrade every node in the cluster, one at a time, letting each rejoin as + ALIVE before moving to the next. +2. Once every node runs this release, DDL and CREATE/DROP DATABASE are + accepted again. +3. Do not run DDL or CREATE/DROP DATABASE while the upgrade is in progress — + it will be refused, not queued. + +A database dropped on some nodes but not others before the upgrade started +(a node that missed a DROP DATABASE while it was down) is a divergence that +database-set reconciliation does not repair after the upgrade either; resolve +it before or after the upgrade like any other registry mismatch between nodes. + +**Snapshot restores during the upgrade.** A snapshot taken on a node that has +not been upgraded yet carries no schema version in its database file, so a +node restoring from it takes the version it recorded for that database before +its own upgrade. This is another reason DDL waits until every node is +upgraded: until then, every node's recorded version comes from the same DDL +history and they agree. + +## Log Retention and Membership + +Each node keeps its own commit log so peers can pull what they missed. An +entry is deleted by GC once it is older than `gc_min_retention_hours` **and** +every current cluster member has pulled past it, or unconditionally once it is +older than `gc_max_retention_hours`. + +"Every current member" includes members that are down (SUSPECT or DEAD), not +only the ones that are reachable: + +- While a member is down, GC keeps every entry it has not pulled yet, up to + `gc_max_retention_hours`. Plan disk space for that much log on every node. +- A member down longer than `gc_max_retention_hours` finds that its peers' + logs no longer cover the point it stopped at. When it returns, anti-entropy + restores the affected databases from a peer's snapshot automatically, then + resumes pulling logs. +- A node removed through `/admin/cluster/remove` stops counting at once: its + pull positions are dropped, and it no longer holds back GC on any node. + ## Prometheus Metrics ```toml diff --git a/grpc/anti_entropy.go b/grpc/anti_entropy.go index ed9a17fa..bb877da5 100644 --- a/grpc/anti_entropy.go +++ b/grpc/anti_entropy.go @@ -3,39 +3,51 @@ package grpc import ( "context" "fmt" - "strings" "sync" "time" "github.com/maxpert/marmot/cfg" "github.com/maxpert/marmot/db" - "github.com/maxpert/marmot/hlc" "github.com/maxpert/marmot/telemetry" "github.com/rs/zerolog/log" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -// AntiEntropyService manages background anti-entropy process -// It periodically checks for lagging peers and brings them up to date +// AntiEntropyService periodically reconciles this node's databases and +// their local commit logs with every alive cluster member: it merges each +// peer's database +// registry, pulls each database's log through a LogPuller with a +// per-pair deadline, and falls back to a full snapshot plus local-log +// re-apply when a peer's log has been truncated past this node's cursor. type AntiEntropyService struct { - nodeID uint64 - registry *NodeRegistry - client *Client - dbManager *db.DatabaseManager - deltaSync *DeltaSyncClient - clock *hlc.Clock - snapshotFunc SnapshotTransferFunc - schemaVersionMgr *db.SchemaVersionManager + nodeID uint64 + registry *NodeRegistry + client *Client + dbManager *db.DatabaseManager + logPuller *LogPuller + snapshotFunc SnapshotTransferFunc // Configuration - interval time.Duration - deltaThresholdTxns int - deltaThresholdSeconds int - enabled bool + interval time.Duration + enabled bool + + // snapshotRestoreTimeout bounds the whole restore sequence + // (restoreContext), independent of interval: a restore + // transfers a database's full current size and can run far longer than + // one anti-entropy round. + snapshotRestoreTimeout time.Duration // Control stopCh chan struct{} running bool mu sync.Mutex + + // Per-database status as of the last completed round, + // read by admin and tests. + statusMu sync.RWMutex + caughtUp map[string]bool + promotionReady map[string]bool } // SnapshotTransferFunc initiates a snapshot transfer to a peer @@ -44,36 +56,45 @@ type SnapshotTransferFunc func(ctx context.Context, peerNodeID uint64, peerAddr // AntiEntropyConfig holds configuration for anti-entropy service type AntiEntropyConfig struct { - NodeID uint64 - Registry *NodeRegistry - Client *Client - DBManager *db.DatabaseManager - DeltaSync *DeltaSyncClient - Clock *hlc.Clock - SnapshotFunc SnapshotTransferFunc - SchemaVersionMgr *db.SchemaVersionManager - Interval time.Duration - DeltaThresholdTxns int - DeltaThresholdSeconds int - Enabled bool + NodeID uint64 + Registry *NodeRegistry + Client *Client + DBManager *db.DatabaseManager + LogPuller *LogPuller + SnapshotFunc SnapshotTransferFunc + Interval time.Duration + Enabled bool + // SnapshotRestoreTimeout bounds a snapshot restore's own deadline, + // independent of Interval. Zero or negative falls back to + // defaultSnapshotRestoreTimeout. + SnapshotRestoreTimeout time.Duration } +// defaultSnapshotRestoreTimeout is used when AntiEntropyConfig does not name +// one - direct construction (tests, embedded use) rather than +// NewAntiEntropyServiceFromConfig, which always sets it from +// cfg.Config.Replication.SnapshotRestoreTimeoutS. +const defaultSnapshotRestoreTimeout = 30 * time.Minute + // NewAntiEntropyService creates a new anti-entropy service func NewAntiEntropyService(config AntiEntropyConfig) *AntiEntropyService { + restoreTimeout := config.SnapshotRestoreTimeout + if restoreTimeout <= 0 { + restoreTimeout = defaultSnapshotRestoreTimeout + } return &AntiEntropyService{ - nodeID: config.NodeID, - registry: config.Registry, - client: config.Client, - dbManager: config.DBManager, - deltaSync: config.DeltaSync, - clock: config.Clock, - snapshotFunc: config.SnapshotFunc, - schemaVersionMgr: config.SchemaVersionMgr, - interval: config.Interval, - deltaThresholdTxns: config.DeltaThresholdTxns, - deltaThresholdSeconds: config.DeltaThresholdSeconds, - enabled: config.Enabled, - stopCh: make(chan struct{}), + nodeID: config.NodeID, + registry: config.Registry, + client: config.Client, + dbManager: config.DBManager, + logPuller: config.LogPuller, + snapshotFunc: config.SnapshotFunc, + interval: config.Interval, + enabled: config.Enabled, + snapshotRestoreTimeout: restoreTimeout, + stopCh: make(chan struct{}), + caughtUp: make(map[string]bool), + promotionReady: make(map[string]bool), } } @@ -83,29 +104,40 @@ func NewAntiEntropyServiceFromConfig( registry *NodeRegistry, client *Client, dbManager *db.DatabaseManager, - deltaSync *DeltaSyncClient, - clock *hlc.Clock, + logPuller *LogPuller, snapshotFunc SnapshotTransferFunc, - schemaVersionMgr *db.SchemaVersionManager, ) *AntiEntropyService { config := cfg.Config.Replication return NewAntiEntropyService(AntiEntropyConfig{ - NodeID: nodeID, - Registry: registry, - Client: client, - DBManager: dbManager, - DeltaSync: deltaSync, - Clock: clock, - SnapshotFunc: snapshotFunc, - SchemaVersionMgr: schemaVersionMgr, - Interval: time.Duration(config.AntiEntropyIntervalS) * time.Second, - DeltaThresholdTxns: config.DeltaSyncThresholdTxns, - DeltaThresholdSeconds: config.DeltaSyncThresholdSeconds, - Enabled: config.EnableAntiEntropy, + NodeID: nodeID, + Registry: registry, + Client: client, + DBManager: dbManager, + LogPuller: logPuller, + SnapshotFunc: snapshotFunc, + Interval: time.Duration(config.AntiEntropyIntervalS) * time.Second, + Enabled: config.EnableAntiEntropy, + SnapshotRestoreTimeout: time.Duration(config.SnapshotRestoreTimeoutS) * time.Second, }) } +// restoreContext returns a context bounded by snapshotRestoreTimeout, used by +// every call site that runs a snapshot restore (syncDatabase's fallback and +// restoreAwaitingDatabases): deliberately not ae.interval, which bounds one +// anti-entropy round, not a one-shot transfer of a database's full current +// size. +func (ae *AntiEntropyService) restoreContext() (context.Context, context.CancelFunc) { + timeout := ae.snapshotRestoreTimeout + if timeout <= 0 { + // A zero value here means an AntiEntropyService built as a struct + // literal rather than through NewAntiEntropyService (some tests do + // this) - fall back rather than expire the context immediately. + timeout = defaultSnapshotRestoreTimeout + } + return context.WithTimeout(context.Background(), timeout) +} + // Start starts the anti-entropy background process func (ae *AntiEntropyService) Start() { ae.mu.Lock() @@ -128,8 +160,6 @@ func (ae *AntiEntropyService) Start() { log.Info(). Dur("interval", ae.interval). - Int("delta_threshold_txns", ae.deltaThresholdTxns). - Int("delta_threshold_seconds", ae.deltaThresholdSeconds). Msg("Anti-entropy service started") } @@ -167,31 +197,23 @@ func (ae *AntiEntropyService) runLoop() { } } -// runStartupSync performs anti-entropy at startup, waiting for peers if needed +// runStartupSync performs anti-entropy at startup, waiting briefly for peers +// if needed, and always runs one round itself before returning: a JOINING +// node's first round must not wait a full ae.interval for the ticker +// (the promotion gate - checkPromotionCriteria needs a completed +// round to see PromotionReady at all). func (ae *AntiEntropyService) runStartupSync() { - log.Debug(). - Uint64("node_id", ae.nodeID). - Str("timestamp", time.Now().Format("15:04:05.000")). - Msg("ANTI-ENTROPY: Running startup sync") - // Try up to 5 times with 2 second delays to find ALIVE peers // This gives gossip time to propagate membership after a restart maxAttempts := 5 for attempt := 1; attempt <= maxAttempts; attempt++ { - nodes := ae.registry.GetAll() - aliveCount := 0 - for _, node := range nodes { - if node.Status == NodeStatus_ALIVE && node.NodeId != ae.nodeID { - aliveCount++ - } - } + _, alive := ae.currentMembers() - if aliveCount > 0 { + if len(alive) > 0 { log.Debug(). Uint64("node_id", ae.nodeID). - Int("alive_peers", aliveCount). + Int("alive_peers", len(alive)). Int("attempt", attempt). - Str("timestamp", time.Now().Format("15:04:05.000")). Msg("ANTI-ENTROPY: Found ALIVE peers, starting sync") ae.performAntiEntropy() return @@ -201,7 +223,6 @@ func (ae *AntiEntropyService) runStartupSync() { Uint64("node_id", ae.nodeID). Int("attempt", attempt). Int("max_attempts", maxAttempts). - Str("timestamp", time.Now().Format("15:04:05.000")). Msg("ANTI-ENTROPY: No ALIVE peers found, waiting for gossip") // Wait for gossip to discover peers (unless this is the last attempt) @@ -217,447 +238,382 @@ func (ae *AntiEntropyService) runStartupSync() { log.Debug(). Uint64("node_id", ae.nodeID). - Str("timestamp", time.Now().Format("15:04:05.000")). - Msg("ANTI-ENTROPY: No ALIVE peers after startup wait, will sync on next interval") + Msg("ANTI-ENTROPY: No ALIVE peers found within the startup wait, running the first round anyway") + // performAntiEntropy is a cheap no-op with no alive peers; running it now + // still means the first round runs at Start(), not after waiting a full + // ae.interval on the ticker. + ae.performAntiEntropy() } -// performAntiEntropy performs a single anti-entropy round +// currentMembers returns two views of this node's cluster membership, +// self excluded: members is every registry +// node whose status is not REMOVED (LEAVING, DEAD and SUSPECT are members; +// only ALIVE ones are contacted), and alive is the subset of those that are +// ALIVE. +func (ae *AntiEntropyService) currentMembers() (members, alive []*NodeState) { + for _, node := range ae.registry.GetAll() { + if node.NodeId == ae.nodeID || node.Status == NodeStatus_REMOVED { + continue + } + members = append(members, node) + if node.Status == NodeStatus_ALIVE { + alive = append(alive, node) + } + } + return members, alive +} + +// performAntiEntropy performs a single anti-entropy round. Every network +// step gets its own deadline: no pair, and no peer's registry listing, +// shares one with another, so one slow peer cannot starve the rest. func (ae *AntiEntropyService) performAntiEntropy() { roundStart := time.Now() telemetry.AntiEntropyRoundsTotal.Inc() - - ctx, cancel := context.WithTimeout(context.Background(), ae.interval) - defer cancel() defer func() { telemetry.AntiEntropyDurationSeconds.Observe(time.Since(roundStart).Seconds()) }() - log.Debug().Msg("Starting anti-entropy round") - - // Get all ALIVE nodes from registry - nodes := ae.registry.GetAll() - aliveNodes := make([]*NodeState, 0) - for _, node := range nodes { - if node.Status == NodeStatus_ALIVE && node.NodeId != ae.nodeID { - aliveNodes = append(aliveNodes, node) - } - } - - if len(aliveNodes) == 0 { - log.Debug().Msg("No alive peers to sync with") + members, alive := ae.currentMembers() + if len(alive) == 0 { + log.Debug().Msg("Anti-entropy: no alive peers to sync with") return } log.Debug(). Uint64("node_id", ae.nodeID). - Int("peer_count", len(aliveNodes)). - Str("timestamp", time.Now().Format("15:04:05.000")). + Int("member_count", len(members)). + Int("alive_count", len(alive)). Msg("ANTI-ENTROPY: Starting round") - // For each database, find the peer with highest max_txn_id and sync ONLY from that peer - // This prevents race conditions from concurrent snapshot downloads - databases := ae.dbManager.ListDatabases() - log.Debug(). - Uint64("node_id", ae.nodeID). - Int("database_count", len(databases)). - Strs("databases", databases). - Msg("ANTI-ENTROPY: Checking databases for sync") + // Reconcile the database set with every alive peer before pulling any + // log, so a database this node missed CREATE for exists before its log + // is ever pulled. + ae.reconcileDatabaseRegistry(alive) - for _, dbName := range databases { - bestPeer := ae.findBestPeerForDatabase(ctx, aliveNodes, dbName) - if bestPeer != nil { - log.Debug(). - Uint64("node_id", ae.nodeID). - Uint64("best_peer", bestPeer.NodeId). - Str("database", dbName). - Msg("ANTI-ENTROPY: Found best peer, starting sync") - if err := ae.syncPeerDatabase(ctx, bestPeer, dbName); err != nil { - log.Warn(). - Err(err). - Uint64("peer_node", bestPeer.NodeId). - Str("database", dbName). - Msg("Failed to sync from best peer") - } - } else { - log.Debug(). - Uint64("node_id", ae.nodeID). - Str("database", dbName). - Msg("ANTI-ENTROPY: No better peer found (we may be up to date)") - } + for _, dbName := range ae.dbManager.ListDatabases() { + ae.syncDatabase(dbName, members, alive) } - ae.restoreAwaitingDatabases(ctx, aliveNodes) + ae.restoreAwaitingDatabases(alive) log.Debug().Msg("Anti-entropy round completed") } -// restoreAwaitingDatabases retries the snapshot restore of every database -// whose reattach after an earlier restore failed (DatabasesAwaitingRestore). -// Such a database is out of service and absent from ListDatabases, so the -// loop above never reaches it; each alive peer is tried in turn until one -// restore succeeds. -func (ae *AntiEntropyService) restoreAwaitingDatabases(ctx context.Context, aliveNodes []*NodeState) { - if ae.snapshotFunc == nil { - return - } - for _, dbName := range ae.dbManager.DatabasesAwaitingRestore() { - for _, peer := range aliveNodes { - err := ae.snapshotFunc(ctx, peer.NodeId, peer.Address, dbName) - if err == nil { - telemetry.AntiEntropySyncsTotal.With("snapshot", "success").Inc() - log.Info().Uint64("peer_node", peer.NodeId).Str("database", dbName). - Msg("Restored a database whose earlier reattach failed") - break - } - telemetry.AntiEntropySyncsTotal.With("snapshot", "failed").Inc() - log.Warn().Err(err).Uint64("peer_node", peer.NodeId).Str("database", dbName). - Msg("Restore of a database awaiting one failed") +// reconcileDatabaseRegistry merges every alive peer's database registry +// into this node's own (reconcileRegistryWithPeer). A peer that answers +// Unimplemented (rolling upgrade) has its registry skipped; every other +// per-peer error is logged and never aborts the round. +func (ae *AntiEntropyService) reconcileDatabaseRegistry(alive []*NodeState) { + for _, peer := range alive { + ctx, cancel := context.WithTimeout(context.Background(), ae.interval) + err := reconcileRegistryWithPeer(ctx, ae.client, ae.dbManager, ae.nodeID, peer.Address) + cancel() + if err == nil { + continue } + if status.Code(err) == codes.Unimplemented { + log.Warn().Uint64("peer_node", peer.NodeId). + Msg("anti-entropy: peer does not support database registry listing yet, skipping (rolling upgrade)") + continue + } + log.Debug().Err(err).Uint64("peer_node", peer.NodeId). + Msg("anti-entropy: database registry reconciliation failed") } } -// findBestPeerForDatabase finds the peer with highest max_txn_id or most transactions -// Uses two criteria: -// 1. Higher max_txn_id indicates more recent transactions -// 2. Higher committed_txn_count indicates more data (tiebreaker when max_txn_id is the same) -// CRITICAL: Skips peers with older schema versions to prevent data loss from DDL changes -// Returns nil if we're already up to date with all peers -func (ae *AntiEntropyService) findBestPeerForDatabase(ctx context.Context, aliveNodes []*NodeState, database string) *NodeState { - localMaxTxnID, err := ae.dbManager.GetMaxTxnID(database) +// reconcileRegistryWithPeer merges the database registry of the peer at +// address into dbMgr's own: a database this node +// missed CREATE for is created, and one this node still has that the peer +// already tombstoned is dropped - never the reverse, by DatabaseRegistryKey's +// total order (db.DatabaseManager.ApplyDatabaseOp). A failure to apply one +// entry is logged and does not stop the others; the returned error is the +// listing's own. +// +// A legacy live entry (see db.DatabaseManager.ApplyDatabaseOp's doc +// comment) is never applied when this node has no row at all for the name: +// that is the one case where applying it would CREATE the database, spreading +// a pre-upgrade divergence this node never had a chance to see for itself. +// Every other entry, legacy or not, is merged as usual. +func reconcileRegistryWithPeer(ctx context.Context, client *Client, dbMgr *db.DatabaseManager, nodeID uint64, address string) error { + conn, err := client.GetClientByAddress(address) if err != nil { - log.Warn().Err(err).Str("database", database).Msg("Failed to get local max txn_id") - localMaxTxnID = 0 + return err } - - localTxnCount, err := ae.dbManager.GetCommittedTxnCount(database) + resp, err := conn.ListDatabaseRegistry(ctx, &DatabaseRegistryRequest{RequestingNodeId: nodeID}) if err != nil { - log.Warn().Err(err).Str("database", database).Msg("Failed to get local txn count") - localTxnCount = 0 + return err } + for _, entry := range resp.Entries { + key := db.DatabaseRegistryKey{Generation: entry.Generation, Dropped: entry.Dropped} + if entry.Legacy && !entry.Dropped { + local, err := dbMgr.RegistryKey(entry.Name) + if err != nil { + log.Warn().Err(err).Str("peer", address).Str("database", entry.Name). + Msg("failed to read local database registry key") + continue + } + if local.Generation == 0 { + log.Debug().Str("peer", address).Str("database", entry.Name). + Msg("anti-entropy: skipping peer's legacy live registry row for a database this node never had (pre-upgrade divergence, not repaired)") + continue + } + } + if _, err := dbMgr.ApplyDatabaseOp(entry.Name, key); err != nil { + log.Warn().Err(err).Str("peer", address).Str("database", entry.Name). + Msg("failed to apply peer database registry entry") + } + } + return nil +} + +// snapshotSource is the alive member selected as a database's restore +// source: the one with the highest schema version among the members that +// answered this round, ties broken by the lowest node id. +type snapshotSource struct { + peer *NodeState + schemaVersion uint64 +} - var localSchemaVersion uint64 = 0 - if ae.schemaVersionMgr != nil { - localSchemaVersion, err = ae.schemaVersionMgr.GetSchemaVersion(database) +// considerSnapshotSource updates best with candidate if candidate outranks +// it (higher schema version, or the same version and a lower node id). +func considerSnapshotSource(best *snapshotSource, candidate snapshotSource) *snapshotSource { + if best == nil { + return &candidate + } + if candidate.schemaVersion > best.schemaVersion { + return &candidate + } + if candidate.schemaVersion == best.schemaVersion && candidate.peer.NodeId < best.peer.NodeId { + return &candidate + } + return best +} + +// syncDatabase runs one database's pull round against every alive peer, +// each pair under its own deadline, falls back +// to a snapshot restore when any member's log no longer covers this node's +// cursor into it, and records the resulting caught-up status. +func (ae *AntiEntropyService) syncDatabase(dbName string, members, alive []*NodeState) { + // Finish a pending re-apply of this node's own log, left by a + // restore that crashed or failed before completing it, before pulling + // anything more for dbName. + ae.reapplyIfPending(dbName) + + dbCaughtUp := len(alive) == len(members) + dbPromotionReady, needsSnapshot, source := ae.pullEveryAlivePeer(dbName, alive, &dbCaughtUp) + + if needsSnapshot { + dbCaughtUp = false + ctx, cancel := ae.restoreContext() + err := ae.restoreFromPeer(ctx, dbName, source.peer) + cancel() if err != nil { - log.Warn().Err(err).Str("database", database).Msg("Failed to get local schema version, treating as 0") - localSchemaVersion = 0 + telemetry.AntiEntropySyncsTotal.With("snapshot", "failed").Inc() + log.Warn().Err(err).Str("database", dbName).Uint64("source_peer", source.peer.NodeId). + Msg("anti-entropy: snapshot fallback failed") + } else { + telemetry.AntiEntropySyncsTotal.With("snapshot", "success").Inc() } } - log.Debug(). - Uint64("node_id", ae.nodeID). - Str("database", database). - Uint64("local_max_txn", localMaxTxnID). - Int64("local_txn_count", localTxnCount). - Uint64("local_schema_version", localSchemaVersion). - Int("peers_to_check", len(aliveNodes)). - Msg("ANTI-ENTROPY: Comparing with peers") - - var bestPeer *NodeState - var bestMaxTxnID uint64 = localMaxTxnID - var bestTxnCount int64 = localTxnCount - - for _, peer := range aliveNodes { - peerSchemaVersion := uint64(0) - if peer.DatabaseSchemaVersions != nil { - if version, exists := peer.DatabaseSchemaVersions[database]; exists { - peerSchemaVersion = version - } - } + ae.setCaughtUp(dbName, dbCaughtUp) + ae.setPromotionReady(dbName, dbPromotionReady) +} - if peerSchemaVersion < localSchemaVersion { - log.Debug(). - Uint64("node_id", ae.nodeID). - Uint64("peer_node", peer.NodeId). - Str("database", database). - Uint64("local_schema_version", localSchemaVersion). - Uint64("peer_schema_version", peerSchemaVersion). - Msg("ANTI-ENTROPY: Skipping peer with older schema version") - continue - } +// pullEveryAlivePeer runs dbName's PullPair against every alive peer, each +// under its own deadline, and folds the results into three outputs: +// dbCaughtUp (an in-out pointer, cleared by any pair that did not fully +// succeed - it starts set from len(alive)==len(members)), promotionReady +// (see below) and the best snapshot source seen, for the +// needsSnapshot fallback. +// +// promotionReady is deliberately over alive peers only, never all members +// (checkPromotionCriteria): requiring CaughtUp over every member, DEAD ones +// included, would keep a restarted node JOINING forever while any member +// stays down. It starts true and is cleared by any alive peer's pair that +// did not fully succeed; it also requires at least one alive peer was +// actually pulled, so a node with no alive peers yet is never +// promotion-ready. +func (ae *AntiEntropyService) pullEveryAlivePeer(dbName string, alive []*NodeState, dbCaughtUp *bool) (promotionReady, needsSnapshot bool, source *snapshotSource) { + promotionReady = len(alive) > 0 + + for _, peer := range alive { + pairCtx, cancel := context.WithTimeout(context.Background(), ae.interval) + result, err := ae.logPuller.PullPair(pairCtx, PeerRef{NodeID: peer.NodeId, Address: peer.Address}, dbName) + cancel() - client, err := ae.client.GetClientByAddress(peer.Address) if err != nil { - log.Debug(). - Err(err). - Uint64("node_id", ae.nodeID). - Uint64("peer_node", peer.NodeId). - Msg("ANTI-ENTROPY: Failed to get client for peer") + log.Warn().Err(err).Uint64("peer_node", peer.NodeId).Str("database", dbName). + Msg("anti-entropy: pull failed") + *dbCaughtUp = false + promotionReady = false continue } - - resp, err := client.GetReplicationState(ctx, &ReplicationStateRequest{ - RequestingNodeId: ae.nodeID, - Database: database, - }) - if err != nil { - log.Debug(). - Err(err). - Uint64("node_id", ae.nodeID). - Uint64("peer_node", peer.NodeId). - Msg("ANTI-ENTROPY: Failed to get replication state from peer") + if result.Unimplemented || result.DatabaseAbsent { + // A rolling-upgrade peer, or a peer whose registry has not caught + // up yet: its log cannot count toward caught-up this round. + *dbCaughtUp = false + promotionReady = false continue } - - for _, state := range resp.States { - if state.DatabaseName != database { - continue - } - - peerMaxTxn := state.CurrentMaxTxnId - peerTxnCount := state.CommittedTxnCount - - log.Debug(). - Uint64("node_id", ae.nodeID). - Uint64("peer_node", peer.NodeId). - Str("state_db", state.DatabaseName). - Uint64("peer_max_txn", peerMaxTxn). - Int64("peer_txn_count", peerTxnCount). - Uint64("peer_schema_version", peerSchemaVersion). - Uint64("best_max_txn", bestMaxTxnID). - Int64("best_txn_count", bestTxnCount). - Msg("ANTI-ENTROPY: Checking peer state") - - // Select peer if: - // 1. Peer has higher max_txn_id, OR - // 2. Peer has same max_txn_id but more committed transactions (more data) - shouldSelect := peerMaxTxn > bestMaxTxnID || - (peerMaxTxn == bestMaxTxnID && peerTxnCount > bestTxnCount) - - if shouldSelect { - bestMaxTxnID = peerMaxTxn - bestTxnCount = peerTxnCount - bestPeer = peer - log.Debug(). - Uint64("node_id", ae.nodeID). - Uint64("new_best_peer", peer.NodeId). - Uint64("new_best_txn", bestMaxTxnID). - Int64("new_best_count", bestTxnCount). - Msg("ANTI-ENTROPY: Found better peer") - } + source = considerSnapshotSource(source, snapshotSource{peer: peer, schemaVersion: result.PeerSchemaVersion}) + if result.NeedsSnapshot { + needsSnapshot = true + promotionReady = false + } + if !result.CaughtUp { + *dbCaughtUp = false + promotionReady = false } } + return promotionReady, needsSnapshot, source +} - if bestPeer != nil { - log.Debug(). - Uint64("node_id", ae.nodeID). - Uint64("best_peer", bestPeer.NodeId). - Str("database", database). - Uint64("local_max_txn", localMaxTxnID). - Int64("local_txn_count", localTxnCount). - Uint64("peer_max_txn", bestMaxTxnID). - Int64("peer_txn_count", bestTxnCount). - Msg("ANTI-ENTROPY: Found best peer for sync") - } else { - log.Debug(). - Uint64("node_id", ae.nodeID). - Str("database", database). - Uint64("local_max_txn", localMaxTxnID). - Int64("local_txn_count", localTxnCount). - Msg("ANTI-ENTROPY: We have the most data, no sync needed") +// restoreFromPeer restores dbName from source's snapshot and then runs the +// rest of the restore sequence (restoreDatabase). +func (ae *AntiEntropyService) restoreFromPeer(ctx context.Context, dbName string, source *NodeState) error { + if ae.snapshotFunc == nil { + return fmt.Errorf("snapshot transfer not configured") } - - return bestPeer + return restoreDatabase(ctx, ae.dbManager, ae.logPuller, dbName, func(ctx context.Context) error { + return ae.snapshotFunc(ctx, source.NodeId, source.Address, dbName) + }) } -// syncPeerDatabase syncs a single database with a peer -// In a leaderless system, we need to compare our local max txn ID with the peer's -// If peer has more transactions, we pull from them -func (ae *AntiEntropyService) syncPeerDatabase(ctx context.Context, peer *NodeState, database string) error { - // Get peer's replication state (peer returns their max txn ID) - client, err := ae.client.GetClientByAddress(peer.Address) - if err != nil { - return fmt.Errorf("failed to connect to peer: %w", err) +// restoreDatabase is the one restore sequence, used by anti-entropy's snapshot fallback, its retry of databases awaiting +// a restore, and startup catch-up: +// 1. durably mark the re-apply of this node's own log pending, before the +// restore replaces the file, so a crash anywhere after it still re-applies; +// 2. restore; +// 3. re-apply every entry of this node's own log the restored file lacks, +// clearing the mark only on success - a failure leaves it for +// the next round (reapplyIfPending); +// 4. let every member's next pull move its cursor up to that member's +// truncation point (LogPuller.MarkRestored). +func restoreDatabase(ctx context.Context, dbMgr *db.DatabaseManager, lp *LogPuller, dbName string, restore func(context.Context) error) error { + if mdb, err := dbMgr.GetDatabase(dbName); err == nil { + if err := mdb.GetMetaStore().SetReapplyPending(true); err != nil { + return fmt.Errorf("mark local-log re-apply pending for %s: %w", dbName, err) + } } - - resp, err := client.GetReplicationState(ctx, &ReplicationStateRequest{ - RequestingNodeId: ae.nodeID, - Database: database, - }) + if err := restore(ctx); err != nil { + return err + } + mdb, err := dbMgr.GetDatabase(dbName) if err != nil { - return fmt.Errorf("failed to get replication state: %w", err) + return fmt.Errorf("database %s not available after restore: %w", dbName, err) } - - // Find the state for this database - var dbState *DatabaseReplicationState - for _, state := range resp.States { - if state.DatabaseName == database { - dbState = state - break - } + if err := mdb.GetMetaStore().SetReapplyPending(true); err != nil { + return fmt.Errorf("mark local-log re-apply pending for %s: %w", dbName, err) } - - if dbState == nil { - // Peer doesn't have this database yet - log.Debug(). - Uint64("peer_node", peer.NodeId). - Str("database", database). - Msg("Peer doesn't have replication state for database") - return nil + if applied, err := mdb.ReapplyLocalLogIfPending(ctx); err != nil { + log.Warn().Err(err).Str("database", dbName).Int("applied", applied). + Msg("local-log re-apply after restore failed; retried every anti-entropy round") } + lp.MarkRestored(dbName) + return nil +} - // Get our local max txn ID for comparison - localMaxTxnID, err := ae.dbManager.GetMaxTxnID(database) +// reapplyIfPending runs dbName's pending local-log re-apply, if its durable +// mark is set. +func (ae *AntiEntropyService) reapplyIfPending(dbName string) { + mdb, err := ae.dbManager.GetDatabase(dbName) if err != nil { - log.Warn().Err(err).Str("database", database).Msg("Failed to get local max txn_id") - localMaxTxnID = 0 + return } - - peerMaxTxnID := dbState.CurrentMaxTxnId - peerTxnCount := dbState.CommittedTxnCount - - // Get local committed txn count - localTxnCount, err := ae.dbManager.GetCommittedTxnCount(database) + ctx, cancel := context.WithTimeout(context.Background(), ae.interval) + defer cancel() + applied, err := mdb.ReapplyLocalLogIfPending(ctx) if err != nil { - log.Warn().Err(err).Str("database", database).Msg("Failed to get local txn count") - localTxnCount = 0 + log.Warn().Err(err).Str("database", dbName).Int("applied", applied). + Msg("anti-entropy: local-log re-apply failed, will retry next round") + return } + if applied > 0 { + log.Info().Str("database", dbName).Int("applied", applied). + Msg("anti-entropy: local-log re-apply completed after restore") + } +} - log.Debug(). - Uint64("node_id", ae.nodeID). - Uint64("peer_node", peer.NodeId). - Str("database", database). - Uint64("local_max_txn", localMaxTxnID). - Uint64("peer_max_txn", peerMaxTxnID). - Int64("local_txn_count", localTxnCount). - Int64("peer_txn_count", peerTxnCount). - Msg("ANTI-ENTROPY: Comparing with peer") - - // Determine if we need to sync from peer - // Primary metric: transaction COUNT (number of committed transactions) - // Secondary metric: max txn ID (HLC timestamp) as tiebreaker when counts are equal - // This avoids spurious syncs due to HLC drift between nodes - needsSync := peerTxnCount > localTxnCount || - (peerTxnCount == localTxnCount && peerMaxTxnID > localMaxTxnID) - - if needsSync { - // Use transaction count difference for lag calculation, NOT HLC difference - // HLC differences can be millions even for near-simultaneous transactions - txnCountDiff := peerTxnCount - localTxnCount - var lag uint64 - if txnCountDiff > 0 { - lag = uint64(txnCountDiff) - } - - log.Debug(). - Uint64("node_id", ae.nodeID). - Uint64("peer_node", peer.NodeId). - Str("database", database). - Uint64("lag_txns", lag). - Int64("txn_count_diff", txnCountDiff). - Msg("ANTI-ENTROPY: We are behind peer, pulling transactions") - - // If txn_count is the same but max_txn_id differs, we have divergent HLC timestamps - // This is normal in a leaderless system - different nodes may commit at slightly different times - // Delta sync should work fine since both nodes have the same number of transactions - // Only use snapshot if we can't determine which transactions we're missing - if lag == 0 && peerMaxTxnID > localMaxTxnID { - log.Debug(). - Uint64("node_id", ae.nodeID). - Uint64("peer_node", peer.NodeId). - Str("database", database). - Int64("local_txn_count", localTxnCount). - Int64("peer_txn_count", peerTxnCount). - Uint64("local_max_txn", localMaxTxnID). - Uint64("peer_max_txn", peerMaxTxnID). - Msg("ANTI-ENTROPY: Same txn count but different max HLC - using delta sync to verify") - - // Try delta sync - it will pull any transactions we're missing - // If there are no missing transactions, it will be a no-op - } - - // Calculate time lag based on last sync time - // If LastSyncTime is 0 (first sync), use 0 duration to avoid ~54 year lag - var timeLag time.Duration - if dbState.LastSyncTime > 0 { - timeLag = time.Since(time.Unix(0, dbState.LastSyncTime)) - } - - // Decide: delta sync or snapshot? - if ae.shouldUseDeltaSync(lag, timeLag) { - // Use delta sync - pull transactions from peer starting after our local max - log.Debug(). - Uint64("peer_node", peer.NodeId). - Str("database", database). - Uint64("from_txn_id", localMaxTxnID). - Msg("ANTI-ENTROPY: Using delta sync for catch-up") - - result, err := ae.deltaSync.SyncFromPeer( - ctx, - peer.NodeId, - peer.Address, - database, - localMaxTxnID, // Start from our local max, pull newer transactions - ) - if err != nil { - // Check if error is due to gap detection (missing transactions) - // If so, fall back to snapshot instead of failing - errMsg := err.Error() - if strings.Contains(errMsg, "gap detected") { - log.Warn(). - Err(err). - Uint64("peer_node", peer.NodeId). - Str("database", database). - Msg("Gap detected in delta sync, falling back to snapshot") - - // Try snapshot instead - if ae.snapshotFunc != nil { - if snapErr := ae.snapshotFunc(ctx, peer.NodeId, peer.Address, database); snapErr != nil { - telemetry.AntiEntropySyncsTotal.With("snapshot", "failed").Inc() - return fmt.Errorf("delta sync failed with gap (transactions GC'd) and snapshot fallback failed: %w", snapErr) - } - telemetry.AntiEntropySyncsTotal.With("snapshot", "success").Inc() - return nil - } - telemetry.AntiEntropySyncsTotal.With("delta", "failed").Inc() - return fmt.Errorf("delta sync failed with gap (transactions GC'd) but snapshot function not configured: %w", err) - } - // Other errors - fail normally - telemetry.AntiEntropySyncsTotal.With("delta", "failed").Inc() - return fmt.Errorf("delta sync failed: %w", err) - } - - telemetry.AntiEntropySyncsTotal.With("delta", "success").Inc() - telemetry.DeltaSyncTxnsTotal.Add(float64(result.TxnsApplied)) - log.Debug(). - Uint64("peer_node", peer.NodeId). - Str("database", database). - Int("txns_applied", result.TxnsApplied). - Msg("ANTI-ENTROPY: Delta sync completed") - } else { - // Use snapshot (full database transfer) - log.Debug(). - Uint64("peer_node", peer.NodeId). - Str("database", database). - Uint64("lag_txns", lag). - Msg("ANTI-ENTROPY: Lag too large, using snapshot") - - if ae.snapshotFunc != nil { - if err := ae.snapshotFunc(ctx, peer.NodeId, peer.Address, database); err != nil { - telemetry.AntiEntropySyncsTotal.With("snapshot", "failed").Inc() - return fmt.Errorf("snapshot transfer failed: %w", err) - } +// restoreAwaitingDatabases retries the snapshot restore of every database +// whose reattach after an earlier restore failed (DatabasesAwaitingRestore). +// Such a database is out of service and absent from ListDatabases, so the +// loop in performAntiEntropy never reaches it; each alive peer is tried in +// turn until one restore succeeds. +func (ae *AntiEntropyService) restoreAwaitingDatabases(alive []*NodeState) { + if ae.snapshotFunc == nil { + return + } + for _, dbName := range ae.dbManager.DatabasesAwaitingRestore() { + for _, peer := range alive { + ctx, cancel := ae.restoreContext() + err := ae.restoreFromPeer(ctx, dbName, peer) + cancel() + if err == nil { telemetry.AntiEntropySyncsTotal.With("snapshot", "success").Inc() - } else { - log.Warn().Msg("Snapshot function not configured, skipping snapshot transfer") + log.Info().Uint64("peer_node", peer.NodeId).Str("database", dbName). + Msg("Restored a database whose earlier reattach failed") + break } + telemetry.AntiEntropySyncsTotal.With("snapshot", "failed").Inc() + log.Warn().Err(err).Uint64("peer_node", peer.NodeId).Str("database", dbName). + Msg("Restore of a database awaiting one failed") } } +} - return nil +// setCaughtUp records dbName's caught-up status for the round just +// completed. +func (ae *AntiEntropyService) setCaughtUp(dbName string, caughtUp bool) { + ae.statusMu.Lock() + defer ae.statusMu.Unlock() + ae.caughtUp[dbName] = caughtUp +} + +// CaughtUp reports whether, in anti-entropy's last completed round for +// database, every current member's log was reachable and this node's +// cursor into it had reached that member's stable point. False before +// anti-entropy has run a round +// for database. +func (ae *AntiEntropyService) CaughtUp(database string) bool { + ae.statusMu.RLock() + defer ae.statusMu.RUnlock() + return ae.caughtUp[database] } -// shouldUseDeltaSync determines whether to use delta sync vs snapshot -// Returns true if lag is small enough for delta sync -func (ae *AntiEntropyService) shouldUseDeltaSync(lagTxns uint64, lagTime time.Duration) bool { - // Use delta sync if BOTH conditions are met: - // 1. Transaction lag is below threshold - // 2. Time lag is below threshold +// setPromotionReady records dbName's promotion-readiness for the round just +// completed (see syncDatabase's dbPromotionReady comment). +func (ae *AntiEntropyService) setPromotionReady(dbName string, ready bool) { + ae.statusMu.Lock() + defer ae.statusMu.Unlock() + ae.promotionReady[dbName] = ready +} - txnThreshold := uint64(ae.deltaThresholdTxns) - timeThreshold := time.Duration(ae.deltaThresholdSeconds) * time.Second +// PromotionReady reports whether database is ready for this node to be +// promoted from JOINING to ALIVE: true iff, in anti-entropy's +// last completed round, at least one alive peer was pulled and every alive +// peer's pair returned no error, was not Unimplemented or DatabaseAbsent, +// reported CaughtUp and did not need a snapshot. Unlike CaughtUp, this is +// deliberately evaluated over alive peers only, never every member: waiting +// for a DEAD member's log too would keep a restarted node JOINING forever +// while that member stays down. False before anti-entropy has run a round +// for database. +func (ae *AntiEntropyService) PromotionReady(database string) bool { + ae.statusMu.RLock() + defer ae.statusMu.RUnlock() + return ae.promotionReady[database] +} - return lagTxns < txnThreshold && lagTime < timeThreshold +// StuckTxnCount returns how many of database's transactions LogPuller +// currently reports STUCK on a deterministic replay failure. +func (ae *AntiEntropyService) StuckTxnCount(database string) int { + count := 0 + for _, t := range ae.logPuller.StuckTxns() { + if t.Database == database { + count++ + } + } + return count } // ForceSync forces an immediate anti-entropy round @@ -667,104 +623,29 @@ func (ae *AntiEntropyService) ForceSync() { ae.performAntiEntropy() } -// GetStats returns current anti-entropy statistics +// GetStats returns current anti-entropy statistics: overall service state +// plus, per database, whether it is caught up and how many of its +// transactions are stuck. func (ae *AntiEntropyService) GetStats() map[string]interface{} { ae.mu.Lock() - defer ae.mu.Unlock() - - return map[string]interface{}{ - "enabled": ae.enabled, - "running": ae.running, - "interval_seconds": ae.interval.Seconds(), - "delta_threshold_txns": ae.deltaThresholdTxns, - "delta_threshold_seconds": ae.deltaThresholdSeconds, - } -} + enabled := ae.enabled + running := ae.running + ae.mu.Unlock() -// RefreshPeerReplicationStates queries all alive peers for their replication state -// and updates local state. This ensures GC has fresh watermarks before making decisions. -// Called by GC before Phase 2 to prevent data loss from stale watermarks. -func (ae *AntiEntropyService) RefreshPeerReplicationStates(ctx context.Context) error { - if !ae.enabled { - return nil - } - - // Get all ALIVE nodes from registry - nodes := ae.registry.GetAll() - var alivePeers []*NodeState - for _, node := range nodes { - if node.Status == NodeStatus_ALIVE && node.NodeId != ae.nodeID { - alivePeers = append(alivePeers, node) - } - } - - if len(alivePeers) == 0 { - // No peers to query - single node cluster - return nil - } - - // Get all databases to query databases := ae.dbManager.ListDatabases() - if len(databases) == 0 { - return nil - } - - var refreshErrors []string - - // Query each peer for their replication state - for _, peer := range alivePeers { - client, err := ae.client.GetClientByAddress(peer.Address) - if err != nil { - refreshErrors = append(refreshErrors, fmt.Sprintf("peer %d: connection failed", peer.NodeId)) - continue - } - - for _, dbName := range databases { - resp, err := client.GetReplicationState(ctx, &ReplicationStateRequest{ - RequestingNodeId: ae.nodeID, - Database: dbName, - }) - if err != nil { - refreshErrors = append(refreshErrors, fmt.Sprintf("peer %d db %s: %v", peer.NodeId, dbName, err)) - continue - } - - // Update local state with peer's current max txn_id - for _, state := range resp.States { - if state.DatabaseName != dbName { - continue - } - - // Store the peer's current max txn_id as their replication state - // This is what we track for GC safe point calculation - repState := &db.ReplicationState{ - PeerNodeID: peer.NodeId, - DatabaseName: dbName, - LastAppliedTxnID: state.CurrentMaxTxnId, - LastAppliedTSWall: ae.clock.Now().WallTime, - LastAppliedTSLog: 0, - LastSyncTime: time.Now().UnixNano(), - SyncStatus: "REFRESHED", - } - - if err := ae.dbManager.UpdateReplicationState(repState); err != nil { - log.Warn(). - Err(err). - Uint64("peer_node", peer.NodeId). - Str("database", dbName). - Msg("Failed to update replication state during refresh") - } - } + dbStats := make(map[string]interface{}, len(databases)) + for _, dbName := range databases { + dbStats[dbName] = map[string]interface{}{ + "caught_up": ae.CaughtUp(dbName), + "promotion_ready": ae.PromotionReady(dbName), + "stuck_txns": ae.StuckTxnCount(dbName), } } - if len(refreshErrors) > 0 { - // Log but don't fail - we may have partial success - log.Warn(). - Strs("errors", refreshErrors). - Int("peer_count", len(alivePeers)). - Msg("Some peers failed during replication state refresh") + return map[string]interface{}{ + "enabled": enabled, + "running": running, + "interval_seconds": ae.interval.Seconds(), + "databases": dbStats, } - - return nil } diff --git a/grpc/anti_entropy_regression_test.go b/grpc/anti_entropy_regression_test.go new file mode 100644 index 00000000..6177dd4f --- /dev/null +++ b/grpc/anti_entropy_regression_test.go @@ -0,0 +1,502 @@ +package grpc + +import ( + "context" + "fmt" + "net" + "testing" + "time" + + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// aeRegNode is one in-process node for AntiEntropyService regression tests: +// a real db.DatabaseManager, a real CatchUpClient (for the snapshot +// fallback), and, once served, a real gRPC MarmotServiceServer - so a round +// exercises the exact wire path (ListCommittedLog, FetchTransactions, +// ListDatabaseRegistry, snapshot transfer) a production round uses. +// +// This duplicates a little of log_pull_harness_test.go's pullTestNode by +// design: that harness does not expose its DatabaseManager's data +// directory, which the snapshot fallback needs for CatchUpClient. +type aeRegNode struct { + id uint64 + dir string + dm *db.DatabaseManager + registry *NodeRegistry + client *Client + catchUp *CatchUpClient + addr string +} + +func newAERegNode(t *testing.T, id uint64, databases ...string) *aeRegNode { + t.Helper() + dir := t.TempDir() + dm, err := db.NewDatabaseManager(dir, id, hlc.NewClock(id)) + require.NoError(t, err) + t.Cleanup(func() { _ = dm.Close() }) + + for _, name := range databases { + require.NoError(t, dm.CreateDatabase(name)) + mdb, err := dm.GetDatabase(name) + require.NoError(t, err) + _, err = mdb.GetWriteDB().Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + require.NoError(t, mdb.ReloadSchema()) + } + + registry := NewNodeRegistry(id, "127.0.0.1:0") + catchUp := NewCatchUpClient(id, dir, registry, nil) + catchUp.SetDatabaseManager(dm) + + return &aeRegNode{id: id, dir: dir, dm: dm, registry: registry, client: NewClient(id), catchUp: catchUp} +} + +// serve starts n over real gRPC and records its listen address. +func (n *aeRegNode) serve(t *testing.T) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + server := &Server{nodeID: n.id, registry: NewNodeRegistry(n.id, listener.Addr().String())} + server.SetDatabaseManager(n.dm) + gs := grpc.NewServer() + RegisterMarmotServiceServer(gs, server) + go func() { _ = gs.Serve(listener) }() + t.Cleanup(gs.Stop) + n.addr = listener.Addr().String() +} + +// seed commits txnID (coordinated by origin, committed at wall) into +// database d through ReplicatedDatabase.ApplyReplayedTxn - the same path a +// real replay uses - inserting one row (rowID, v) into table t. +func (n *aeRegNode) seed(t *testing.T, d string, txnID, origin uint64, wall int64, rowID int64, v string) { + t.Helper() + mdb, err := n.dm.GetDatabase(d) + require.NoError(t, err) + + row := &db.EncodedCapturedRow{ + Table: "t", + Op: uint8(db.OpTypeInsert), + IntentKey: []byte(fmt.Sprintf("t:%d", rowID)), + NewValues: encodeSeedValues(t, map[string]interface{}{"id": rowID, "v": v}), + } + applied, err := mdb.ApplyReplayedTxn(context.Background(), &db.ReplayTxn{ + TxnID: txnID, + OriginNodeID: origin, + CommitTS: hlc.Timestamp{WallTime: wall, NodeID: origin}, + Rows: []*db.EncodedCapturedRow{row}, + }, true) + require.NoError(t, err) + require.True(t, applied) +} + +// truncateThrough forces d's GC to unconditionally delete every currently +// committed log entry (maxRetention=0), so ListCommittedLog reports a +// non-zero TruncatedThrough to any puller whose cursor is still behind it. +// Used to force a PullPair NeedsSnapshot outcome deterministically, without +// waiting out a real retention window. +func (n *aeRegNode) truncateThrough(t *testing.T, d string) { + t.Helper() + mdb, err := n.dm.GetDatabase(d) + require.NoError(t, err) + _, err = mdb.GetMetaStore().CleanupOldTransactionRecords(time.Hour, 0, db.LogPosition{}) + require.NoError(t, err) +} + +// rows reads table t of database d. +func (n *aeRegNode) rows(t *testing.T, d string) map[int64]string { + t.Helper() + mdb, err := n.dm.GetDatabase(d) + require.NoError(t, err) + rs, err := mdb.GetReadDB().Query("SELECT id, v FROM t") + require.NoError(t, err) + defer rs.Close() + + got := map[int64]string{} + for rs.Next() { + var id int64 + var v string + require.NoError(t, rs.Scan(&id, &v)) + got[id] = v + } + require.NoError(t, rs.Err()) + return got +} + +// antiEntropyFor builds a real AntiEntropyService for n, wired against n's +// own registry, client, database manager and CatchUpFromPeer. +func (n *aeRegNode) antiEntropyFor() *AntiEntropyService { + lp := NewLogPuller(LogPullerConfig{NodeID: n.id, Client: n.client, DBManager: n.dm}) + return NewAntiEntropyService(AntiEntropyConfig{ + NodeID: n.id, + Registry: n.registry, + Client: n.client, + DBManager: n.dm, + LogPuller: lp, + SnapshotFunc: n.catchUp.CatchUpFromPeer, + Interval: 30 * time.Second, + Enabled: true, + }) +} + +func addPeer(registry *NodeRegistry, id uint64, addr string, status NodeStatus) { + registry.Add(&NodeState{NodeId: id, Address: addr, Status: status, Incarnation: 1}) +} + +// TestAntiEntropySyncsTwoLaggingDatabasesInOneRound: every lagging database +// is pulled in the same round. An anti-entropy loop that picks one best peer +// per database and only pulls that one leaves a second lagging database +// unsynced until a later round. +func TestAntiEntropySyncsTwoLaggingDatabasesInOneRound(t *testing.T) { + peer := newAERegNode(t, 2, "db1", "db2") + peer.serve(t) + peer.seed(t, "db1", 101, 2, 1000, 1, "p1") + peer.seed(t, "db2", 201, 2, 1000, 1, "q1") + + local := newAERegNode(t, 1, "db1", "db2") + addPeer(local.registry, 2, peer.addr, NodeStatus_ALIVE) + + local.antiEntropyFor().performAntiEntropy() + + require.Equal(t, map[int64]string{1: "p1"}, local.rows(t, "db1"), "db1 was not synced in a single anti-entropy round") + require.Equal(t, map[int64]string{1: "q1"}, local.rows(t, "db2"), + "db2 was not synced in the same round as db1 (only-the-first-database-synced defect)") +} + +// TestAntiEntropyPullsEveryPeersLog: every peer's log is pulled. Syncing +// each database only from the peer with the highest max txn id never fetches +// a txn only another peer holds - one committed by a quorum that excluded the +// chosen peer. +func TestAntiEntropyPullsEveryPeersLog(t *testing.T) { + p2 := newAERegNode(t, 2, "app") + p2.serve(t) + p2.seed(t, "app", 1000, 2, 1000, 1, "a") + p2.seed(t, "app", 3000, 2, 3000, 3, "c") + p3 := newAERegNode(t, 3, "app") + p3.serve(t) + p3.seed(t, "app", 2000, 4, 2000, 2, "b") + + local := newAERegNode(t, 1, "app") + addPeer(local.registry, 2, p2.addr, NodeStatus_ALIVE) + addPeer(local.registry, 3, p3.addr, NodeStatus_ALIVE) + + ae := local.antiEntropyFor() + ae.performAntiEntropy() + + require.Equal(t, map[int64]string{1: "a", 2: "b", 3: "c"}, local.rows(t, "app"), + "a txn held only by the peer with the lower max txn id was not fetched") + require.True(t, ae.CaughtUp("app")) +} + +// TestAntiEntropyReconcilesMissedCreateAndDrop fails when anti-entropy never +// reconciles the database set: a database +// created on a peer while this node was gone is never created locally, and +// one dropped on a peer while this node was gone is never dropped locally. +func TestAntiEntropyReconcilesMissedCreateAndDrop(t *testing.T) { + peer := newAERegNode(t, 2, "shared") + peer.serve(t) + require.NoError(t, peer.dm.CreateDatabase("newdb")) + require.NoError(t, peer.dm.DropDatabase("shared")) + + local := newAERegNode(t, 1, "shared") + addPeer(local.registry, 2, peer.addr, NodeStatus_ALIVE) + require.NotContains(t, local.dm.ListDatabases(), "newdb") + + local.antiEntropyFor().performAntiEntropy() + + require.Contains(t, local.dm.ListDatabases(), "newdb", + "a database created on a peer was never picked up by registry reconciliation") + require.NotContains(t, local.dm.ListDatabases(), "shared", + "a database dropped on a peer was never dropped locally by registry reconciliation") +} + +// TestAntiEntropyRegistryReconciliationNeverResurrectsALocalDrop exercises +// the same reconciliation path against a peer whose registry is older than +// this node's own DROP: DatabaseRegistryKey's total order +// (db.DatabaseManager.ApplyDatabaseOp) must refuse the peer's stale, still- +// live key. +func TestAntiEntropyRegistryReconciliationNeverResurrectsALocalDrop(t *testing.T) { + peer := newAERegNode(t, 2, "shared") // peer never drops "shared" - a stale registry. + peer.serve(t) + + local := newAERegNode(t, 1, "shared") + require.NoError(t, local.dm.DropDatabase("shared")) + addPeer(local.registry, 2, peer.addr, NodeStatus_ALIVE) + + local.antiEntropyFor().performAntiEntropy() + + require.NotContains(t, local.dm.ListDatabases(), "shared", + "a peer with an older (still-live) registry resurrected a local drop") +} + +// TestAntiEntropyCaughtUpReflectsUnreachablePeers fails if CaughtUp does not +// account for every current member's log being reachable: it must be false while any member cannot be pulled from, and +// true once every current member has answered and been fully drained. +func TestAntiEntropyCaughtUpReflectsUnreachablePeers(t *testing.T) { + peer := newAERegNode(t, 2, "db") + peer.serve(t) + peer.seed(t, "db", 301, 2, 1000, 1, "a1") + + local := newAERegNode(t, 1, "db") + addPeer(local.registry, 2, peer.addr, NodeStatus_ALIVE) + + // Node 3 is ALIVE in the registry but its address refuses connections. + closedListener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + unreachableAddr := closedListener.Addr().String() + require.NoError(t, closedListener.Close()) + addPeer(local.registry, 3, unreachableAddr, NodeStatus_ALIVE) + + ae := local.antiEntropyFor() + ae.performAntiEntropy() + require.False(t, ae.CaughtUp("db"), "expected not caught up while a current member is unreachable") + require.Equal(t, map[int64]string{1: "a1"}, local.rows(t, "db"), "the reachable peer's row was not pulled") + + // Node 3 leaves membership entirely; every remaining member now answers + // and is fully drained. + require.NoError(t, local.registry.MarkRemoved(3)) + ae.performAntiEntropy() + require.True(t, ae.CaughtUp("db"), "expected caught up once every current member answered and was drained") +} + +// TestAntiEntropySnapshotFallbackReappliesLocalOnlyTxn exercises the +// snapshot fallback end to end: the peer's log has been truncated past +// this node's cursor, so anti-entropy restores from it, then re-applies a +// local-only transaction the source's snapshot lacks, from this node's own +// durable local log, and resets its pull cursor. +func TestAntiEntropySnapshotFallbackReappliesLocalOnlyTxn(t *testing.T) { + source := newAERegNode(t, 2, "app") + // Seed and immediately truncate a throwaway entry so the peer's + // TruncatedThrough moves past position zero - runs before the row we + // actually want the restore to carry, so only the throwaway is + // eligible for this unconditional (maxRetention=0) cleanup. + source.seed(t, "app", 1, 2, 1000, 99, "throwaway") + source.truncateThrough(t, "app") + source.seed(t, "app", 101, 2, 2000, 1, "fromsource") + source.serve(t) + + local := newAERegNode(t, 1, "app") + // A transaction local holds in its own durable log that the source + // never had (e.g. local replayed it from a third peer that has since + // been GC'd elsewhere) - restoring from source's snapshot alone would + // silently lose it. + local.seed(t, "app", 501, 1, 3000, 2, "localonly") + addPeer(local.registry, 2, source.addr, NodeStatus_ALIVE) + + ae := local.antiEntropyFor() + ae.performAntiEntropy() + + // The throwaway row's SQLite data was already durably committed on + // source before its log entry was truncated - GC only removes log + // metadata, never table data - so it legitimately survives in source's + // snapshot alongside "fromsource". + require.Equal(t, map[int64]string{1: "fromsource", 2: "localonly", 99: "throwaway"}, local.rows(t, "app"), + "the snapshot fallback did not carry both the source's rows and the re-applied local-only row") + + mdb, err := local.dm.GetDatabase("app") + require.NoError(t, err) + pending, err := mdb.GetMetaStore().ReapplyPending() + require.NoError(t, err) + require.False(t, pending, "the re-apply mark was not cleared after a successful re-apply") + + // The next round resumes the source's log from its truncation point + // instead of restoring again, and drains it. + ae.performAntiEntropy() + cursor, err := mdb.GetMetaStore().GetPullCursor(2) + require.NoError(t, err) + require.NotEqual(t, db.LogPosition{}, cursor, "the pull cursor for the restore's source was not moved past its truncation point") + require.True(t, ae.CaughtUp("app"), "the round after the restore did not drain the source's log") +} + +// TestPerformLogPullCatchesUpAStartingNodeAndReturns: startup catch-up of a +// node that has data merges the seed's registry (a database created while +// the node was down, with its DDL and rows in the seed's log), pulls every +// database's log, and returns. On 7a66596 the startup delta sync streamed +// StreamChanges, which never ends, so it could only return at its 5-minute +// deadline and then log.Fatal. +func TestPerformLogPullCatchesUpAStartingNodeAndReturns(t *testing.T) { + seed := newAERegNode(t, 2, "app") + seed.seed(t, "app", 1000, 2, 1000, 1, "missed-while-down") + require.NoError(t, seed.dm.CreateDatabase("newdb")) + newdb, err := seed.dm.GetDatabase("newdb") + require.NoError(t, err) + applied, err := newdb.ApplyReplayedTxn(context.Background(), &db.ReplayTxn{ + TxnID: 1100, OriginNodeID: 2, CommitTS: hlc.Timestamp{WallTime: 1100, NodeID: 2}, + Rows: []*db.EncodedCapturedRow{{Table: "t", Op: uint8(db.OpTypeDDL), DDLSQL: "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"}}, + }, true) + require.NoError(t, err) + require.True(t, applied) + require.NoError(t, newdb.ReloadSchema()) + seed.seed(t, "newdb", 1200, 2, 1200, 7, "created-while-down") + seed.serve(t) + + local := newAERegNode(t, 1, "app") + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: local.client, DBManager: local.dm}) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + start := time.Now() + require.NoError(t, local.catchUp.PerformLogPull(ctx, &CatchUpDecision{Strategy: DELTA_SYNC, PeerNodeID: 2, PeerAddr: seed.addr}, lp, local.client)) + require.Less(t, time.Since(start), 30*time.Second, "startup catch-up ran into its deadline instead of returning") + + require.Equal(t, map[int64]string{1: "missed-while-down"}, local.rows(t, "app")) + _, err = local.dm.GetDatabase("newdb") + require.NoError(t, err, "a database created while the node was down was not created at startup") + require.Equal(t, map[int64]string{7: "created-while-down"}, local.rows(t, "newdb"), + "a database created while the node was down was not created and pulled at startup") + self, ok := local.registry.Get(1) + require.True(t, ok) + require.Equal(t, NodeStatus_JOINING, self.Status, "a node the startup pull found behind was not marked JOINING") +} + +// markLegacy flips name's registry row to legacy=1 directly in n's system +// database, simulating a row that survived an upgrade from a release +// before generation-stamped registry rows +// (migrateDatabaseRegistrySchema) rather than one this test drove through +// db.DatabaseManager.CreateDatabase/DropDatabase - which always stamp +// legacy=0, since only a real migration ever produces a legacy row. +func (n *aeRegNode) markLegacy(t *testing.T, name string) { + t.Helper() + _, err := n.dm.GetSystemDatabase().GetDB().Exec( + "UPDATE __marmot_databases SET legacy = 1 WHERE name = ?", name) + require.NoError(t, err) +} + +// TestAntiEntropyLegacyLiveRowNeverCreatesOnAPeerThatLacksTheName: a +// registry row migrated from an older release's registry (legacy=1, live) +// must never CREATE the database on a peer that has no row for the name at +// all - reconciliation does not spread a pre-upgrade divergence (a node that missed a DROP DATABASE before the upgrade) cluster +// wide. An ordinary, non-legacy live row still reconciles normally. +func TestAntiEntropyLegacyLiveRowNeverCreatesOnAPeerThatLacksTheName(t *testing.T) { + peer := newAERegNode(t, 2) + require.NoError(t, peer.dm.CreateDatabase("legacyx")) + peer.markLegacy(t, "legacyx") + require.NoError(t, peer.dm.CreateDatabase("stampedx")) // ordinary, non-legacy live row + peer.serve(t) + + local := newAERegNode(t, 1) + addPeer(local.registry, 2, peer.addr, NodeStatus_ALIVE) + require.False(t, local.dm.DatabaseExists("legacyx")) + + local.antiEntropyFor().performAntiEntropy() + + require.False(t, local.dm.DatabaseExists("legacyx"), + "a peer's legacy live registry row created the database on a peer that never had the name") + require.True(t, local.dm.DatabaseExists("stampedx"), + "an ordinary (non-legacy) live registry row was not reconciled") +} + +// TestAntiEntropyLegacyRowStopsBeingLegacyAfterAStampedCreate: once a real, post-upgrade DROP followed by a real +// CREATE re-stamps a legacy row's name with a fresh generation, it is an +// ordinary (non-legacy) live registry key from then on, and reconciles +// normally even on a peer that never had the name at all - the one case +// reconcileRegistryWithPeer otherwise refuses for a legacy live row. +func TestAntiEntropyLegacyRowStopsBeingLegacyAfterAStampedCreate(t *testing.T) { + peer := newAERegNode(t, 2) + require.NoError(t, peer.dm.CreateDatabase("wasLegacy")) + peer.markLegacy(t, "wasLegacy") + require.NoError(t, peer.dm.DropDatabase("wasLegacy")) + require.NoError(t, peer.dm.CreateDatabase("wasLegacy")) // real, post-upgrade stamped re-create: clears legacy + peer.serve(t) + + local := newAERegNode(t, 1) // never had "wasLegacy" at all + addPeer(local.registry, 2, peer.addr, NodeStatus_ALIVE) + + local.antiEntropyFor().performAntiEntropy() + + require.True(t, local.dm.DatabaseExists("wasLegacy"), + "a stamped re-create that followed a legacy row was still treated as legacy and skipped") +} + +// TestCheckPromotionCriteriaRequiresPromotionReadyForEveryDatabase: matching +// schema versions is not enough to promote a JOINING node - a database can +// be current on schema while still missing rows anti-entropy has not pulled +// yet. checkPromotionCriteria must also require AntiEntropyService. +// PromotionReady for every local database. +func TestCheckPromotionCriteriaRequiresPromotionReadyForEveryDatabase(t *testing.T) { + peer := newAERegNode(t, 2, "pdb") + peer.seed(t, "pdb", 101, 2, 1000, 1, "unpulled") // no DDL: schema versions already match + peer.serve(t) + + local := newAERegNode(t, 1, "pdb") + addPeer(local.registry, 2, peer.addr, NodeStatus_ALIVE) + + clock := hlc.NewClock(1) + schemaVersionMgr := db.NewSchemaVersionManager(local.dm) + replicationHandler := NewReplicationHandler(1, local.dm, clock, schemaVersionMgr) + + server := &Server{ + nodeID: 1, + registry: local.registry, + dbManager: local.dm, + replicationHandler: replicationHandler, + } + ae := local.antiEntropyFor() + server.SetAntiEntropy(ae) + + require.False(t, server.checkPromotionCriteria(), + "expected promotion refused while a database is not PromotionReady, even with matching schema versions") + + ae.performAntiEntropy() // pulls the row; pdb becomes PromotionReady + + require.True(t, server.checkPromotionCriteria(), + "expected promotion once every local database is PromotionReady") +} + +// TestAntiEntropyStartRunsFirstRoundWithoutWaitingForATick: a JOINING +// node's first anti-entropy round +// runs promptly at Start(), not after a full ae.interval on the ticker: a +// long interval must not delay checkPromotionCriteria's first look at +// PromotionReady. +func TestAntiEntropyStartRunsFirstRoundWithoutWaitingForATick(t *testing.T) { + peer := newAERegNode(t, 2, "app") + peer.seed(t, "app", 1, 2, 1000, 1, "a") + peer.serve(t) + + local := newAERegNode(t, 1, "app") + addPeer(local.registry, 2, peer.addr, NodeStatus_ALIVE) + + ae := NewAntiEntropyService(AntiEntropyConfig{ + NodeID: 1, + Registry: local.registry, + Client: local.client, + DBManager: local.dm, + LogPuller: NewLogPuller(LogPullerConfig{NodeID: 1, Client: local.client, DBManager: local.dm}), + Interval: time.Hour, // long enough that the ticker must never fire during this test + Enabled: true, + }) + ae.Start() + t.Cleanup(ae.Stop) + + require.Eventually(t, func() bool { return ae.CaughtUp("app") }, 5*time.Second, 20*time.Millisecond, + "the first anti-entropy round did not run promptly at Start(); it appears to be waiting for the interval's first tick") +} + +// TestPerformLogPullLeavesACurrentNodeOutOfJoining: every startup of a node +// with data runs the log pull, so a node that is already current - the whole +// cluster restarted idle - must not be demoted to JOINING by it; it would +// then refuse to count toward quorums until promoted, and the first write +// after the restart would fail with "prepare quorum not achieved". +// +// Mutation: mark JOINING before the pull unconditionally. "a current node +// was marked JOINING" fires. +func TestPerformLogPullLeavesACurrentNodeOutOfJoining(t *testing.T) { + seed := newAERegNode(t, 2, "app") + seed.seed(t, "app", 1000, 2, 1000, 1, "both") + seed.serve(t) + + local := newAERegNode(t, 1, "app") + local.seed(t, "app", 1000, 2, 1000, 1, "both") + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: local.client, DBManager: local.dm}) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + require.NoError(t, local.catchUp.PerformLogPull(ctx, &CatchUpDecision{Strategy: DELTA_SYNC, PeerNodeID: 2, PeerAddr: seed.addr}, lp, local.client)) + + self, ok := local.registry.Get(1) + require.True(t, ok) + require.NotEqual(t, NodeStatus_JOINING, self.Status, "a current node was marked JOINING") +} diff --git a/grpc/anti_entropy_test.go b/grpc/anti_entropy_test.go index b38bc072..aa15e846 100644 --- a/grpc/anti_entropy_test.go +++ b/grpc/anti_entropy_test.go @@ -1,462 +1,233 @@ package grpc import ( - "context" "os" "testing" "time" ) -// TestAntiEntropyShouldUseDeltaSync tests the decision logic for delta vs snapshot -func TestAntiEntropyShouldUseDeltaSync(t *testing.T) { - ae := &AntiEntropyService{ - deltaThresholdTxns: 10000, - deltaThresholdSeconds: 3600, - } - - tests := []struct { - name string - lagTxns uint64 - lagTime time.Duration - expectDelta bool - }{ - { - name: "Small lag - use delta", - lagTxns: 100, - lagTime: 5 * time.Minute, - expectDelta: true, - }, - { - name: "Large transaction lag - use snapshot", - lagTxns: 15000, - lagTime: 10 * time.Minute, - expectDelta: false, - }, - { - name: "Large time lag - use snapshot", - lagTxns: 5000, - lagTime: 2 * time.Hour, - expectDelta: false, - }, - { - name: "At threshold - use delta", - lagTxns: 9999, - lagTime: 59 * time.Minute, - expectDelta: true, - }, - { - name: "Just over threshold - use snapshot", - lagTxns: 10001, - lagTime: 10 * time.Minute, - expectDelta: false, - }, - { - name: "Zero lag - use delta", - lagTxns: 0, - lagTime: 0, - expectDelta: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := ae.shouldUseDeltaSync(tt.lagTxns, tt.lagTime) - if result != tt.expectDelta { - t.Errorf("shouldUseDeltaSync(%d txns, %v) = %v, want %v", - tt.lagTxns, tt.lagTime, result, tt.expectDelta) - } - }) +// TestAntiEntropyRestoreContextOutlivesInterval: a snapshot restore's own +// deadline must be independent of ae.interval, which bounds one anti-entropy round, not a +// one-shot transfer of a database's full current size. With a short +// interval and a much longer configured restore timeout, the context +// restoreContext returns must reflect the restore timeout, not the interval. +func TestAntiEntropyRestoreContextOutlivesInterval(t *testing.T) { + ae := NewAntiEntropyService(AntiEntropyConfig{ + NodeID: 1, + Interval: time.Second, + SnapshotRestoreTimeout: time.Hour, + Enabled: true, + }) + + ctx, cancel := ae.restoreContext() + defer cancel() + + deadline, ok := ctx.Deadline() + if !ok { + t.Fatal("restoreContext returned a context with no deadline") + } + remaining := time.Until(deadline) + if remaining <= ae.interval { + t.Fatalf("restore context deadline (%s remaining) did not outlive ae.interval (%s); "+ + "it appears bounded by the anti-entropy round interval instead of SnapshotRestoreTimeout", + remaining, ae.interval) } } -// TestAntiEntropyGetStats tests stats reporting -func TestAntiEntropyGetStats(t *testing.T) { - ae := &AntiEntropyService{ - enabled: true, - running: false, - interval: 60 * time.Second, - deltaThresholdTxns: 10000, - deltaThresholdSeconds: 3600, - } - - stats := ae.GetStats() - - if enabled, ok := stats["enabled"].(bool); !ok || !enabled { - t.Errorf("Expected enabled=true, got %v", stats["enabled"]) - } - - if running, ok := stats["running"].(bool); !ok || running { - t.Errorf("Expected running=false, got %v", stats["running"]) - } - - if interval, ok := stats["interval_seconds"].(float64); !ok || interval != 60.0 { - t.Errorf("Expected interval_seconds=60, got %v", stats["interval_seconds"]) - } - - if threshold, ok := stats["delta_threshold_txns"].(int); !ok || threshold != 10000 { - t.Errorf("Expected delta_threshold_txns=10000, got %v", stats["delta_threshold_txns"]) - } -} - -// TestAntiEntropyStartStop tests start/stop lifecycle -func TestAntiEntropyStartStop(t *testing.T) { - // Create a minimal registry for testing +// TestAntiEntropyCurrentMembers tests that currentMembers excludes self and +// REMOVED nodes, but keeps LEAVING/DEAD/SUSPECT as members while only +// ALIVE nodes appear in the alive slice. +func TestAntiEntropyCurrentMembers(t *testing.T) { registry := NewNodeRegistry(1, "localhost:8081") + registry.Add(&NodeState{NodeId: 1, Address: "localhost:8081", Status: NodeStatus_ALIVE}) + registry.Add(&NodeState{NodeId: 2, Address: "localhost:8082", Status: NodeStatus_ALIVE}) + registry.Add(&NodeState{NodeId: 3, Address: "localhost:8083", Status: NodeStatus_SUSPECT}) + registry.Add(&NodeState{NodeId: 4, Address: "localhost:8084", Status: NodeStatus_DEAD}) + registry.Add(&NodeState{NodeId: 5, Address: "localhost:8085", Status: NodeStatus_LEAVING}) + registry.Add(&NodeState{NodeId: 6, Address: "localhost:8086", Status: NodeStatus_REMOVED}) - ae := &AntiEntropyService{ - nodeID: 1, - registry: registry, // Add registry to avoid nil pointer - enabled: true, - interval: 100 * time.Millisecond, - deltaThresholdTxns: 10000, - deltaThresholdSeconds: 3600, - stopCh: make(chan struct{}), - } + ae := &AntiEntropyService{nodeID: 1, registry: registry} - // Test start - ae.Start() - time.Sleep(50 * time.Millisecond) + members, alive := ae.currentMembers() - ae.mu.Lock() - running := ae.running - ae.mu.Unlock() - - if !running { - t.Error("Expected service to be running after Start()") + memberIDs := make(map[uint64]bool) + for _, m := range members { + memberIDs[m.NodeId] = true } - - // Test stop - ae.Stop() - time.Sleep(150 * time.Millisecond) // Give more time for goroutine to exit - - ae.mu.Lock() - running = ae.running - ae.mu.Unlock() - - if running { - t.Error("Expected service to be stopped after Stop()") + // Self and REMOVED are excluded; everything else (including + // LEAVING/DEAD/SUSPECT) is a member. + for _, id := range []uint64{2, 3, 4, 5} { + if !memberIDs[id] { + t.Errorf("expected node %d to be a member", id) + } } -} - -// TestAntiEntropyDisabled tests that disabled service doesn't start -func TestAntiEntropyDisabled(t *testing.T) { - ae := &AntiEntropyService{ - enabled: false, - stopCh: make(chan struct{}), + if memberIDs[1] || memberIDs[6] { + t.Errorf("expected self and REMOVED to be excluded, got members %v", memberIDs) } - ae.Start() - time.Sleep(50 * time.Millisecond) - - ae.mu.Lock() - running := ae.running - ae.mu.Unlock() - - if running { - t.Error("Disabled service should not be running") + if len(alive) != 1 || alive[0].NodeId != 2 { + t.Errorf("expected only node 2 to be alive, got %v", alive) } } -// TestLastSyncTimeZeroHandling tests that LastSyncTime=0 doesn't cause 54-year lag -func TestLastSyncTimeZeroHandling(t *testing.T) { - ae := &AntiEntropyService{ - deltaThresholdTxns: 10000, - deltaThresholdSeconds: 3600, // 1 hour - } +// TestConsiderSnapshotSource tests the restore-source tie-break rule: +// highest schema version wins, ties broken by the lowest node id. +func TestConsiderSnapshotSource(t *testing.T) { + peer2 := &NodeState{NodeId: 2} + peer3 := &NodeState{NodeId: 3} + peer5 := &NodeState{NodeId: 5} - // Test case 1: LastSyncTime=0 should result in 0 duration (use delta sync) - // The fix ensures we don't compute time.Since(0) which would be ~54 years - var timeLag time.Duration - lastSyncTime := int64(0) - if lastSyncTime > 0 { - timeLag = time.Since(time.Unix(0, lastSyncTime)) + var best *snapshotSource + best = considerSnapshotSource(best, snapshotSource{peer: peer5, schemaVersion: 3}) + if best.peer.NodeId != 5 { + t.Fatalf("expected first candidate to become best, got node %d", best.peer.NodeId) } - // With 0 lag, should use delta sync - if !ae.shouldUseDeltaSync(100, timeLag) { - t.Error("LastSyncTime=0 should result in 0 duration, allowing delta sync") + // Lower schema version never displaces a higher one. + best = considerSnapshotSource(best, snapshotSource{peer: peer2, schemaVersion: 1}) + if best.peer.NodeId != 5 { + t.Fatalf("expected node 5 (higher schema version) to remain best, got node %d", best.peer.NodeId) } - // Test case 2: Verify that non-zero LastSyncTime still works - lastSyncTime = time.Now().Add(-30 * time.Minute).UnixNano() - timeLag = time.Since(time.Unix(0, lastSyncTime)) - - // 30 minutes is within 1 hour threshold, should use delta - if !ae.shouldUseDeltaSync(100, timeLag) { - t.Error("30 minute time lag should allow delta sync with 1 hour threshold") + // Higher schema version wins outright. + best = considerSnapshotSource(best, snapshotSource{peer: peer3, schemaVersion: 9}) + if best.peer.NodeId != 3 { + t.Fatalf("expected node 3 (higher schema version) to become best, got node %d", best.peer.NodeId) } - // Test case 3: Old LastSyncTime exceeding threshold - lastSyncTime = time.Now().Add(-2 * time.Hour).UnixNano() - timeLag = time.Since(time.Unix(0, lastSyncTime)) - - // 2 hours exceeds 1 hour threshold, should use snapshot - if ae.shouldUseDeltaSync(100, timeLag) { - t.Error("2 hour time lag should trigger snapshot with 1 hour threshold") + // Same schema version: lower node id wins. + best = considerSnapshotSource(best, snapshotSource{peer: peer2, schemaVersion: 9}) + if best.peer.NodeId != 2 { + t.Fatalf("expected node 2 (tie broken by lowest id) to become best, got node %d", best.peer.NodeId) } } -// TestFindBestPeerForDatabase_SkipsOlderSchema tests that peers with older schema are skipped -func TestFindBestPeerForDatabase_SkipsOlderSchema(t *testing.T) { - tmpDir, dbMgr, schemaVersionMgr := setupTestEnvironment(t, "test_anti_entropy_skip_older") - defer os.RemoveAll(tmpDir) - defer dbMgr.Close() - - err := dbMgr.CreateDatabase("marmot") - if err != nil { - t.Fatalf("Failed to create database: %v", err) - } - - err = schemaVersionMgr.SetSchemaVersion("marmot", 2, "CREATE TABLE test (id INT)", 1) - if err != nil { - t.Fatalf("Failed to set local schema version: %v", err) - } - - registry := NewNodeRegistry(1, "localhost:8081") - client := NewClient(1) - - ae := &AntiEntropyService{ - nodeID: 1, - registry: registry, - client: client, - dbManager: dbMgr, - schemaVersionMgr: schemaVersionMgr, - } - - peerWithOlderSchema := &NodeState{ - NodeId: 2, - Address: "peer:8082", - Status: NodeStatus_ALIVE, - Incarnation: 1, - DatabaseSchemaVersions: map[string]uint64{ - "marmot": 0, - }, - } - - aliveNodes := []*NodeState{peerWithOlderSchema} - ctx := context.Background() - - result := ae.findBestPeerForDatabase(ctx, aliveNodes, "marmot") - - if result != nil { - t.Errorf("Expected nil (no valid peer), got peer with ID %d", result.NodeId) +// TestAntiEntropyCaughtUpDefaultsFalse tests that a database anti-entropy +// has never run a round for reports not caught up. +func TestAntiEntropyCaughtUpDefaultsFalse(t *testing.T) { + ae := NewAntiEntropyService(AntiEntropyConfig{NodeID: 1}) + if ae.CaughtUp("marmot") { + t.Error("expected a database with no completed round to report not caught up") } } -// TestFindBestPeerForDatabase_AcceptsSameSchema tests that peers with same schema are considered -func TestFindBestPeerForDatabase_AcceptsSameSchema(t *testing.T) { - tmpDir, dbMgr, schemaVersionMgr := setupTestEnvironment(t, "test_anti_entropy_same_schema") - defer os.RemoveAll(tmpDir) - defer dbMgr.Close() +// TestAntiEntropySetCaughtUp tests that setCaughtUp/CaughtUp round-trip per +// database independently. +func TestAntiEntropySetCaughtUp(t *testing.T) { + ae := NewAntiEntropyService(AntiEntropyConfig{NodeID: 1}) - err := dbMgr.CreateDatabase("marmot") - if err != nil { - t.Fatalf("Failed to create database: %v", err) - } + ae.setCaughtUp("a", true) + ae.setCaughtUp("b", false) - err = schemaVersionMgr.SetSchemaVersion("marmot", 2, "CREATE TABLE test (id INT)", 1) - if err != nil { - t.Fatalf("Failed to set local schema version: %v", err) + if !ae.CaughtUp("a") { + t.Error("expected database a to be caught up") } - - registry := NewNodeRegistry(1, "localhost:8081") - client := NewClient(1) - - ae := &AntiEntropyService{ - nodeID: 1, - registry: registry, - client: client, - dbManager: dbMgr, - schemaVersionMgr: schemaVersionMgr, + if ae.CaughtUp("b") { + t.Error("expected database b to not be caught up") } - - peerWithSameSchema := &NodeState{ - NodeId: 2, - Address: "peer:8082", - Status: NodeStatus_ALIVE, - Incarnation: 1, - DatabaseSchemaVersions: map[string]uint64{ - "marmot": 2, - }, - } - - aliveNodes := []*NodeState{peerWithSameSchema} - ctx := context.Background() - - result := ae.findBestPeerForDatabase(ctx, aliveNodes, "marmot") - - if result != nil && result.NodeId != 2 { - t.Errorf("Expected peer ID 2 or nil (no replication state), got %v", result) + if ae.CaughtUp("c") { + t.Error("expected an untouched database to default to not caught up") } } -// TestFindBestPeerForDatabase_AcceptsNewerSchema tests that peers with newer schema are considered -func TestFindBestPeerForDatabase_AcceptsNewerSchema(t *testing.T) { - tmpDir, dbMgr, schemaVersionMgr := setupTestEnvironment(t, "test_anti_entropy_newer_schema") - defer os.RemoveAll(tmpDir) - defer dbMgr.Close() - - err := dbMgr.CreateDatabase("marmot") - if err != nil { - t.Fatalf("Failed to create database: %v", err) - } - - err = schemaVersionMgr.SetSchemaVersion("marmot", 2, "CREATE TABLE test (id INT)", 1) - if err != nil { - t.Fatalf("Failed to set local schema version: %v", err) - } - +// TestAntiEntropyGetStats tests the stats shape: overall fields plus a +// per-database caught-up/stuck-txns entry. +func TestAntiEntropyGetStats(t *testing.T) { registry := NewNodeRegistry(1, "localhost:8081") - client := NewClient(1) - - ae := &AntiEntropyService{ - nodeID: 1, - registry: registry, - client: client, - dbManager: dbMgr, - schemaVersionMgr: schemaVersionMgr, - } + tmpDir, dbMgr, _ := setupTestEnvironment(t, "test_anti_entropy_get_stats") + t.Cleanup(func() { dbMgr.Close() }) + t.Cleanup(func() { os.RemoveAll(tmpDir) }) - peerWithNewerSchema := &NodeState{ - NodeId: 2, - Address: "peer:8082", - Status: NodeStatus_ALIVE, - Incarnation: 1, - DatabaseSchemaVersions: map[string]uint64{ - "marmot": 5, - }, + if err := dbMgr.CreateDatabase("marmot"); err != nil { + t.Fatalf("failed to create database: %v", err) } - aliveNodes := []*NodeState{peerWithNewerSchema} - ctx := context.Background() + ae := NewAntiEntropyService(AntiEntropyConfig{ + NodeID: 1, + Registry: registry, + DBManager: dbMgr, + LogPuller: NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: dbMgr}), + Interval: 60 * time.Second, + Enabled: true, + }) + ae.setCaughtUp("marmot", true) - result := ae.findBestPeerForDatabase(ctx, aliveNodes, "marmot") + stats := ae.GetStats() - if result != nil && result.NodeId != 2 { - t.Errorf("Expected peer ID 2 or nil (no replication state), got %v", result) + if enabled, ok := stats["enabled"].(bool); !ok || !enabled { + t.Errorf("expected enabled=true, got %v", stats["enabled"]) } -} - -// TestFindBestPeerForDatabase_AllPeersOlderSchema tests that nil is returned when all peers are behind -func TestFindBestPeerForDatabase_AllPeersOlderSchema(t *testing.T) { - tmpDir, dbMgr, schemaVersionMgr := setupTestEnvironment(t, "test_anti_entropy_all_older") - defer os.RemoveAll(tmpDir) - defer dbMgr.Close() - - err := dbMgr.CreateDatabase("marmot") - if err != nil { - t.Fatalf("Failed to create database: %v", err) + if running, ok := stats["running"].(bool); !ok || running { + t.Errorf("expected running=false, got %v", stats["running"]) } - - err = schemaVersionMgr.SetSchemaVersion("marmot", 5, "CREATE TABLE test (id INT)", 1) - if err != nil { - t.Fatalf("Failed to set local schema version: %v", err) + if interval, ok := stats["interval_seconds"].(float64); !ok || interval != 60.0 { + t.Errorf("expected interval_seconds=60, got %v", stats["interval_seconds"]) } - registry := NewNodeRegistry(1, "localhost:8081") - client := NewClient(1) - - ae := &AntiEntropyService{ - nodeID: 1, - registry: registry, - client: client, - dbManager: dbMgr, - schemaVersionMgr: schemaVersionMgr, + dbStats, ok := stats["databases"].(map[string]interface{}) + if !ok { + t.Fatalf("expected databases map in stats, got %T", stats["databases"]) } - - peer1 := &NodeState{ - NodeId: 2, - Address: "peer1:8082", - Status: NodeStatus_ALIVE, - Incarnation: 1, - DatabaseSchemaVersions: map[string]uint64{ - "marmot": 2, - }, + marmotStats, ok := dbStats["marmot"].(map[string]interface{}) + if !ok { + t.Fatalf("expected marmot entry in databases stats, got %v", dbStats) } - - peer2 := &NodeState{ - NodeId: 3, - Address: "peer2:8083", - Status: NodeStatus_ALIVE, - Incarnation: 1, - DatabaseSchemaVersions: map[string]uint64{ - "marmot": 4, - }, + if caughtUp, ok := marmotStats["caught_up"].(bool); !ok || !caughtUp { + t.Errorf("expected marmot caught_up=true, got %v", marmotStats["caught_up"]) } - - aliveNodes := []*NodeState{peer1, peer2} - ctx := context.Background() - - result := ae.findBestPeerForDatabase(ctx, aliveNodes, "marmot") - - if result != nil { - t.Errorf("Expected nil (all peers have older schema), got peer with ID %d", result.NodeId) + if stuck, ok := marmotStats["stuck_txns"].(int); !ok || stuck != 0 { + t.Errorf("expected marmot stuck_txns=0, got %v", marmotStats["stuck_txns"]) } } -// TestAntiEntropy_PeerSelectionWithMixedSchemaVersions tests peer selection with mixed schema versions -func TestAntiEntropy_PeerSelectionWithMixedSchemaVersions(t *testing.T) { - tmpDir, dbMgr, schemaVersionMgr := setupTestEnvironment(t, "test_anti_entropy_mixed_schema") - defer os.RemoveAll(tmpDir) - defer dbMgr.Close() - - err := dbMgr.CreateDatabase("marmot") - if err != nil { - t.Fatalf("Failed to create database: %v", err) - } - - err = schemaVersionMgr.SetSchemaVersion("marmot", 2, "CREATE TABLE test (id INT)", 1) - if err != nil { - t.Fatalf("Failed to set local schema version: %v", err) - } - +// TestAntiEntropyStartStop tests start/stop lifecycle. +func TestAntiEntropyStartStop(t *testing.T) { registry := NewNodeRegistry(1, "localhost:8081") - client := NewClient(1) - - ae := &AntiEntropyService{ - nodeID: 1, - registry: registry, - client: client, - dbManager: dbMgr, - schemaVersionMgr: schemaVersionMgr, - } + tmpDir, dbMgr, _ := setupTestEnvironment(t, "test_anti_entropy_start_stop") + t.Cleanup(func() { dbMgr.Close() }) + t.Cleanup(func() { os.RemoveAll(tmpDir) }) + + ae := NewAntiEntropyService(AntiEntropyConfig{ + NodeID: 1, + Registry: registry, + DBManager: dbMgr, + Client: NewClient(1), + Interval: 100 * time.Millisecond, + Enabled: true, + }) - peerOlder := &NodeState{ - NodeId: 2, - Address: "peer_old:8082", - Status: NodeStatus_ALIVE, - Incarnation: 1, - DatabaseSchemaVersions: map[string]uint64{ - "marmot": 0, - }, - } + ae.Start() + time.Sleep(50 * time.Millisecond) - peerSame := &NodeState{ - NodeId: 3, - Address: "peer_same:8083", - Status: NodeStatus_ALIVE, - Incarnation: 1, - DatabaseSchemaVersions: map[string]uint64{ - "marmot": 2, - }, + ae.mu.Lock() + running := ae.running + ae.mu.Unlock() + if !running { + t.Error("expected service to be running after Start()") } - peerNewer := &NodeState{ - NodeId: 4, - Address: "peer_new:8084", - Status: NodeStatus_ALIVE, - Incarnation: 1, - DatabaseSchemaVersions: map[string]uint64{ - "marmot": 3, - }, + ae.Stop() + time.Sleep(150 * time.Millisecond) + + ae.mu.Lock() + running = ae.running + ae.mu.Unlock() + if running { + t.Error("expected service to be stopped after Stop()") } +} - aliveNodes := []*NodeState{peerOlder, peerSame, peerNewer} - ctx := context.Background() +// TestAntiEntropyDisabled tests that a disabled service doesn't start. +func TestAntiEntropyDisabled(t *testing.T) { + ae := NewAntiEntropyService(AntiEntropyConfig{NodeID: 1, Enabled: false}) - result := ae.findBestPeerForDatabase(ctx, aliveNodes, "marmot") + ae.Start() + time.Sleep(50 * time.Millisecond) - if result != nil && result.NodeId == 2 { - t.Errorf("Should not select peer with older schema (ID 2), got peer ID %d", result.NodeId) + ae.mu.Lock() + running := ae.running + ae.mu.Unlock() + if running { + t.Error("disabled service should not be running") } } diff --git a/grpc/auth_test.go b/grpc/auth_test.go index 2e0e213c..2fdabb8f 100644 --- a/grpc/auth_test.go +++ b/grpc/auth_test.go @@ -229,6 +229,78 @@ func TestStreamPSKAuthentication(t *testing.T) { }) } +// TestLogPullRPCsRequireClusterSecret confirms the log-pull RPCs +// (ListCommittedLog, FetchTransactions, ListDatabaseRegistry) go through the +// same blanket cluster-secret interceptor as every other peer RPC: they are +// registered on the same grpc.Server under the same +// ChainUnaryInterceptor/ChainStreamInterceptor as Ping and StreamChanges +// above, so nothing method-specific is required, but this pins that a wrong +// secret is refused for them too rather than silently falling through +// testMarmotServer's Unimplemented default. +func TestLogPullRPCsRequireClusterSecret(t *testing.T) { + origSecret := cfg.Config.Cluster.ClusterSecret + cfg.Config.Cluster.ClusterSecret = "log-pull-secret" + defer func() { cfg.Config.Cluster.ClusterSecret = origSecret }() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer listener.Close() + + server := grpc.NewServer( + grpc.ChainUnaryInterceptor(UnaryServerInterceptor()), + grpc.ChainStreamInterceptor(StreamServerInterceptor()), + ) + RegisterMarmotServiceServer(server, &testMarmotServer{}) + go server.Serve(listener) + defer server.Stop() + + conn, err := grpc.NewClient( + listener.Addr().String(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithChainUnaryInterceptor(clientInterceptorWithSecret("wrong-secret")), + grpc.WithChainStreamInterceptor(streamClientInterceptorWithSecret("wrong-secret")), + ) + if err != nil { + t.Fatalf("failed to connect: %v", err) + } + defer conn.Close() + client := NewMarmotServiceClient(conn) + + t.Run("ListCommittedLog", func(t *testing.T) { + _, err := client.ListCommittedLog(context.Background(), &LogListRequest{Database: "app"}) + st, ok := status.FromError(err) + if !ok || st.Code() != codes.Unauthenticated { + t.Fatalf("expected Unauthenticated, got: %v", err) + } + }) + + t.Run("ListDatabaseRegistry", func(t *testing.T) { + _, err := client.ListDatabaseRegistry(context.Background(), &DatabaseRegistryRequest{}) + st, ok := status.FromError(err) + if !ok || st.Code() != codes.Unauthenticated { + t.Fatalf("expected Unauthenticated, got: %v", err) + } + }) + + t.Run("FetchTransactions", func(t *testing.T) { + stream, err := client.FetchTransactions(context.Background(), &FetchTransactionsRequest{Database: "app", TxnIds: []uint64{1}}) + if err != nil { + st, ok := status.FromError(err) + if ok && st.Code() == codes.Unauthenticated { + return + } + t.Fatalf("failed to create stream: %v", err) + } + _, err = stream.Recv() + st, ok := status.FromError(err) + if !ok || st.Code() != codes.Unauthenticated { + t.Fatalf("expected Unauthenticated, got: %v", err) + } + }) +} + // testMarmotServer implements MarmotServiceServer for testing type testMarmotServer struct { UnimplementedMarmotServiceServer diff --git a/grpc/autoinc_claim_handler_test.go b/grpc/autoinc_claim_handler_test.go index 89d25865..381fa2e0 100644 --- a/grpc/autoinc_claim_handler_test.go +++ b/grpc/autoinc_claim_handler_test.go @@ -28,7 +28,7 @@ func newClaimTestHandler(t *testing.T, ddl string) (*ReplicationHandler, *db.Dat _, err = mdb.GetWriteDB().Exec(ddl) require.NoError(t, err) require.NoError(t, mdb.ReloadSchema()) - return NewReplicationHandler(1, dm, clock, db.NewSchemaVersionManager(dm.GetSystemDatabase().GetMetaStore())), dm + return NewReplicationHandler(1, dm, clock, db.NewSchemaVersionManager(dm)), dm } // prepareOverWire sends statements through the wire conversion and the diff --git a/grpc/autoinc_claim_schema_gate_test.go b/grpc/autoinc_claim_schema_gate_test.go index 97d83698..4af17934 100644 --- a/grpc/autoinc_claim_schema_gate_test.go +++ b/grpc/autoinc_claim_schema_gate_test.go @@ -79,9 +79,7 @@ func newGateCluster(t *testing.T, ids []uint64) (*handlerFanout, map[uint64]*gat require.NoError(t, dm.MergeAutoIncBasesAndReleaseVotes(nil)) dm.SetClusterMembership(provider.GetTotalMembershipSize) require.NoError(t, dm.CreateDatabase("gate")) - systemDB, err := dm.GetDatabase(db.SystemDatabaseName) - require.NoError(t, err) - versions := db.NewSchemaVersionManager(systemDB.GetMetaStore()) + versions := db.NewSchemaVersionManager(dm) wc := coordinator.NewWriteCoordinator(id, provider, fanout, db.NewLocalReplicator(id, dm, clock), 30*time.Second, clock) wc.SetSchemaVersionSource(versions.GetSchemaVersion) fanout.nodes[id] = &gateNode{id: id, clock: clock, dm: dm, versions: versions, @@ -90,9 +88,11 @@ func newGateCluster(t *testing.T, ids []uint64) (*handlerFanout, map[uint64]*gat return fanout, fanout.nodes } -// gateDDL replicates ddl from node through 2PC and bumps that node's own -// schema version, as CoordinatorHandler does after a DDL commit; remote -// participants bump theirs when they apply the COMMIT. +// gateDDL replicates ddl from node through 2PC. Every participant's own +// commit (TransactionManager.applyNonDMLIntents) bumps its +// __marmot_schema_version atomically with the DDL itself as part of that +// commit, coordinator and remotes alike - there is nothing left to do here +// after WriteTransaction returns. func gateDDL(t *testing.T, node *gateNode, table, ddl string) { t.Helper() startTS := node.clock.Now() @@ -102,8 +102,6 @@ func gateDDL(t *testing.T, node *gateNode, table, ddl string) { Statements: []protocol.Statement{{Type: protocol.StatementDDL, SQL: ddl, TableName: table, Database: "gate"}}, } require.NoError(t, node.wc.WriteTransaction(context.Background(), txn), ddl) - _, err := node.versions.IncrementSchemaVersion("gate", ddl, txn.ID) - require.NoError(t, err) } // waitGate polls until check passes, failing after a deadline: a commit @@ -129,7 +127,7 @@ func (n *gateNode) base(table string) (uint64, error) { return db.NewAutoIncClaimStore(n.dm.GetSystemDatabase()).ReadBase("gate", table) } -// TestClaimFromALaggingVoterIsDeclined is R3c-12's counterexample, driven +// TestClaimFromALaggingVoterIsDeclined is a lagging voter's counterexample, driven // through the real DDL apply path and every participant's ReplicationHandler. // // Nodes g=1, h=2, k=3. A claim on t2, (0,64], commits on {k,g} while h is diff --git a/grpc/catch_up.go b/grpc/catch_up.go index 64d63426..852b8458 100644 --- a/grpc/catch_up.go +++ b/grpc/catch_up.go @@ -26,9 +26,9 @@ type CatchUpStrategy int const ( // NO_CATCHUP - Node is up to date, no catch-up needed NO_CATCHUP CatchUpStrategy = iota - // DELTA_SYNC - Node is slightly behind, use transaction log replay + // DELTA_SYNC - Node has data: pull the seed's log for every database DELTA_SYNC - // FULL_SNAPSHOT - Node is far behind or has no data, need full snapshot + // FULL_SNAPSHOT - Node has no data, need full snapshot FULL_SNAPSHOT ) @@ -69,17 +69,6 @@ func (c *CatchUpClient) SetDatabaseManager(dbMgr *db.DatabaseManager) { c.dbManager.Store(dbMgr) } -// persistSchemaVersions restores a snapshot's schema versions, writing through -// the live DatabaseManager's system MetaStore when one is available (the -// anti-entropy runtime path), or by opening the MetaStore directly by path -// when it is not (the startup join path, before the DatabaseManager exists). -func (c *CatchUpClient) persistSchemaVersions(versions map[string]uint64) error { - if dbMgr := c.dbManager.Load(); dbMgr != nil { - return persistSnapshotSchemaVersionsViaManager(dbMgr, versions) - } - return persistSnapshotSchemaVersions(c.dataDir, versions) -} - // CatchUpFromPeer downloads a snapshot of a specific database from a peer // Used by anti-entropy to trigger snapshots for lagging databases func (c *CatchUpClient) CatchUpFromPeer(ctx context.Context, peerNodeID uint64, peerAddr string, database string) error { @@ -340,19 +329,8 @@ func (c *CatchUpClient) applySnapshot(ctx context.Context, client MarmotServiceC } // From this point on the snapshot's files are already swapped onto disk. - // Restore the schema versions the snapshot was taken at - skipping this - // leaves the node reporting version 0, after which it refuses every - // transaction that requires a newer schema and can never rejoin - // replication. Prefer the versions captured atomically with these exact - // files (the stream trailer) over the earlier, potentially stale read from - // GetSnapshotInfo. - versions := SnapshotVersionsForRestore(info.DatabaseMetadata, stream) - if database != "" { - versions = versionsFor(versions, database) - } - if err := c.persistSchemaVersions(versions); err != nil { - return fmt.Errorf("failed to restore schema versions from snapshot: %w", err) - } + // Each installed SQLite file carries its own __marmot_schema_version + // table, so nothing needs to be restored out of band here any more. log.Info(). Uint64("snapshot_txn_id", info.SnapshotTxnId). @@ -382,15 +360,6 @@ func snapshotFilesToRestore(databases []*DatabaseFileInfo, database string) ([]s return files, nil } -// versionsFor keeps only database's entry of a snapshot's schema versions. -func versionsFor(versions map[string]uint64, database string) map[string]uint64 { - v, ok := versions[database] - if !ok { - return nil - } - return map[string]uint64{database: v} -} - // grpcSnapshotStreamAdapter adapts MarmotService_StreamSnapshotClient to snapshot.ChunkReceiver type grpcSnapshotStreamAdapter struct { stream MarmotService_StreamSnapshotClient @@ -409,6 +378,8 @@ func (a *grpcSnapshotStreamAdapter) Recv() (*snapshot.Chunk, error) { Data: chunk.GetData(), MD5Checksum: chunk.GetChecksum(), IsLastForFile: chunk.GetIsLastForFile(), + FileSHA256: chunk.GetFileSha256(), + FileSizeBytes: chunk.GetFileSizeBytes(), }, nil } @@ -528,29 +499,44 @@ func (c *CatchUpClient) GetPeerMaxTxnIDs(ctx context.Context, peerAddr string) ( return resp.DatabaseTxnIds, nil } -// CatchUpDecision contains the strategy and sync information +// CatchUpDecision is the chosen strategy and the seed it runs against. type CatchUpDecision struct { - Strategy CatchUpStrategy - PeerNodeID uint64 - PeerAddr string - DatabaseDeltas map[string]DeltaInfo // Per-database sync info + Strategy CatchUpStrategy + PeerNodeID uint64 + PeerAddr string } -// DeltaInfo contains delta sync information for a database -type DeltaInfo struct { - DatabaseName string - LocalTxnID uint64 - PeerTxnID uint64 - TxnsBehind uint64 +// catchUpStrategyFor decides how a starting node catches up with a seed from +// the max txn ids each side reports per database: +// - a seed with no data: nothing to catch up; +// - a node with no data of its own: a full snapshot; +// - otherwise a log pull of every database the seed has. Whether a node is +// behind cannot be read from txn ids (they are not a position in any +// one node's log), and a +// pull that finds nothing new costs one listing; PerformLogPull restores +// a database from a snapshot only when the seed's log no longer covers +// this node's cursor into it. +func catchUpStrategyFor(localTxnIDs, peerTxnIDs map[string]uint64) CatchUpStrategy { + peerHasData := false + for _, txnID := range peerTxnIDs { + if txnID > 0 { + peerHasData = true + break + } + } + if !peerHasData { + return NO_CATCHUP + } + if len(localTxnIDs) == 0 { + return FULL_SNAPSHOT + } + return DELTA_SYNC } // DetermineCatchUpStrategy determines the best catch-up strategy by comparing local vs cluster state // This is the main entry point for catch-up detection func (c *CatchUpClient) DetermineCatchUpStrategy(ctx context.Context) (*CatchUpDecision, error) { - decision := &CatchUpDecision{ - Strategy: NO_CATCHUP, - DatabaseDeltas: make(map[string]DeltaInfo), - } + decision := &CatchUpDecision{Strategy: NO_CATCHUP} // Step 1: Get local transaction IDs localTxnIDs, err := c.GetLocalMaxTxnID(ctx) @@ -572,136 +558,89 @@ func (c *CatchUpClient) DetermineCatchUpStrategy(ctx context.Context) (*CatchUpD return nil, fmt.Errorf("failed to get peer txn IDs: %w", err) } - // Step 4: Compare local vs peer state - // Check if peer has any actual data (non-zero txn IDs) - // A peer with {"marmot": 0} has no data, just an empty database - var peerHasData bool - for _, txnID := range peerTxnIDs { - if txnID > 0 { - peerHasData = true - break - } - } + decision.Strategy = catchUpStrategyFor(localTxnIDs, peerTxnIDs) + log.Info(). + Str("peer", seedAddr). + Int("strategy", int(decision.Strategy)). + Msg("Catch-up strategy selected") + return decision, nil +} - if len(localTxnIDs) == 0 && peerHasData { - // We have no data, peer has actual data - need full snapshot - decision.Strategy = FULL_SNAPSHOT - log.Info(). - Str("peer", seedAddr). - Msg("No local data found - full snapshot required") - return decision, nil +// PerformLogPull catches a starting node up with its seed: it merges the seed's database registry, so a database created +// while this node was down exists before its log is pulled, then runs one +// bounded LogPuller.PullPair pass over the seed for every database, restoring +// one from the seed's snapshot when the seed's log no longer covers this +// node's cursor into it. Every step terminates: ListCommittedLog is a bounded +// RPC and FetchTransactions ends with EOF. +// +// A database that fails is logged and left to anti-entropy, which pulls every +// member's log every round; startup is not failed for it. Only a missing +// puller or database manager is an error. +func (c *CatchUpClient) PerformLogPull(ctx context.Context, decision *CatchUpDecision, lp *LogPuller, client *Client) error { + if lp == nil { + return fmt.Errorf("log puller not provided") } - - if !peerHasData { - // Peer has no actual data - we're up to date (or we're the first node) - decision.Strategy = NO_CATCHUP - log.Info().Msg("Peer has no data - no catch-up needed") - return decision, nil + dbMgr := c.dbManager.Load() + if dbMgr == nil { + return fmt.Errorf("startup log pull needs the running node's DatabaseManager") } - // Step 5: Calculate deltas per database - var totalTxnsBehind uint64 - var maxDeltaForAnyDB uint64 - - // Check all databases that exist on peer - for dbName, peerTxnID := range peerTxnIDs { - localTxnID := localTxnIDs[dbName] // 0 if database doesn't exist locally + log.Info().Str("peer", decision.PeerAddr).Msg("Starting startup log pull") - if peerTxnID > localTxnID { - delta := peerTxnID - localTxnID - totalTxnsBehind += delta + // A node the pull finds behind is marked JOINING, so it stays out of + // quorums until the promotion checker admits it. A node the pull finds + // current keeps its status: every startup of a node with data runs this + // pull, and a current node is exactly what a restart of an idle cluster + // brings back. + behind := false + defer func() { + if behind { + c.registry.MarkJoining(c.nodeID) + } + }() - if delta > maxDeltaForAnyDB { - maxDeltaForAnyDB = delta - } + if err := reconcileRegistryWithPeer(ctx, client, dbMgr, c.nodeID, decision.PeerAddr); err != nil { + log.Warn().Err(err).Str("peer", decision.PeerAddr). + Msg("Startup log pull: database registry reconciliation failed; anti-entropy will retry it") + } - decision.DatabaseDeltas[dbName] = DeltaInfo{ - DatabaseName: dbName, - LocalTxnID: localTxnID, - PeerTxnID: peerTxnID, - TxnsBehind: delta, + peer := PeerRef{NodeID: decision.PeerNodeID, Address: decision.PeerAddr} + for _, dbName := range dbMgr.ListDatabases() { + result, err := lp.PullPair(ctx, peer, dbName) + if err != nil || result.NeedsSnapshot || result.Applied > 0 || !result.CaughtUp { + behind = true + } + if err != nil { + log.Warn().Err(err).Str("database", dbName). + Msg("Startup log pull failed; anti-entropy will retry it") + continue + } + if result.NeedsSnapshot { + log.Info().Str("database", dbName).Uint64("peer", decision.PeerNodeID). + Msg("Startup log pull: the seed's log no longer covers this node's cursor, restoring from its snapshot") + if err := c.restoreFromSeed(ctx, dbName, peer, lp); err != nil { + log.Warn().Err(err).Str("database", dbName). + Msg("Startup restore failed; anti-entropy will retry it") } - - log.Debug(). - Str("database", dbName). - Uint64("local_txn_id", localTxnID). - Uint64("peer_txn_id", peerTxnID). - Uint64("behind_by", delta). - Msg("Database delta calculated") + continue } + log.Info().Str("database", dbName).Int("txns_applied", result.Applied).Bool("caught_up", result.CaughtUp). + Msg("Startup log pull: database pass completed") } - - // Step 6: Determine strategy based on delta size - if totalTxnsBehind == 0 { - decision.Strategy = NO_CATCHUP - log.Info().Msg("Node is up to date - no catch-up needed") - } else if maxDeltaForAnyDB > uint64(cfg.Config.Replication.DeltaSyncThresholdTxns) { - // Any database is too far behind - use snapshot - decision.Strategy = FULL_SNAPSHOT - log.Info(). - Uint64("max_delta", maxDeltaForAnyDB). - Int("threshold", cfg.Config.Replication.DeltaSyncThresholdTxns). - Msg("Delta too large for any database - full snapshot required") - } else { - // Small delta - use incremental sync - decision.Strategy = DELTA_SYNC - log.Info(). - Uint64("total_txns_behind", totalTxnsBehind). - Uint64("max_delta", maxDeltaForAnyDB). - Int("databases_to_sync", len(decision.DatabaseDeltas)). - Msg("Delta sync strategy selected") - } - - return decision, nil + return nil } -// PerformDeltaSync performs incremental catch-up using transaction logs -// This is called when the node is only slightly behind and can catch up via delta sync -func (c *CatchUpClient) PerformDeltaSync(ctx context.Context, decision *CatchUpDecision, deltaSyncClient *DeltaSyncClient) error { - if deltaSyncClient == nil { - return fmt.Errorf("delta sync client not provided") +// restoreFromSeed restores dbName from peer's snapshot through the same +// restore sequence anti-entropy runs (restoreDatabase). +func (c *CatchUpClient) restoreFromSeed(ctx context.Context, dbName string, peer PeerRef, lp *LogPuller) error { + dbMgr := c.dbManager.Load() + if dbMgr == nil { + return fmt.Errorf("startup log pull of %s needs the running node's DatabaseManager", dbName) } - - log.Info(). - Int("databases_to_sync", len(decision.DatabaseDeltas)). - Str("peer", decision.PeerAddr). - Msg("Starting delta sync") - - // Mark ourselves as JOINING during sync - c.registry.MarkJoining(c.nodeID) - - // Sync each database that's behind - for dbName, deltaInfo := range decision.DatabaseDeltas { - log.Info(). - Str("database", dbName). - Uint64("from_txn_id", deltaInfo.LocalTxnID). - Uint64("to_txn_id", deltaInfo.PeerTxnID). - Uint64("txns_to_apply", deltaInfo.TxnsBehind). - Msg("Starting database delta sync") - - // Use existing DeltaSyncClient to sync from peer - result, err := deltaSyncClient.SyncFromPeer( - ctx, - decision.PeerNodeID, - decision.PeerAddr, - dbName, - deltaInfo.LocalTxnID, - ) - - if err != nil { - return fmt.Errorf("failed to sync database %s: %w", dbName, err) + return restoreDatabase(ctx, dbMgr, lp, dbName, func(ctx context.Context) error { + if err := c.CatchUpFromPeer(ctx, peer.NodeID, peer.Address, dbName); err != nil { + return fmt.Errorf("failed to restore database %s from peer: %w", dbName, err) } - - log.Info(). - Str("database", dbName). - Int("txns_applied", result.TxnsApplied). - Uint64("final_txn_id", result.LastAppliedTxnID). - Msg("Database delta sync completed") - } - - log.Info(). - Int("databases_synced", len(decision.DatabaseDeltas)). - Msg("Delta sync completed successfully") - - return nil + return nil + }) } diff --git a/grpc/catch_up_detach_test.go b/grpc/catch_up_detach_test.go index 022a74d7..7ea1a321 100644 --- a/grpc/catch_up_detach_test.go +++ b/grpc/catch_up_detach_test.go @@ -313,6 +313,8 @@ func TestAntiEntropyRestoresADatabaseWhoseReattachFailed(t *testing.T) { var tried []uint64 ae := &AntiEntropyService{ dbManager: local, + interval: 30 * time.Second, + logPuller: NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local}), snapshotFunc: func(ctx context.Context, peerNodeID uint64, peerAddr string, database string) error { tried = append(tried, peerNodeID) return client.CatchUpFromPeer(ctx, peerNodeID, peerAddr, database) @@ -323,9 +325,8 @@ func TestAntiEntropyRestoresADatabaseWhoseReattachFailed(t *testing.T) { require.NoError(t, err) unreachable := closed.Addr().String() require.NoError(t, closed.Close()) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - ae.restoreAwaitingDatabases(ctx, []*NodeState{{NodeId: 2, Address: unreachable}, {NodeId: 3, Address: addr}}) + nodes := []*NodeState{{NodeId: 2, Address: unreachable}, {NodeId: 3, Address: addr}} + ae.restoreAwaitingDatabases(nodes) require.Empty(t, local.DatabasesAwaitingRestore(), "a database whose reattach failed was never restored") require.Equal(t, []uint64{2, 3}, tried) @@ -387,10 +388,17 @@ func TestTwoNodesAwaitingRestoresRecoverFromEachOther(t *testing.T) { antiEntropy := func(dm *db.DatabaseManager, nodeID uint64, dir string) *AntiEntropyService { client := NewCatchUpClient(nodeID, dir, NewNodeRegistry(nodeID, "localhost:5001"), nil) client.SetDatabaseManager(dm) - return &AntiEntropyService{dbManager: dm, snapshotFunc: client.CatchUpFromPeer} + return &AntiEntropyService{ + dbManager: dm, + interval: 30 * time.Second, + logPuller: NewLogPuller(LogPullerConfig{NodeID: nodeID, Client: NewClient(nodeID), DBManager: dm}), + snapshotFunc: client.CatchUpFromPeer, + } } - antiEntropy(x, 1, xDir).restoreAwaitingDatabases(ctx, []*NodeState{{NodeId: 2, Address: yAddr}}) - antiEntropy(y, 2, yDir).restoreAwaitingDatabases(ctx, []*NodeState{{NodeId: 1, Address: xAddr}}) + yNode := []*NodeState{{NodeId: 2, Address: yAddr}} + xNode := []*NodeState{{NodeId: 1, Address: xAddr}} + antiEntropy(x, 1, xDir).restoreAwaitingDatabases(yNode) + antiEntropy(y, 2, yDir).restoreAwaitingDatabases(xNode) require.Empty(t, x.DatabasesAwaitingRestore(), "a database awaiting a restore was never restored by a peer awaiting another") require.Empty(t, y.DatabasesAwaitingRestore(), "a database awaiting a restore was never restored by a peer awaiting another") diff --git a/grpc/catch_up_test.go b/grpc/catch_up_test.go index 1765bc12..75d1cdc7 100644 --- a/grpc/catch_up_test.go +++ b/grpc/catch_up_test.go @@ -2,112 +2,58 @@ package grpc import ( "context" - "io" "net" "sync" "sync/atomic" "testing" - "github.com/maxpert/marmot/cfg" "github.com/maxpert/marmot/db" "github.com/maxpert/marmot/hlc" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/metadata" ) -// TestDetermineCatchUpStrategy_UsesConfigThreshold verifies that the catch-up strategy -// uses the configured threshold from cfg.Config.Replication.DeltaSyncThresholdTxns -// instead of the hard-coded DeltaSyncThreshold constant -func TestDetermineCatchUpStrategy_UsesConfigThreshold(t *testing.T) { - // Save original config - originalThreshold := cfg.Config.Replication.DeltaSyncThresholdTxns - defer func() { - cfg.Config.Replication.DeltaSyncThresholdTxns = originalThreshold - }() - - // Set a custom threshold much lower than the hard-coded value - customThreshold := 100 - cfg.Config.Replication.DeltaSyncThresholdTxns = customThreshold - - // Create a mock registry with no alive nodes (we'll control the seed) - registry := NewNodeRegistry(1, "localhost:5001") - - // Create catch-up client - _ = NewCatchUpClient(1, "/tmp/test", registry, []string{}) - - // Mock: Inject a controlled scenario - // We can't easily test the full flow without mocking gRPC calls, - // but we can verify the threshold logic directly - - // Create test decision with delta just over the custom threshold - decision := &CatchUpDecision{ - Strategy: NO_CATCHUP, - PeerAddr: "localhost:5002", - DatabaseDeltas: make(map[string]DeltaInfo), - } - - // Simulate a database with delta = customThreshold + 1 (should trigger FULL_SNAPSHOT) - decision.DatabaseDeltas["test_db"] = DeltaInfo{ - DatabaseName: "test_db", - LocalTxnID: 0, - PeerTxnID: uint64(customThreshold + 1), - TxnsBehind: uint64(customThreshold + 1), - } - - // Calculate max delta - var maxDelta uint64 - for _, delta := range decision.DatabaseDeltas { - if delta.TxnsBehind > maxDelta { - maxDelta = delta.TxnsBehind - } - } - - // Verify the logic: maxDelta > config threshold should trigger FULL_SNAPSHOT - expectedStrategy := FULL_SNAPSHOT - if maxDelta <= uint64(cfg.Config.Replication.DeltaSyncThresholdTxns) { - expectedStrategy = DELTA_SYNC - } - - assert.Equal(t, FULL_SNAPSHOT, expectedStrategy, - "With delta=%d and threshold=%d, should choose FULL_SNAPSHOT", - maxDelta, customThreshold) - - // Test with delta = customThreshold (should use DELTA_SYNC) - decision.DatabaseDeltas["test_db"] = DeltaInfo{ - DatabaseName: "test_db", - LocalTxnID: 0, - PeerTxnID: uint64(customThreshold), - TxnsBehind: uint64(customThreshold), - } - - maxDelta = 0 - for _, delta := range decision.DatabaseDeltas { - if delta.TxnsBehind > maxDelta { - maxDelta = delta.TxnsBehind - } +// TestCatchUpStrategyFor: a starting node with data always catches up by a +// log pull, however far apart the two sides' max txn ids are. Those ids are +// start timestamps, so their difference is no count of missing txns: on +// 7a66596 a restarted node that missed a few seconds of writes computed a +// "delta" of ~7.5e18 and took a whole-node snapshot, which under write load +// failed its checksum and killed the node. +// +// Mutation: choose FULL_SNAPSHOT when the seed's id is far above the local +// one. "a node with data was sent to a full snapshot" fires. +func TestCatchUpStrategyFor(t *testing.T) { + cases := []struct { + name string + local, peer map[string]uint64 + want CatchUpStrategy + }{ + {"seed without data", map[string]uint64{"marmot": 5}, map[string]uint64{"marmot": 0}, NO_CATCHUP}, + {"new node", map[string]uint64{}, map[string]uint64{"marmot": 7509703466722328577}, FULL_SNAPSHOT}, + {"restarted far behind by id", map[string]uint64{"marmot": 1}, map[string]uint64{"marmot": 7509703466722328577}, DELTA_SYNC}, + {"restarted ahead by id", map[string]uint64{"marmot": 900}, map[string]uint64{"marmot": 100}, DELTA_SYNC}, + {"database missing locally", map[string]uint64{"marmot": 100}, map[string]uint64{"marmot": 100, "other": 5}, DELTA_SYNC}, } - - expectedStrategy = DELTA_SYNC - if maxDelta > uint64(cfg.Config.Replication.DeltaSyncThresholdTxns) { - expectedStrategy = FULL_SNAPSHOT + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := catchUpStrategyFor(tc.local, tc.peer) + if tc.want == DELTA_SYNC { + require.Equal(t, DELTA_SYNC, got, "a node with data was sent to a full snapshot or skipped") + return + } + require.Equal(t, tc.want, got) + }) } - - assert.Equal(t, DELTA_SYNC, expectedStrategy, - "With delta=%d and threshold=%d, should choose DELTA_SYNC", - maxDelta, customThreshold) } // TestCatchUpDecision_IncludesPeerNodeID verifies that CatchUpDecision // includes the peer node ID, not just the address func TestCatchUpDecision_IncludesPeerNodeID(t *testing.T) { decision := &CatchUpDecision{ - Strategy: DELTA_SYNC, - PeerNodeID: 12345, // Should be populated with actual peer node ID - PeerAddr: "localhost:5002", - DatabaseDeltas: make(map[string]DeltaInfo), + Strategy: DELTA_SYNC, + PeerNodeID: 12345, + PeerAddr: "localhost:5002", } require.NotZero(t, decision.PeerNodeID, @@ -116,262 +62,6 @@ func TestCatchUpDecision_IncludesPeerNodeID(t *testing.T) { "Peer node ID should match expected value") } -// TestFindAvailableSeed_ReturnsNodeID verifies that findAvailableSeed -// returns both node ID and address (not just address with node ID = 0) -func TestFindAvailableSeed_ReturnsNodeID(t *testing.T) { - // This is a structural test to ensure the function signature is correct - // We verify the return type includes node ID - - registry := NewNodeRegistry(1, "localhost:5001") - - // Note: We can't easily test the actual gRPC connectivity without - // standing up a real server, but we can verify that GetAlive() returns - // nodes with both NodeId and Address fields - - // The registry should track nodes with their IDs - // This verifies the data structure is correct for our fix - _ = NewCatchUpClient(1, "/tmp/test", registry, []string{}) - - // Verify that NodeState has both NodeId and Address - // This is a compile-time check that the data structure supports our fix - var testNode *NodeState - if testNode != nil { - _ = testNode.NodeId - _ = testNode.Address - } -} - -// TestPerformDeltaSync_PassesPeerNodeID verifies that PerformDeltaSync -// passes the correct peer node ID (not 0) to SyncFromPeer -func TestPerformDeltaSync_PassesPeerNodeID(t *testing.T) { - // This is a structural test - the actual fix ensures that - // decision.PeerNodeID is passed instead of hard-coded 0 - - decision := &CatchUpDecision{ - Strategy: DELTA_SYNC, - PeerNodeID: 999, // Actual peer node ID - PeerAddr: "localhost:5003", - DatabaseDeltas: map[string]DeltaInfo{ - "test_db": { - DatabaseName: "test_db", - LocalTxnID: 100, - PeerTxnID: 200, - TxnsBehind: 100, - }, - }, - } - - // Verify decision has the correct peer node ID - require.NotZero(t, decision.PeerNodeID, - "CatchUpDecision should have non-zero peer node ID") - assert.Equal(t, uint64(999), decision.PeerNodeID, - "Peer node ID should be set correctly") - - // The actual fix in PerformDeltaSync should use: - // deltaSyncClient.SyncFromPeer(ctx, decision.PeerNodeID, ...) - // instead of: - // deltaSyncClient.SyncFromPeer(ctx, 0, ...) -} - -// TestThresholdConfiguration verifies that the configured threshold -// is respected by the catch-up strategy logic -func TestThresholdConfiguration(t *testing.T) { - testCases := []struct { - name string - configThreshold int - delta uint64 - expectedStrategy CatchUpStrategy - }{ - { - name: "Delta below threshold uses DELTA_SYNC", - configThreshold: 1000, - delta: 500, - expectedStrategy: DELTA_SYNC, - }, - { - name: "Delta at threshold uses DELTA_SYNC", - configThreshold: 1000, - delta: 1000, - expectedStrategy: DELTA_SYNC, - }, - { - name: "Delta above threshold uses FULL_SNAPSHOT", - configThreshold: 1000, - delta: 1001, - expectedStrategy: FULL_SNAPSHOT, - }, - { - name: "Large delta uses FULL_SNAPSHOT", - configThreshold: 10000, - delta: 50000, - expectedStrategy: FULL_SNAPSHOT, - }, - { - name: "Small delta with large threshold uses DELTA_SYNC", - configThreshold: 100000, - delta: 10000, - expectedStrategy: DELTA_SYNC, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // Save original config - originalThreshold := cfg.Config.Replication.DeltaSyncThresholdTxns - defer func() { - cfg.Config.Replication.DeltaSyncThresholdTxns = originalThreshold - }() - - // Set test threshold - cfg.Config.Replication.DeltaSyncThresholdTxns = tc.configThreshold - - // Determine strategy using the same logic as DetermineCatchUpStrategy - var strategy CatchUpStrategy - if tc.delta > uint64(cfg.Config.Replication.DeltaSyncThresholdTxns) { - strategy = FULL_SNAPSHOT - } else if tc.delta > 0 { - strategy = DELTA_SYNC - } else { - strategy = NO_CATCHUP - } - - assert.Equal(t, tc.expectedStrategy, strategy, - "Strategy should match expected for delta=%d, threshold=%d", - tc.delta, tc.configThreshold) - }) - } -} - -// TestCatchUpDecision_FieldsPopulated verifies all fields in CatchUpDecision -// are properly populated -func TestCatchUpDecision_FieldsPopulated(t *testing.T) { - decision := &CatchUpDecision{ - Strategy: DELTA_SYNC, - PeerNodeID: 42, - PeerAddr: "localhost:5002", - DatabaseDeltas: make(map[string]DeltaInfo), - } - - decision.DatabaseDeltas["db1"] = DeltaInfo{ - DatabaseName: "db1", - LocalTxnID: 100, - PeerTxnID: 200, - TxnsBehind: 100, - } - - // Verify all critical fields are set - assert.NotEqual(t, NO_CATCHUP, decision.Strategy, "Strategy should be set") - assert.NotZero(t, decision.PeerNodeID, "PeerNodeID should be non-zero") - assert.NotEmpty(t, decision.PeerAddr, "PeerAddr should not be empty") - assert.NotEmpty(t, decision.DatabaseDeltas, "DatabaseDeltas should not be empty") - - // Verify delta info - delta := decision.DatabaseDeltas["db1"] - assert.Equal(t, "db1", delta.DatabaseName) - assert.Equal(t, uint64(100), delta.LocalTxnID) - assert.Equal(t, uint64(200), delta.PeerTxnID) - assert.Equal(t, uint64(100), delta.TxnsBehind) -} - -// Before a DatabaseManager is wired in (the startup join path), schema -// versions must be persisted by opening the MetaStore directly by path - -// there is nothing else running yet to hold it open. -func TestCatchUpClient_PersistSchemaVersions_NoManagerUsesPath(t *testing.T) { - dataDir := t.TempDir() - registry := NewNodeRegistry(1, "localhost:5001") - client := NewCatchUpClient(1, dataDir, registry, nil) - - require.NoError(t, client.persistSchemaVersions(map[string]uint64{"appdb": 6})) - - require.Equal(t, int64(6), readSchemaVersions(t, dataDir)["appdb"]) -} - -// Once SetDatabaseManager has been called (the anti-entropy runtime path), -// persistSchemaVersions must write through the live DatabaseManager instead - -// opening the MetaStore by path a second time would deadlock against Pebble's -// exclusive lock, since the DatabaseManager already holds it open. -func TestCatchUpClient_PersistSchemaVersions_WithManagerUsesLiveStore(t *testing.T) { - dataDir := t.TempDir() - dbMgr, err := db.NewDatabaseManager(dataDir, 1, hlc.NewClock(1)) - require.NoError(t, err) - defer dbMgr.Close() - - registry := NewNodeRegistry(1, "localhost:5001") - client := NewCatchUpClient(1, dataDir, registry, nil) - client.SetDatabaseManager(dbMgr) - - require.NoError(t, client.persistSchemaVersions(map[string]uint64{"appdb": 6})) - - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) - require.NoError(t, err) - got, err := systemDB.GetMetaStore().GetSchemaVersion("appdb") - require.NoError(t, err) - require.Equal(t, int64(6), got) -} - -// Pins the exact failure this fix removes: without SetDatabaseManager, the -// runtime path would try to open the system MetaStore a second time while the -// DatabaseManager already holds it open, and fail. This proves -// persistSchemaVersions only avoids that failure because it dispatches to the -// live-manager path once one is set. -func TestCatchUpClient_PersistSchemaVersions_PathOpenFailsAgainstLiveManager(t *testing.T) { - dataDir := t.TempDir() - dbMgr, err := db.NewDatabaseManager(dataDir, 1, hlc.NewClock(1)) - require.NoError(t, err) - defer dbMgr.Close() - - registry := NewNodeRegistry(1, "localhost:5001") - client := NewCatchUpClient(1, dataDir, registry, nil) - // Deliberately do NOT call SetDatabaseManager, to force the path-based - // branch while dbMgr is already holding the MetaStore open. - - err = client.persistSchemaVersions(map[string]uint64{"appdb": 6}) - require.Error(t, err) -} - -// trailerOnlyStreamSnapshotServer implements just enough of MarmotServiceServer -// to prove schema versions set via stream.SetTrailer on the server side really -// do arrive at stream.Trailer() on the client side over a real gRPC -// connection - the mechanism SnapshotVersionsForRestore depends on. -type trailerOnlyStreamSnapshotServer struct { - UnimplementedMarmotServiceServer - trailer metadata.MD -} - -func (s *trailerOnlyStreamSnapshotServer) StreamSnapshot(req *SnapshotRequest, stream MarmotService_StreamSnapshotServer) error { - stream.SetTrailer(s.trailer) - return nil -} - -func TestStreamSnapshotTrailer_PropagatesOverRealGRPCConnection(t *testing.T) { - listener, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - defer listener.Close() - - wantVersions := map[string]uint64{"appdb": 4, "otherdb": 1} - mockServer := &trailerOnlyStreamSnapshotServer{trailer: snapshotSchemaVersionsTrailer(wantVersions)} - grpcServer := grpc.NewServer() - RegisterMarmotServiceServer(grpcServer, mockServer) - go func() { _ = grpcServer.Serve(listener) }() - defer grpcServer.Stop() - - conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) - require.NoError(t, err) - defer conn.Close() - - client := NewMarmotServiceClient(conn) - stream, err := client.StreamSnapshot(context.Background(), &SnapshotRequest{RequestingNodeId: 1}) - require.NoError(t, err) - - // Drain the stream: trailer metadata is only populated once Recv() has - // observed the RPC's end (io.EOF here, since the mock sends no chunks). - _, err = stream.Recv() - require.ErrorIs(t, err, io.EOF) - - got := SnapshotVersionsForRestore(nil, stream) - require.Equal(t, wantVersions, got) -} - // TestSnapshotFilesToRestore_RunningNodeInstallsOnlyItsDatabase: anti-entropy // on a running node restores one database. Every other file in the peer's // snapshot, the system database included, is open on this node; swapping it diff --git a/grpc/cdc_apply_helpers.go b/grpc/cdc_apply_helpers.go index 26a500c1..c651808a 100644 --- a/grpc/cdc_apply_helpers.go +++ b/grpc/cdc_apply_helpers.go @@ -1,6 +1,8 @@ package grpc import ( + "context" + "errors" "fmt" "github.com/maxpert/marmot/common" @@ -10,6 +12,20 @@ import ( "github.com/maxpert/marmot/protocol" ) +// ErrRowCountMismatch is returned by ApplyPulledEvent when a pulled +// ChangeEvent's statement count does not match its RowCount: the event is +// never applied. FetchTransactions +// already refuses to serve such a transaction server-side, but a puller must +// not trust the wire without its own check. +var ErrRowCountMismatch = errors.New("pulled change event: statement count does not match row_count") + +// ErrNilCapturedRow is returned by ApplyPulledEvent when a wire statement +// decodes to no captured row at all: an unrecognized +// payload shape, such as one a newer peer added and this node's binary +// predates. Silently dropping it would apply a partial transaction under the +// original RowCount, so the whole event is refused instead, nothing applied. +var ErrNilCapturedRow = errors.New("pulled change event: statement decoded to no captured row") + func wireDMLToOp(stmtType pb.StatementType) (db.OpType, error) { stmtCode, ok := common.FromWireType(stmtType) if !ok { @@ -23,46 +39,6 @@ func wireDMLToOp(stmtType pb.StatementType) (db.OpType, error) { } } -func StoreCapturedRowsFromStatements(metaStore db.MetaStore, txnID uint64, statements []*Statement) error { - if metaStore == nil || len(statements) == 0 { - return nil - } - for i, stmt := range statements { - data, err := encodedCapturedRowFromStatement(stmt) - if err != nil { - return err - } - if len(data) == 0 { - continue - } - if err := metaStore.WriteCapturedRow(txnID, uint64(i+1), data); err != nil { - return fmt.Errorf("write captured row: %w", err) - } - } - return nil -} - -func encodedCapturedRowFromStatement(stmt *Statement) ([]byte, error) { - if stmt == nil { - return nil, nil - } - if rowChange := stmt.GetRowChange(); rowChange != nil { - if _, err := decodeRowChange(stmt); err != nil { - return nil, err - } - return rowChange.EncodedRow, nil - } - row, err := capturedRowFromStatement(stmt) - if err != nil || row == nil { - return nil, err - } - data, err := db.EncodeRow(row) - if err != nil { - return nil, fmt.Errorf("encode captured row: %w", err) - } - return data, nil -} - func HLCToTimestamp(ts *HLC) hlc.Timestamp { if ts == nil { return hlc.Timestamp{} @@ -74,28 +50,92 @@ func HLCToTimestamp(ts *HLC) hlc.Timestamp { } } -func StoreAppliedChangeEvent(metaStore db.MetaStore, txnID uint64, timestamp *HLC, database string, statements []*Statement) (uint64, error) { - if metaStore == nil { +// StoreAppliedChangeEvent records a replayed transaction's log entry after +// it has been applied, delegating to db.ReplicatedDatabase.LogReplayedTxn +// (db/replay_apply.go), which this function's logic moved into. +// originNodeID is the transaction's coordinator (the DDL incarnation owner), +// not necessarily the peer this event was pulled from; +// requiredSchemaVersion is the value the originating commit recorded. +// Callers without either value (a rolling-upgrade peer that omits them on +// the wire) pass 0. +func StoreAppliedChangeEvent(mdb *db.ReplicatedDatabase, txnID uint64, timestamp *HLC, statements []*Statement, originNodeID uint64, requiredSchemaVersion uint64) (uint64, error) { + if mdb == nil { return 0, nil } - if rec, err := metaStore.GetTransaction(txnID); err == nil && rec != nil && rec.Status == db.TxnStatusCommitted { - return rec.SeqNum, nil - } - if err := StoreCapturedRowsFromStatements(metaStore, txnID, statements); err != nil { - return 0, err + rows := make([]*db.EncodedCapturedRow, 0, len(statements)) + for _, stmt := range statements { + row, err := capturedRowFromStatement(stmt) + if err != nil { + return 0, err + } + if row == nil { + continue + } + rows = append(rows, row) + } + return mdb.LogReplayedTxn(&db.ReplayTxn{ + TxnID: txnID, + OriginNodeID: originNodeID, + CommitTS: HLCToTimestamp(timestamp), + RequiredSchemaVersion: requiredSchemaVersion, + Rows: rows, + }) +} + +// ApplyPulledEvent applies one transaction pulled from a peer's local commit +// log via FetchTransactions, exactly-once and atomically with the +// applied-txn marker (db.ReplicatedDatabase.ApplyReplayedTxn). This is the +// log puller's (D2a) sole entry point for applying a pulled ChangeEvent: it +// is called in-process, directly, and never goes through +// TransactionRequest/ReplicateTransaction. +// +// It refuses (ErrRowCountMismatch, touching nothing) any event whose +// statement count does not match RowCount - the same check +// FetchTransactions applies server-side, repeated here because a puller must +// not trust the wire. OriginNodeID and CommitTS.NodeID both fall back to +// ev.Timestamp.NodeId when OriginNodeId is unset, for a rolling-upgrade peer +// that omits it. +func ApplyPulledEvent(ctx context.Context, dbMgr *db.DatabaseManager, ev *ChangeEvent) (bool, error) { + if ev == nil { + return false, fmt.Errorf("apply pulled event: nil event") + } + if ev.Database == "" { + return false, fmt.Errorf("apply pulled event: missing database") + } + if uint32(len(ev.Statements)) != ev.RowCount { + return false, fmt.Errorf("%w: txn %d has %d statements, want %d", ErrRowCountMismatch, ev.TxnId, len(ev.Statements), ev.RowCount) } - if err := metaStore.SealCapturedRows(txnID); err != nil { - return 0, err + + mdb, err := dbMgr.GetDatabase(ev.Database) + if err != nil { + return false, fmt.Errorf("apply pulled event: database %s: %w", ev.Database, err) } - commitTS := HLCToTimestamp(timestamp) - if err := metaStore.StoreReplayedTransaction(txnID, commitTS.NodeID, commitTS, database, uint32(len(statements))); err != nil { - return 0, err + + rows := make([]*db.EncodedCapturedRow, 0, len(ev.Statements)) + for _, stmt := range ev.Statements { + row, err := capturedRowFromStatement(stmt) + if err != nil { + return false, fmt.Errorf("apply pulled event: decode statement: %w", err) + } + if row == nil { + return false, fmt.Errorf("%w: txn %d", ErrNilCapturedRow, ev.TxnId) + } + rows = append(rows, row) } - rec, err := metaStore.GetTransaction(txnID) - if err != nil || rec == nil { - return 0, err + + origin := ev.OriginNodeId + if origin == 0 { + origin = ev.GetTimestamp().GetNodeId() } - return rec.SeqNum, nil + + t := &db.ReplayTxn{ + TxnID: ev.TxnId, + OriginNodeID: origin, + CommitTS: HLCToTimestamp(ev.Timestamp), + RequiredSchemaVersion: ev.RequiredSchemaVersion, + Rows: rows, + } + return mdb.ApplyReplayedTxn(ctx, t, true) } func capturedRowFromStatement(stmt *Statement) (*db.EncodedCapturedRow, error) { diff --git a/grpc/cdc_apply_helpers_test.go b/grpc/cdc_apply_helpers_test.go index 828d98cb..64143812 100644 --- a/grpc/cdc_apply_helpers_test.go +++ b/grpc/cdc_apply_helpers_test.go @@ -1,19 +1,29 @@ package grpc import ( - "path/filepath" + "context" + "errors" "testing" "github.com/maxpert/marmot/db" pb "github.com/maxpert/marmot/grpc/common" + "github.com/maxpert/marmot/hlc" ) func TestStoreAppliedChangeEventPersistsRowsAndIsIdempotent(t *testing.T) { - store, err := db.NewPebbleMetaStore(filepath.Join(t.TempDir(), "meta"), db.DefaultPebbleOptions()) + dbMgr, err := db.NewDatabaseManager(t.TempDir(), 1, hlc.NewClock(1)) if err != nil { - t.Fatalf("NewPebbleMetaStore: %v", err) + t.Fatalf("NewDatabaseManager: %v", err) } - defer store.Close() + defer dbMgr.Close() + if err := dbMgr.CreateDatabase("rag"); err != nil { + t.Fatalf("CreateDatabase: %v", err) + } + mdb, err := dbMgr.GetDatabase("rag") + if err != nil { + t.Fatalf("GetDatabase: %v", err) + } + store := mdb.GetMetaStore() statements := []*Statement{{ Type: pb.StatementType_INSERT, @@ -26,11 +36,11 @@ func TestStoreAppliedChangeEventPersistsRowsAndIsIdempotent(t *testing.T) { }} ts := &HLC{WallTime: 100, Logical: 2, NodeId: 7} - seq1, err := StoreAppliedChangeEvent(store, 42, ts, "rag", statements) + seq1, err := StoreAppliedChangeEvent(mdb, 42, ts, statements, 7, 3) if err != nil { t.Fatalf("StoreAppliedChangeEvent first call: %v", err) } - seq2, err := StoreAppliedChangeEvent(store, 42, ts, "rag", statements) + seq2, err := StoreAppliedChangeEvent(mdb, 42, ts, statements, 7, 3) if err != nil { t.Fatalf("StoreAppliedChangeEvent second call: %v", err) } @@ -45,6 +55,12 @@ func TestStoreAppliedChangeEventPersistsRowsAndIsIdempotent(t *testing.T) { if rec == nil || rec.Status != db.TxnStatusCommitted || rec.DatabaseName != "rag" { t.Fatalf("transaction record = %+v, want committed rag", rec) } + if rec.NodeID != 7 { + t.Fatalf("NodeID = %d, want the origin node id 7", rec.NodeID) + } + if rec.RequiredSchemaVersion != 3 { + t.Fatalf("RequiredSchemaVersion = %d, want 3", rec.RequiredSchemaVersion) + } cursor, err := store.IterateCapturedRows(42) if err != nil { @@ -69,3 +85,110 @@ func TestStoreAppliedChangeEventPersistsRowsAndIsIdempotent(t *testing.T) { t.Fatalf("cursor error: %v", err) } } + +// TestApplyPulledEventRefusesRowCountMismatch is ApplyPulledEvent's own +// RowCount check: a pulled ChangeEvent whose statement count does not match its +// RowCount must be refused, with nothing applied, even though +// FetchTransactions already checks this server-side - a puller must not +// trust the wire. +func TestApplyPulledEventRefusesRowCountMismatch(t *testing.T) { + dbMgr, err := db.NewDatabaseManager(t.TempDir(), 1, hlc.NewClock(1)) + if err != nil { + t.Fatalf("NewDatabaseManager: %v", err) + } + defer dbMgr.Close() + if err := dbMgr.CreateDatabase("rag"); err != nil { + t.Fatalf("CreateDatabase: %v", err) + } + mdb, err := dbMgr.GetDatabase("rag") + if err != nil { + t.Fatalf("GetDatabase: %v", err) + } + + statements := []*Statement{{ + Type: pb.StatementType_INSERT, + TableName: "docs", + Database: "rag", + Payload: &Statement_RowChange{RowChange: testInsertRowChange("docs", []byte("pk:1"), map[string][]byte{ + "id": {1}, + "name": []byte("doc"), + })}, + }} + ev := &ChangeEvent{ + TxnId: 77, + OriginNodeId: 7, + Database: "rag", + Timestamp: &HLC{WallTime: 100, Logical: 2, NodeId: 7}, + Statements: statements, + RowCount: 2, // one statement short of what it claims + } + + applied, err := ApplyPulledEvent(context.Background(), dbMgr, ev) + if applied { + t.Fatal("a row-count mismatch must never be applied") + } + if !errors.Is(err, ErrRowCountMismatch) { + t.Fatalf("err = %v, want ErrRowCountMismatch", err) + } + + rec, err := mdb.GetMetaStore().GetTransaction(77) + if err != nil { + t.Fatalf("GetTransaction: %v", err) + } + if rec != nil { + t.Fatalf("a refused row-count mismatch must not write a log entry, got: %+v", rec) + } +} + +// TestApplyPulledEventRefusesNilDecodedRow: a wire +// statement whose payload capturedRowFromStatement does not recognize (for +// example an unrecognized payload shape from a newer peer) decodes to a nil +// row. Before the fix, ApplyPulledEvent silently dropped it and applied the +// remaining rows anyway; it must instead refuse the whole event. +func TestApplyPulledEventRefusesNilDecodedRow(t *testing.T) { + dbMgr, err := db.NewDatabaseManager(t.TempDir(), 1, hlc.NewClock(1)) + if err != nil { + t.Fatalf("NewDatabaseManager: %v", err) + } + defer dbMgr.Close() + if err := dbMgr.CreateDatabase("rag"); err != nil { + t.Fatalf("CreateDatabase: %v", err) + } + mdb, err := dbMgr.GetDatabase("rag") + if err != nil { + t.Fatalf("GetDatabase: %v", err) + } + + // No Payload set at all: none of GetRowChange/GetDdlChange/ + // GetLoadDataChange/GetVectorIndexChange match, so + // capturedRowFromStatement returns a nil row with no error. + statements := []*Statement{{ + Type: pb.StatementType_INSERT, + TableName: "docs", + Database: "rag", + }} + ev := &ChangeEvent{ + TxnId: 78, + OriginNodeId: 7, + Database: "rag", + Timestamp: &HLC{WallTime: 100, Logical: 2, NodeId: 7}, + Statements: statements, + RowCount: 1, + } + + applied, err := ApplyPulledEvent(context.Background(), dbMgr, ev) + if applied { + t.Fatal("a nil decoded row must never be applied") + } + if !errors.Is(err, ErrNilCapturedRow) { + t.Fatalf("err = %v, want ErrNilCapturedRow", err) + } + + rec, err := mdb.GetMetaStore().GetTransaction(78) + if err != nil { + t.Fatalf("GetTransaction: %v", err) + } + if rec != nil { + t.Fatalf("a refused nil-row event must not write a log entry, got: %+v", rec) + } +} diff --git a/grpc/delta_sync.go b/grpc/delta_sync.go deleted file mode 100644 index 95962708..00000000 --- a/grpc/delta_sync.go +++ /dev/null @@ -1,534 +0,0 @@ -package grpc - -import ( - "context" - "fmt" - "io" - "time" - - "github.com/maxpert/marmot/cfg" - "github.com/maxpert/marmot/common" - "github.com/maxpert/marmot/db" - "github.com/maxpert/marmot/hlc" - "github.com/rs/zerolog/log" -) - -// DeltaSyncClient handles incremental replication catch-up -// It streams changes from a peer and applies them locally -type DeltaSyncClient struct { - client *Client - dbManager *db.DatabaseManager - nodeID uint64 - clock *hlc.Clock - applyTxnsFn ApplyTransactionFunc - schemaVersionMgr *db.SchemaVersionManager -} - -// ApplyTransactionFunc applies a replicated transaction to the local database -// This is injected to avoid circular dependencies with the replication handler -type ApplyTransactionFunc func(ctx context.Context, txnReq *TransactionRequest) (*TransactionResponse, error) - -// DeltaSyncConfig holds configuration for delta sync -type DeltaSyncConfig struct { - NodeID uint64 - Client *Client - DBManager *db.DatabaseManager - Clock *hlc.Clock - ApplyTxnsFn ApplyTransactionFunc - SchemaVersionMgr *db.SchemaVersionManager -} - -// NewDeltaSyncClient creates a new delta sync client -func NewDeltaSyncClient(config DeltaSyncConfig) *DeltaSyncClient { - return &DeltaSyncClient{ - client: config.Client, - dbManager: config.DBManager, - nodeID: config.NodeID, - clock: config.Clock, - applyTxnsFn: config.ApplyTxnsFn, - schemaVersionMgr: config.SchemaVersionMgr, - } -} - -// DeltaSyncResult contains the result of a delta sync operation -type DeltaSyncResult struct { - Database string - TxnsApplied int - LastAppliedTxnID uint64 - LastAppliedTS *hlc.Timestamp - ExpectedNextTxn uint64 // Track expected next txn_id for gap detection - Err error -} - -// SyncFromPeer performs delta sync for a specific database from a peer -// It streams changes from fromTxnID onwards and applies them locally -func (ds *DeltaSyncClient) SyncFromPeer(ctx context.Context, peerNodeID uint64, peerAddr string, database string, fromTxnID uint64) (*DeltaSyncResult, error) { - result := &DeltaSyncResult{ - Database: database, - TxnsApplied: 0, - LastAppliedTxnID: fromTxnID, - } - - // Get client connection to peer - client, err := ds.client.GetClientByAddress(peerAddr) - if err != nil { - result.Err = fmt.Errorf("failed to connect to peer %d (%s): %w", peerNodeID, peerAddr, err) - return result, result.Err - } - - log.Info(). - Uint64("peer_node", peerNodeID). - Str("database", database). - Uint64("from_txn_id", fromTxnID). - Msg("Starting delta sync from peer") - - // Stream changes from peer - stream, err := client.StreamChanges(ctx, &StreamRequest{ - FromTxnId: fromTxnID, - RequestingNodeId: ds.nodeID, - Database: database, - }) - if err != nil { - result.Err = fmt.Errorf("failed to start change stream: %w", err) - return result, result.Err - } - - // Apply changes as they stream in - // Initialize expected next txn to detect both initial and intermediate gaps - result.ExpectedNextTxn = fromTxnID + 1 - - for { - event, err := stream.Recv() - if err == io.EOF { - // Stream completed successfully - break - } - if err != nil { - result.Err = fmt.Errorf("stream error: %w", err) - return result, result.Err - } - - // Skip if we've already applied this transaction - if event.TxnId <= fromTxnID { - log.Debug(). - Uint64("txn_id", event.TxnId). - Msg("Skipping already-applied transaction") - continue - } - - // Gap detection: Check EVERY event for missing transactions - // If seq_num is available, use it (more reliable than txn_id) - // Otherwise fall back to txn_id based gap detection - if event.SeqNum > 0 && result.ExpectedNextTxn > 0 { - // Use seq_num based gap detection (preferred) - if event.SeqNum > result.ExpectedNextTxn { - gap := event.SeqNum - result.ExpectedNextTxn - if gap > uint64(cfg.Config.Replication.DeltaSyncThresholdTxns) { - result.Err = fmt.Errorf( - "gap detected: expected seq_num %d but received %d (gap: %d, threshold: %d). "+ - "Transactions likely GC'd - full snapshot required", - result.ExpectedNextTxn, - event.SeqNum, - gap, - cfg.Config.Replication.DeltaSyncThresholdTxns, - ) - log.Warn(). - Uint64("expected_seq", result.ExpectedNextTxn). - Uint64("received_seq", event.SeqNum). - Uint64("gap", gap). - Int("threshold", cfg.Config.Replication.DeltaSyncThresholdTxns). - Str("database", database). - Msg("Gap detected in sequence stream - snapshot required") - return result, result.Err - } - } - // Update expected next seq_num - result.ExpectedNextTxn = event.SeqNum + 1 - } else { - // Fallback to txn_id based gap detection - if event.TxnId > result.ExpectedNextTxn { - gap := event.TxnId - result.ExpectedNextTxn - if gap > uint64(cfg.Config.Replication.DeltaSyncThresholdTxns) { - result.Err = fmt.Errorf( - "gap detected: expected txn_id %d but received %d (gap: %d txns, threshold: %d). "+ - "Transactions likely GC'd - full snapshot required", - result.ExpectedNextTxn, - event.TxnId, - gap, - cfg.Config.Replication.DeltaSyncThresholdTxns, - ) - log.Warn(). - Uint64("expected_txn", result.ExpectedNextTxn). - Uint64("received_txn", event.TxnId). - Uint64("gap", gap). - Int("threshold", cfg.Config.Replication.DeltaSyncThresholdTxns). - Str("database", database). - Msg("Gap detected in transaction stream - snapshot required") - return result, result.Err - } - } - // Update expected next txn_id - result.ExpectedNextTxn = event.TxnId + 1 - } - - // Apply the transaction locally - if err := ds.applyChangeEvent(ctx, event); err != nil { - result.Err = fmt.Errorf("failed to apply txn %d: %w", event.TxnId, err) - return result, result.Err - } - - // Update result - result.TxnsApplied++ - result.LastAppliedTxnID = event.TxnId - result.LastAppliedTS = &hlc.Timestamp{ - WallTime: event.Timestamp.WallTime, - Logical: event.Timestamp.Logical, - NodeID: event.Timestamp.NodeId, - } - - // Log progress every 100 transactions - if result.TxnsApplied%100 == 0 { - log.Info(). - Uint64("peer_node", peerNodeID). - Str("database", database). - Int("txns_applied", result.TxnsApplied). - Uint64("last_txn_id", result.LastAppliedTxnID). - Msg("Delta sync progress") - } - } - - // Update replication state in database - if result.TxnsApplied > 0 { - repState := &db.ReplicationState{ - PeerNodeID: peerNodeID, - DatabaseName: database, - LastAppliedTxnID: result.LastAppliedTxnID, - LastAppliedTSWall: result.LastAppliedTS.WallTime, - LastAppliedTSLog: result.LastAppliedTS.Logical, - LastSyncTime: time.Now().UnixNano(), - SyncStatus: "SYNCED", - } - - if err := ds.dbManager.UpdateReplicationState(repState); err != nil { - log.Warn().Err(err).Msg("Failed to update replication state") - // Don't fail the whole sync for this - } - } - - log.Info(). - Uint64("peer_node", peerNodeID). - Str("database", database). - Int("txns_applied", result.TxnsApplied). - Uint64("final_txn_id", result.LastAppliedTxnID). - Msg("Delta sync completed") - - return result, nil -} - -// SyncAllDatabasesFromPeer performs delta sync for all databases from a peer -// Returns a map of database name to sync result -func (ds *DeltaSyncClient) SyncAllDatabasesFromPeer(ctx context.Context, peerNodeID uint64, peerAddr string) (map[string]*DeltaSyncResult, error) { - results := make(map[string]*DeltaSyncResult) - - // Get list of all databases - databases := ds.dbManager.ListDatabases() - - log.Info(). - Uint64("peer_node", peerNodeID). - Int("database_count", len(databases)). - Msg("Starting multi-database delta sync") - - for _, dbName := range databases { - // Get last applied txn_id for this peer/database - repState, err := ds.dbManager.GetReplicationState(peerNodeID, dbName) - var fromTxnID uint64 - if err != nil { - // No state yet, start from beginning - fromTxnID = 0 - } else { - fromTxnID = repState.LastAppliedTxnID - } - - // Sync this database - result, err := ds.SyncFromPeer(ctx, peerNodeID, peerAddr, dbName, fromTxnID) - if err != nil { - log.Warn(). - Err(err). - Str("database", dbName). - Msg("Failed to sync database") - // Continue with other databases - } - results[dbName] = result - } - - return results, nil -} - -// applyChangeEvent applies a single change event to the local database -func (ds *DeltaSyncClient) applyChangeEvent(ctx context.Context, event *ChangeEvent) error { - // Determine target database - database name is required - database := event.Database - if database == "" { - return fmt.Errorf("change event %d missing database name", event.TxnId) - } - - // Schema version validation: Check if we need to catch up DDL first - if ds.schemaVersionMgr != nil && event.RequiredSchemaVersion > 0 { - localVersion, err := ds.schemaVersionMgr.GetSchemaVersion(database) - if err != nil { - log.Warn().Err(err).Str("database", database).Msg("Failed to get local schema version") - } else if localVersion < event.RequiredSchemaVersion { - // We're behind! This transaction requires a newer schema version - // This means we missed a DDL statement - we need to catch up - log.Warn(). - Str("database", database). - Uint64("local_version", localVersion). - Uint64("required_version", event.RequiredSchemaVersion). - Uint64("txn_id", event.TxnId). - Msg("Schema version gap detected - transaction requires newer schema") - - // For now, we'll try to apply anyway (DDL should be idempotent) - // In a more sophisticated implementation, we would: - // 1. Fetch missing DDL transactions from peer - // 2. Apply them in order - // 3. Then apply this transaction - // But since DDL is replicated through the same stream and is idempotent, - // we should have already received it if we're streaming in order - } - } - - // Convert to TransactionRequest using REPLAY phase - // REPLAY bypasses 2PC state tracking since these transactions are already committed on source - txnReq := &TransactionRequest{ - TxnId: event.TxnId, - SourceNodeId: event.Timestamp.NodeId, // Use original source node ID - Statements: event.Statements, - Timestamp: event.Timestamp, - Phase: TransactionPhase_REPLAY, // Use REPLAY for anti-entropy catch-up - Consistency: ConsistencyLevel_CONSISTENCY_ONE, - Database: database, - } - - // Apply using the injected function - if ds.applyTxnsFn != nil { - resp, err := ds.applyTxnsFn(ctx, txnReq) - if err != nil { - return err - } - if !resp.Success { - return fmt.Errorf("replication failed: %s", resp.ErrorMessage) - } - return nil - } - - // Fallback: apply directly to database (simplified) - // In production, this should go through the full replication handler - mdb, err := ds.dbManager.GetDatabase(database) - if err != nil { - return fmt.Errorf("database %s not found: %w", database, err) - } - if len(event.Statements) == 1 { - if change := event.Statements[0].GetVectorIndexChange(); change != nil { - if err := ds.applyVectorIndexChange(ctx, vectorChangeFromProto(change)); err != nil { - return err - } - _, err := StoreAppliedChangeEvent(mdb.GetMetaStore(), event.TxnId, event.Timestamp, database, event.Statements) - return err - } - } - - // Execute each statement in a transaction - tx, err := mdb.GetDB().BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer tx.Rollback() - var schemaChange db.SchemaChange - - for _, stmt := range event.Statements { - // Check for CDC data (RowChange payload) - if rowChange := stmt.GetRowChange(); rowChange != nil && len(rowChange.EncodedRow) > 0 { - // CDC path: apply row data directly using unified applier - schemaAdapter := &deltaSyncSchemaAdapter{dbMgr: ds.dbManager, dbName: database} - opType, opErr := wireDMLToOp(stmt.Type) - if opErr != nil { - return opErr - } - row, rowErr := decodeRowChange(stmt) - if rowErr != nil { - return rowErr - } - if err := db.ApplyCDCValues(tx, schemaAdapter, opType, stmt.TableName, row.OldValues, row.NewValues); err != nil { - return fmt.Errorf("failed to apply CDC statement: %w", err) - } - log.Debug(). - Str("table", stmt.TableName). - Hex("intent_key", row.IntentKey). - Int("new_values", len(row.NewValues)). - Int("old_values", len(row.OldValues)). - Msg("DELTA-SYNC: Applied CDC data") - continue - } - - // LOAD DATA path. - if loadData := stmt.GetLoadDataChange(); loadData != nil { - if _, err := db.ApplyLoadDataInTx(tx, loadData.Sql, loadData.Data); err != nil { - return fmt.Errorf("failed to apply LOAD DATA statement: %w", err) - } - log.Debug().Msg("DELTA-SYNC: Applied LOAD DATA") - continue - } - - // SQL path: execute SQL statement - sql := stmt.GetSQL() - if sql == "" { - // CRITICAL: Don't silently skip - this means CDC data was lost during serialization - // Fail the sync so anti-entropy knows to try again or use snapshot - return fmt.Errorf("statement has no SQL and no CDC data (table=%s, type=%d) - CDC data may have been lost during serialization", stmt.TableName, stmt.Type) - } - if err := mdb.ApplyReplayedDDL(ctx, tx, sql, stmt.TableName, event.Timestamp.GetNodeId(), &schemaChange); err != nil { - return fmt.Errorf("failed to execute statement: %w", err) - } - log.Debug(). - Str("sql_prefix", func() string { - if len(sql) > 50 { - return sql[:50] - } - return sql - }()). - Msg("DELTA-SYNC: Executed SQL") - } - - if err := mdb.FinishReplayedSchemaChange(tx, &schemaChange); err != nil { - return fmt.Errorf("failed to finish replayed DDL: %w", err) - } - - // Commit the transaction - if err := db.MarkSQLiteTxnApplied(tx, event.TxnId, HLCToTimestamp(event.Timestamp)); err != nil { - return fmt.Errorf("failed to mark applied txn: %w", err) - } - if err := tx.Commit(); err != nil { - return fmt.Errorf("failed to commit: %w", err) - } - if !schemaChange.Empty() { - if err := mdb.ReloadSchema(); err != nil { - log.Warn().Err(err).Str("database", database).Msg("DELTA-SYNC: failed to reload schema after DDL") - } - } - seqNum, err := StoreAppliedChangeEvent(mdb.GetMetaStore(), event.TxnId, event.Timestamp, database, event.Statements) - if err != nil { - return fmt.Errorf("failed to store delta-sync rows: %w", err) - } - if err := ds.applyVectorCDCFromEvent(ctx, database, event.TxnId, seqNum, event.Statements); err != nil { - log.Error(). - Err(err). - Str("database", database). - Uint64("txn_id", event.TxnId). - Uint64("seq_num", seqNum). - Msg("DELTA-SYNC: vector CDC failed after row commit; local vector index is dirty") - } - - return nil -} - -func (ds *DeltaSyncClient) applyVectorIndexChange(ctx context.Context, change common.VectorIndexChange) error { - vecMgr := ds.dbManager.GetVectorIndexManager() - if vecMgr == nil { - return fmt.Errorf("vector index manager not configured") - } - applier, ok := vecMgr.(interface { - ApplyVectorControl(context.Context, common.VectorIndexChange) error - }) - if !ok { - return fmt.Errorf("vector index manager cannot apply replicated control metadata") - } - return applier.ApplyVectorControl(ctx, change) -} - -func (ds *DeltaSyncClient) applyVectorCDCFromEvent(ctx context.Context, database string, txnID, seqNum uint64, statements []*Statement) error { - entries := make([]common.CDCEntry, 0, len(statements)) - for _, stmt := range statements { - rowChange := stmt.GetRowChange() - if rowChange == nil || len(rowChange.EncodedRow) == 0 { - continue - } - row, err := decodeRowChange(stmt) - if err != nil { - return err - } - entries = append(entries, common.CDCEntry{ - Table: stmt.TableName, - IntentKey: row.IntentKey, - Operation: row.Op, - OldValues: row.OldValues, - NewValues: row.NewValues, - EncodedRow: rowChange.EncodedRow, - EncodedCodec: rowChange.EncodedRowCodec, - CommitTxnID: txnID, - CommitSeqNum: seqNum, - }) - } - if len(entries) == 0 { - return nil - } - vecMgr := ds.dbManager.GetVectorIndexManager() - if vecMgr == nil { - return nil - } - applier, ok := vecMgr.(interface { - ApplyCommittedVectorCDC(context.Context, string, uint64, uint64, []common.CDCEntry) error - }) - if !ok { - return nil - } - if err := applier.ApplyCommittedVectorCDC(ctx, database, txnID, seqNum, entries); err != nil { - return fmt.Errorf("DELTA-SYNC: apply vector CDC: %w", err) - } - return nil -} - -// deltaSyncSchemaAdapter adapts DatabaseManager schema access to CDCSchemaProvider -type deltaSyncSchemaAdapter struct { - dbMgr *db.DatabaseManager - dbName string -} - -func (a *deltaSyncSchemaAdapter) GetPrimaryKeys(tableName string) ([]string, error) { - dbInstance, err := a.dbMgr.GetDatabase(a.dbName) - if err != nil { - return nil, fmt.Errorf("database %s not found: %w", a.dbName, err) - } - schema, err := dbInstance.GetCachedTableSchema(tableName) - if err != nil { - return nil, fmt.Errorf("schema not found for table %s: %w", tableName, err) - } - return schema.PrimaryKeys, nil -} - -// GetPeerReplicationLag queries a peer to get its current replication state -// Returns the lag in number of transactions for each database -func (ds *DeltaSyncClient) GetPeerReplicationLag(ctx context.Context, peerAddr string, database string) (map[string]int64, error) { - client, err := ds.client.GetClientByAddress(peerAddr) - if err != nil { - return nil, fmt.Errorf("failed to connect to peer: %w", err) - } - - // Get peer's replication state - resp, err := client.GetReplicationState(ctx, &ReplicationStateRequest{ - RequestingNodeId: ds.nodeID, - Database: database, // Empty = all databases - }) - if err != nil { - return nil, fmt.Errorf("failed to get replication state: %w", err) - } - - // Calculate lag for each database - lags := make(map[string]int64) - for _, state := range resp.States { - // Lag = peer's current max - what we've applied - lag := int64(state.CurrentMaxTxnId) - int64(state.LastAppliedTxnId) - lags[state.DatabaseName] = lag - } - - return lags, nil -} diff --git a/grpc/delta_sync_apply_test.go b/grpc/delta_sync_apply_test.go deleted file mode 100644 index 9d1282d1..00000000 --- a/grpc/delta_sync_apply_test.go +++ /dev/null @@ -1,249 +0,0 @@ -package grpc - -import ( - "context" - "errors" - "os" - "testing" - "time" - - "github.com/maxpert/marmot/common" - "github.com/maxpert/marmot/db" - pb "github.com/maxpert/marmot/grpc/common" - "github.com/maxpert/marmot/hlc" -) - -func TestDeltaSyncApplyChangeEvent_DDLIdempotent(t *testing.T) { - tmpDir, dbMgr, schemaVersionMgr := setupTestEnvironment(t, "test_delta_apply_ddl_idempotent") - defer os.RemoveAll(tmpDir) - defer dbMgr.Close() - - const testDB = "test_delta_ddl" - if err := dbMgr.CreateDatabase(testDB); err != nil { - t.Fatalf("Failed to create test database: %v", err) - } - - ds := NewDeltaSyncClient(DeltaSyncConfig{ - NodeID: 1, - DBManager: dbMgr, - Clock: hlc.NewClock(1), - SchemaVersionMgr: schemaVersionMgr, - }) - - event := &ChangeEvent{ - TxnId: 300, - Database: testDB, - Timestamp: &HLC{ - WallTime: 1, - Logical: 1, - NodeId: 2, - }, - Statements: []*Statement{ - { - Type: pb.StatementType_DDL, - Database: testDB, - Payload: &Statement_DdlChange{ - DdlChange: &DDLChange{ - Sql: "CREATE TABLE delta_users (id INTEGER PRIMARY KEY, name TEXT)", - }, - }, - }, - }, - } - - for i := 0; i < 2; i++ { - if err := ds.applyChangeEvent(context.Background(), event); err != nil { - t.Fatalf("applyChangeEvent call %d should be idempotent, got error: %v", i+1, err) - } - } -} - -func TestDeltaSyncApplyChangeEvent_ReloadsSchemaAfterDDL(t *testing.T) { - tmpDir, dbMgr, schemaVersionMgr := setupTestEnvironment(t, "test_delta_apply_schema_reload") - defer os.RemoveAll(tmpDir) - defer dbMgr.Close() - - const testDB = "test_delta_schema" - if err := dbMgr.CreateDatabase(testDB); err != nil { - t.Fatalf("Failed to create test database: %v", err) - } - - ds := NewDeltaSyncClient(DeltaSyncConfig{ - NodeID: 1, - DBManager: dbMgr, - Clock: hlc.NewClock(1), - SchemaVersionMgr: schemaVersionMgr, - }) - - ddlEvent := &ChangeEvent{ - TxnId: 400, - Database: testDB, - Timestamp: &HLC{ - WallTime: 1, - Logical: 1, - NodeId: 2, - }, - Statements: []*Statement{ - { - Type: pb.StatementType_DDL, - TableName: "delta_updates", - Database: testDB, - Payload: &Statement_DdlChange{ - DdlChange: &DDLChange{ - Sql: "CREATE TABLE delta_updates (id INTEGER PRIMARY KEY, name TEXT)", - }, - }, - }, - }, - } - if err := ds.applyChangeEvent(context.Background(), ddlEvent); err != nil { - t.Fatalf("applyChangeEvent DDL failed: %v", err) - } - - insertEvent := &ChangeEvent{ - TxnId: 401, - Database: testDB, - Timestamp: &HLC{ - WallTime: 2, - Logical: 1, - NodeId: 2, - }, - Statements: []*Statement{ - { - Type: pb.StatementType_INSERT, - TableName: "delta_updates", - Database: testDB, - Payload: &Statement_RowChange{ - RowChange: testInsertRowChange("delta_updates", []byte("delta_updates:1"), map[string][]byte{ - "id": mustMarshalMsgpack(t, int64(1)), - "name": mustMarshalMsgpack(t, "alice"), - }), - }, - }, - }, - } - if err := ds.applyChangeEvent(context.Background(), insertEvent); err != nil { - t.Fatalf("applyChangeEvent INSERT failed: %v", err) - } - - updateEvent := &ChangeEvent{ - TxnId: 402, - Database: testDB, - Timestamp: &HLC{ - WallTime: 3, - Logical: 1, - NodeId: 2, - }, - Statements: []*Statement{ - { - Type: pb.StatementType_UPDATE, - TableName: "delta_updates", - Database: testDB, - Payload: &Statement_RowChange{ - RowChange: testUpdateRowChange("delta_updates", []byte("delta_updates:1"), map[string][]byte{ - "id": mustMarshalMsgpack(t, int64(1)), - "name": mustMarshalMsgpack(t, "alice"), - }, map[string][]byte{ - "id": mustMarshalMsgpack(t, int64(1)), - "name": mustMarshalMsgpack(t, "bob"), - }), - }, - }, - }, - } - if err := ds.applyChangeEvent(context.Background(), updateEvent); err != nil { - t.Fatalf("applyChangeEvent UPDATE after DDL should succeed, got: %v", err) - } -} - -func TestDeltaSyncApplyChangeEvent_VectorCDCFailureDoesNotBlockRows(t *testing.T) { - tmpDir, dbMgr, schemaVersionMgr := setupTestEnvironment(t, "test_delta_vector_dirty") - defer os.RemoveAll(tmpDir) - defer dbMgr.Close() - - const testDB = "test_delta_vector" - if err := dbMgr.CreateDatabase(testDB); err != nil { - t.Fatalf("Failed to create test database: %v", err) - } - mdb, err := dbMgr.GetDatabase(testDB) - if err != nil { - t.Fatalf("Failed to get test database: %v", err) - } - if _, err := mdb.GetDB().Exec(`CREATE TABLE docs (id INTEGER PRIMARY KEY, embed BLOB, title TEXT)`); err != nil { - t.Fatalf("Failed to create docs: %v", err) - } - if err := mdb.ReloadSchema(); err != nil { - t.Fatalf("ReloadSchema: %v", err) - } - - vecMgr := db.NewVectorIndexManager(dbMgr) - vecMgr.SetLifecycleHook(focusedVectorHook{err: errors.New("overlay write failed")}) - dbMgr.SetVectorIndexManager(vecMgr) - if err := vecMgr.ApplyVectorControl(context.Background(), common.VectorIndexChange{ - Action: common.VectorIndexActionCreate, - Database: testDB, - IndexName: "docs_embed_idx", - TableName: "docs", - ColumnName: "embed", - Metric: "cosine", - Dim: 4, - Nlist: 8, - Nprobe: 8, - TargetPartitionSize: 512, - CreatedAt: time.Now().UnixNano(), - }); err != nil { - t.Fatalf("ApplyVectorControl: %v", err) - } - - ds := NewDeltaSyncClient(DeltaSyncConfig{ - NodeID: 1, - DBManager: dbMgr, - Clock: hlc.NewClock(1), - SchemaVersionMgr: schemaVersionMgr, - }) - - event := &ChangeEvent{ - TxnId: 403, - Database: testDB, - Timestamp: &HLC{ - WallTime: 4, - Logical: 1, - NodeId: 2, - }, - Statements: []*Statement{ - { - Type: pb.StatementType_INSERT, - TableName: "docs", - Database: testDB, - Payload: &Statement_RowChange{ - RowChange: testInsertRowChange("docs", []byte("docs:1"), map[string][]byte{ - "id": mustMarshalMsgpack(t, int64(1)), - "embed": mustMarshalMsgpack(t, []byte{1, 2, 3, 4}), - "title": mustMarshalMsgpack(t, "delta vector row"), - }), - }, - }, - }, - } - - if err := ds.applyChangeEvent(context.Background(), event); err != nil { - t.Fatalf("vector CDC failure should not block delta row replication: %v", err) - } - var title string - if err := mdb.GetDB().QueryRow(`SELECT title FROM docs WHERE id = 1`).Scan(&title); err != nil { - t.Fatalf("Failed to query delta row: %v", err) - } - if title != "delta vector row" { - t.Fatalf("title=%q, want delta vector row", title) - } - var status string - if err := mdb.GetDB().QueryRow( - `SELECT status FROM __marmot_vector_indexes WHERE index_name = ?`, - "docs_embed_idx", - ).Scan(&status); err != nil { - t.Fatalf("Failed to read vector status: %v", err) - } - if status != "dirty" { - t.Fatalf("vector status=%q, want dirty", status) - } -} diff --git a/grpc/delta_sync_test.go b/grpc/delta_sync_test.go deleted file mode 100644 index ea3ede71..00000000 --- a/grpc/delta_sync_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package grpc - -import ( - "testing" - - "github.com/maxpert/marmot/cfg" -) - -// TestDeltaSyncIntermediateGapDetection tests that gaps in the middle of stream are detected -func TestDeltaSyncIntermediateGapDetection(t *testing.T) { - // Set up threshold for testing - originalThreshold := cfg.Config.Replication.DeltaSyncThresholdTxns - cfg.Config.Replication.DeltaSyncThresholdTxns = 10 // Small threshold for testing - defer func() { cfg.Config.Replication.DeltaSyncThresholdTxns = originalThreshold }() - - tests := []struct { - name string - fromTxnID uint64 - receivedTxnIDs []uint64 - expectGapAfter int // Index after which we expect gap error (-1 = no gap) - }{ - { - name: "No gap - sequential", - fromTxnID: 10, - receivedTxnIDs: []uint64{11, 12, 13, 14, 15}, - expectGapAfter: -1, - }, - { - name: "Initial gap - first event too far", - fromTxnID: 10, - receivedTxnIDs: []uint64{50, 51, 52}, // Gap of 39 from 10, exceeds threshold - expectGapAfter: 0, // Gap detected on first event - }, - { - name: "Intermediate gap", - fromTxnID: 10, - receivedTxnIDs: []uint64{11, 12, 13, 50, 51}, // Gap between 13→50 - expectGapAfter: 3, // Gap detected on 4th event (index 3) - }, - { - name: "Small gap within threshold", - fromTxnID: 10, - receivedTxnIDs: []uint64{11, 15, 16, 17}, // Gap of 3 from 11→15, within threshold - expectGapAfter: -1, - }, - { - name: "Gap exactly at threshold", - fromTxnID: 10, - receivedTxnIDs: []uint64{11, 22, 23}, // Gap of 10 from 11→22, exactly at threshold - expectGapAfter: -1, // Threshold is ">" not ">=" - }, - { - name: "Gap just over threshold", - fromTxnID: 10, - receivedTxnIDs: []uint64{11, 23, 24}, // Gap of 11 from 11→23, over threshold - expectGapAfter: 1, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := &DeltaSyncResult{ - ExpectedNextTxn: tt.fromTxnID + 1, - } - - gapDetectedAt := -1 - for i, txnID := range tt.receivedTxnIDs { - // Skip already-applied transactions - if txnID <= tt.fromTxnID { - continue - } - - // Check for gap - if txnID > result.ExpectedNextTxn { - gap := txnID - result.ExpectedNextTxn - if gap > uint64(cfg.Config.Replication.DeltaSyncThresholdTxns) { - gapDetectedAt = i - break - } - } - - // Update expected next - result.ExpectedNextTxn = txnID + 1 - } - - if gapDetectedAt != tt.expectGapAfter { - t.Errorf("Gap detection mismatch: expected gap at index %d, got %d", - tt.expectGapAfter, gapDetectedAt) - } - }) - } -} - -// TestDeltaSyncResultExpectedNextTxn tests that ExpectedNextTxn is properly tracked -func TestDeltaSyncResultExpectedNextTxn(t *testing.T) { - result := &DeltaSyncResult{ - ExpectedNextTxn: 100, - } - - // Simulate receiving sequential events - for txnID := uint64(100); txnID <= 105; txnID++ { - result.ExpectedNextTxn = txnID + 1 - result.LastAppliedTxnID = txnID - result.TxnsApplied++ - } - - if result.ExpectedNextTxn != 106 { - t.Errorf("Expected ExpectedNextTxn=106, got %d", result.ExpectedNextTxn) - } - - if result.TxnsApplied != 6 { - t.Errorf("Expected TxnsApplied=6, got %d", result.TxnsApplied) - } - - if result.LastAppliedTxnID != 105 { - t.Errorf("Expected LastAppliedTxnID=105, got %d", result.LastAppliedTxnID) - } -} diff --git a/grpc/log_protocol_version_test.go b/grpc/log_protocol_version_test.go new file mode 100644 index 00000000..5bbca2c9 --- /dev/null +++ b/grpc/log_protocol_version_test.go @@ -0,0 +1,170 @@ +package grpc + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestNodeRegistry_SelfEntryReportsLogPullProtocolVersion verifies a freshly +// constructed registry's own node state advertises LogPullProtocolVersion, and +// that GetAll (what gossip actually sends) carries it through copyNodeState. +// Without setting the field on the self entry, or without copying it in +// copyNodeState, every peer would observe this node as version 0 - the same +// as an older release that does not serve the commit-log pull protocol - and +// the rolling-upgrade gate would refuse DDL forever even on a fully upgraded +// cluster. +func TestNodeRegistry_SelfEntryReportsLogPullProtocolVersion(t *testing.T) { + nr := NewNodeRegistry(1, "localhost:8081") + + self, ok := nr.Get(1) + if !ok { + t.Fatal("expected self entry in registry") + } + if self.LogProtocolVersion != LogPullProtocolVersion { + t.Errorf("Get(self).LogProtocolVersion = %d, want %d", self.LogProtocolVersion, LogPullProtocolVersion) + } + + all := nr.GetAll() + if len(all) != 1 { + t.Fatalf("expected 1 node, got %d", len(all)) + } + if all[0].LogProtocolVersion != LogPullProtocolVersion { + t.Errorf("GetAll()[0].LogProtocolVersion = %d, want %d (gossip would send a stale version)", all[0].LogProtocolVersion, LogPullProtocolVersion) + } +} + +// TestNodeRegistry_UpdateNeverDowngradesLogProtocolVersion_SameIncarnation +// pins that a relayed same-incarnation gossip copy of a node's +// own state can lack the version an earlier copy carried (for example a +// seed's Join response, which builds a bare NodeState for the joining node). +// Update() must take the max for the same incarnation, never overwrite a +// known version with a lower or missing one. +func TestNodeRegistry_UpdateNeverDowngradesLogProtocolVersion_SameIncarnation(t *testing.T) { + nr := NewNodeRegistry(1, "localhost:8081") + + nr.Add(&NodeState{NodeId: 2, Address: "node2:8080", Status: NodeStatus_ALIVE, Incarnation: 5, LogProtocolVersion: LogPullProtocolVersion}) + + // A same-incarnation relay that lacks the field (zero value) must not + // downgrade the known version. + nr.Update(&NodeState{NodeId: 2, Address: "node2:8080", Status: NodeStatus_ALIVE, Incarnation: 5, LogProtocolVersion: 0}) + + got, ok := nr.Get(2) + if !ok { + t.Fatal("expected node 2 in registry") + } + if got.LogProtocolVersion != LogPullProtocolVersion { + t.Errorf("LogProtocolVersion downgraded: got %d, want %d", got.LogProtocolVersion, LogPullProtocolVersion) + } +} + +// TestNodeRegistry_UpdateTakesMaxLogProtocolVersion_SameIncarnation covers the +// other half: a same-incarnation update carrying a HIGHER version than what +// is on record must raise it (the max, not "first write wins"). +func TestNodeRegistry_UpdateTakesMaxLogProtocolVersion_SameIncarnation(t *testing.T) { + nr := NewNodeRegistry(1, "localhost:8081") + + nr.Add(&NodeState{NodeId: 2, Address: "node2:8080", Status: NodeStatus_ALIVE, Incarnation: 5, LogProtocolVersion: 0}) + nr.Update(&NodeState{NodeId: 2, Address: "node2:8080", Status: NodeStatus_ALIVE, Incarnation: 5, LogProtocolVersion: LogPullProtocolVersion}) + + got, ok := nr.Get(2) + if !ok { + t.Fatal("expected node 2 in registry") + } + if got.LogProtocolVersion != LogPullProtocolVersion { + t.Errorf("LogProtocolVersion not raised: got %d, want %d", got.LogProtocolVersion, LogPullProtocolVersion) + } +} + +// TestNodeRegistry_LegacyLogProtocolMembers verifies the membership scan the +// coordinator and PREPARE handler gate on: every member except REMOVED, +// self included, whose LogProtocolVersion is below LogPullProtocolVersion, +// sorted ascending. +func TestNodeRegistry_LegacyLogProtocolMembers(t *testing.T) { + nr := NewNodeRegistry(1, "localhost:8081") // self: LogPullProtocolVersion + + nr.Add(&NodeState{NodeId: 4, Address: "n4", Status: NodeStatus_ALIVE, LogProtocolVersion: LogPullProtocolVersion}) + nr.Add(&NodeState{NodeId: 3, Address: "n3", Status: NodeStatus_SUSPECT, LogProtocolVersion: 0}) // older release + nr.Add(&NodeState{NodeId: 2, Address: "n2", Status: NodeStatus_ALIVE, LogProtocolVersion: 0}) // older release + nr.Add(&NodeState{NodeId: 5, Address: "n5", Status: NodeStatus_REMOVED, LogProtocolVersion: 0}) // excluded: REMOVED + + got := nr.LegacyLogProtocolMembers() + want := []uint64{2, 3} + if !reflect.DeepEqual(got, want) { + t.Errorf("LegacyLogProtocolMembers() = %v, want %v", got, want) + } +} + +// TestNodeRegistry_LegacyLogProtocolMembers_EmptyWhenEveryMemberIsCurrent verifies the +// all-upgraded cluster reports no legacy members, so DDL is not refused. +func TestNodeRegistry_LegacyLogProtocolMembers_EmptyWhenEveryMemberIsCurrent(t *testing.T) { + nr := NewNodeRegistry(1, "localhost:8081") + nr.Add(&NodeState{NodeId: 2, Address: "n2", Status: NodeStatus_ALIVE, LogProtocolVersion: LogPullProtocolVersion}) + + got := nr.LegacyLogProtocolMembers() + if len(got) != 0 { + t.Errorf("LegacyLogProtocolMembers() = %v, want empty", got) + } +} + +// TestNodeRegistry_RefutesStaleLogProtocolVersionOfSelf pins the rolling +// upgrade's last step: a node restarted onto the new release keeps its old +// incarnation, while its peers may still hold their view of it from before +// the upgrade at a higher incarnation. Such a view must be refuted with a +// higher incarnation still - SWIM never lets a lower incarnation replace it +// - or every peer keeps treating the node as not yet upgraded and refusing +// DDL forever. +func TestNodeRegistry_RefutesStaleLogProtocolVersionOfSelf(t *testing.T) { + upgraded := NewNodeRegistry(3, "localhost:8083") + upgraded.Update(&NodeState{NodeId: 3, Address: "localhost:8083", Status: NodeStatus_ALIVE, Incarnation: 4, LogProtocolVersion: 0}) + + var self *NodeState + for _, n := range upgraded.GetAll() { + if n.NodeId == 3 { + self = n + } + } + require.NotNil(t, self) + require.Greater(t, self.Incarnation, uint64(4), "a stale pre-upgrade view of self must be refuted with a higher incarnation") + + peer := NewNodeRegistry(1, "localhost:8081") + peer.Update(&NodeState{NodeId: 3, Address: "localhost:8083", Status: NodeStatus_ALIVE, Incarnation: 4, LogProtocolVersion: 0}) + require.Equal(t, []uint64{3}, peer.LegacyLogProtocolMembers()) + peer.Update(self) + require.Empty(t, peer.LegacyLogProtocolMembers(), "the refutation must replace the peer's pre-upgrade view") +} + +// TestNodeRegistry_StrippedRelaysDoNotClimbIncarnation pins the bound on the +// version refutation: a peer on an older release relays this node's state +// with its log protocol version stripped, at this node's current +// incarnation, on every gossip round. Refuting each of those would climb the incarnation for as +// long as that peer runs; it must not move at all. +func TestNodeRegistry_StrippedRelaysDoNotClimbIncarnation(t *testing.T) { + nr := NewNodeRegistry(3, "localhost:8083") + self := func() *NodeState { + for _, n := range nr.GetAll() { + if n.NodeId == 3 { + return n + } + } + t.Fatal("self missing") + return nil + } + start := self().Incarnation + for i := 0; i < 100; i++ { + cur := self() + nr.Update(&NodeState{NodeId: 3, Address: cur.Address, Status: NodeStatus_ALIVE, Incarnation: cur.Incarnation}) + } + require.Equal(t, start, self().Incarnation, "stripped relays at the current incarnation must not bump it") + + // One stale view from a higher incarnation is refuted exactly once, and + // the stripped relays of the refuted incarnation that follow are not. + nr.Update(&NodeState{NodeId: 3, Address: "localhost:8083", Status: NodeStatus_ALIVE, Incarnation: start + 5}) + refuted := self().Incarnation + require.Equal(t, start+6, refuted) + for i := 0; i < 100; i++ { + nr.Update(&NodeState{NodeId: 3, Address: "localhost:8083", Status: NodeStatus_ALIVE, Incarnation: refuted}) + } + require.Equal(t, refuted, self().Incarnation) +} diff --git a/grpc/log_pull_harness_test.go b/grpc/log_pull_harness_test.go new file mode 100644 index 00000000..de473062 --- /dev/null +++ b/grpc/log_pull_harness_test.go @@ -0,0 +1,157 @@ +package grpc + +import ( + "context" + "fmt" + "net" + "testing" + + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/encoding" + "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// pullTestNode is one in-process node for LogPuller regression tests: a +// real db.DatabaseManager, optionally served over a real gRPC +// MarmotServiceServer, so LogPuller exercises the exact +// ListCommittedLog/FetchTransactions wire path a production pull round +// uses. +type pullTestNode struct { + id uint64 + dir string + dm *db.DatabaseManager + addr string +} + +// newPullTestNode creates a node with the given databases, each holding a +// table t(id INTEGER PRIMARY KEY, v TEXT) ready for seeding through +// ReplicatedDatabase.ApplyReplayedTxn (the same path production replay +// uses), so a test can build a peer's local commit log directly, without a +// second node or the 2PC wire path. +func newPullTestNode(t *testing.T, id uint64, databases ...string) *pullTestNode { + t.Helper() + dir := t.TempDir() + dm, err := db.NewDatabaseManager(dir, id, hlc.NewClock(id)) + require.NoError(t, err) + n := &pullTestNode{id: id, dir: dir, dm: dm} + t.Cleanup(func() { _ = n.dm.Close() }) + + for _, name := range databases { + require.NoError(t, dm.CreateDatabase(name)) + mdb, err := dm.GetDatabase(name) + require.NoError(t, err) + _, err = mdb.GetWriteDB().Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + require.NoError(t, mdb.ReloadSchema()) + } + return n +} + +// restart closes n's DatabaseManager and reopens it over the same data +// directory, as a process restart would. n must not be serving. +func (n *pullTestNode) restart(t *testing.T) { + t.Helper() + require.NoError(t, n.dm.Close()) + dm, err := db.NewDatabaseManager(n.dir, n.id, hlc.NewClock(n.id)) + require.NoError(t, err) + n.dm = dm +} + +// serve starts n over real gRPC, optionally through wrap (for example, a +// server that deliberately omits ListCommittedLog to simulate a +// rolling-upgrade peer), and records the listen address. +func (n *pullTestNode) serve(t *testing.T, wrap func(*Server) MarmotServiceServer) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + server := &Server{nodeID: n.id, registry: NewNodeRegistry(n.id, listener.Addr().String())} + server.SetDatabaseManager(n.dm) + + var service MarmotServiceServer = server + if wrap != nil { + service = wrap(server) + } + gs := grpc.NewServer() + RegisterMarmotServiceServer(gs, service) + go func() { _ = gs.Serve(listener) }() + t.Cleanup(gs.Stop) + n.addr = listener.Addr().String() +} + +// encodeSeedValues msgpack-encodes each value, as EncodedCapturedRow's +// OldValues/NewValues require (CDC never uses JSON - CLAUDE.md). +func encodeSeedValues(t *testing.T, values map[string]interface{}) map[string][]byte { + t.Helper() + out := make(map[string][]byte, len(values)) + for k, v := range values { + b, err := encoding.Marshal(v) + require.NoError(t, err) + out[k] = b + } + return out +} + +// seed commits txnID (coordinated by origin, committed at wall) into +// database d through ReplicatedDatabase.ApplyReplayedTxn, inserting one row +// (rowID, v) into table t. +func (n *pullTestNode) seed(t *testing.T, d string, txnID, origin uint64, wall int64, rowID int64, v string) { + t.Helper() + mdb, err := n.dm.GetDatabase(d) + require.NoError(t, err) + + row := &db.EncodedCapturedRow{ + Table: "t", + Op: uint8(db.OpTypeInsert), + IntentKey: []byte(fmt.Sprintf("t:%d", rowID)), + NewValues: encodeSeedValues(t, map[string]interface{}{"id": rowID, "v": v}), + } + applied, err := mdb.ApplyReplayedTxn(context.Background(), &db.ReplayTxn{ + TxnID: txnID, + OriginNodeID: origin, + CommitTS: hlc.Timestamp{WallTime: wall, NodeID: origin}, + Rows: []*db.EncodedCapturedRow{row}, + }, true) + require.NoError(t, err) + require.True(t, applied) +} + +// seedDDL commits txnID (coordinated by origin) into database d as a DDL +// transaction, through the same ApplyReplayedTxn path, bumping the schema +// version exactly as a 2PC DDL commit or a real replay would. +func (n *pullTestNode) seedDDL(t *testing.T, d string, txnID, origin uint64, wall int64, sql string) { + t.Helper() + mdb, err := n.dm.GetDatabase(d) + require.NoError(t, err) + + row := &db.EncodedCapturedRow{Table: "t", Op: uint8(db.OpTypeDDL), DDLSQL: sql} + applied, err := mdb.ApplyReplayedTxn(context.Background(), &db.ReplayTxn{ + TxnID: txnID, + OriginNodeID: origin, + CommitTS: hlc.Timestamp{WallTime: wall, NodeID: origin}, + Rows: []*db.EncodedCapturedRow{row}, + }, true) + require.NoError(t, err) + require.True(t, applied) +} + +// rows reads table t of database d. +func (n *pullTestNode) rows(t *testing.T, d string) map[int64]string { + t.Helper() + mdb, err := n.dm.GetDatabase(d) + require.NoError(t, err) + rs, err := mdb.GetReadDB().Query("SELECT id, v FROM t") + require.NoError(t, err) + defer rs.Close() + + got := map[int64]string{} + for rs.Next() { + var id int64 + var v string + require.NoError(t, rs.Scan(&id, &v)) + got[id] = v + } + require.NoError(t, rs.Err()) + return got +} diff --git a/grpc/log_puller.go b/grpc/log_puller.go new file mode 100644 index 00000000..e9280912 --- /dev/null +++ b/grpc/log_puller.go @@ -0,0 +1,712 @@ +package grpc + +import ( + "context" + "errors" + "fmt" + "io" + "sync" + + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/telemetry" + "github.com/rs/zerolog/log" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Default tuning for LogPuller, applied by NewLogPuller when the +// corresponding LogPullerConfig field is left zero. +const ( + // DefaultLogPullPageSize bounds how many log entries one ListCommittedLog + // page returns. + DefaultLogPullPageSize = 256 + + // DefaultMaxReplayAttempts is the number of consecutive replay failures + // (per peer, database and txn) after which a transaction is reported + // STUCK. + DefaultMaxReplayAttempts = 5 + + // DefaultStuckRetryEveryRounds bounds how often a STUCK transaction is + // retried, in PullPair calls for its (peer, database) pair, once it has + // stopped being retried every round. + DefaultStuckRetryEveryRounds = 20 + + // maxPagesPerPull bounds how many ListCommittedLog pages one PullPair + // call walks, so a pair whose cursor is held back by an entry it cannot + // apply yet still delivers the entries after it without rescanning an + // unbounded tail every round. + maxPagesPerPull = 64 +) + +// PeerRef identifies a peer node a LogPuller pulls a database's local commit +// log from. +type PeerRef struct { + NodeID uint64 + Address string +} + +// LogPullerConfig configures a LogPuller. PageSize, MaxReplayAttempts and +// StuckRetryEveryRounds default to the Default* constants above when zero. +type LogPullerConfig struct { + NodeID uint64 + Client *Client + DBManager *db.DatabaseManager + PageSize int + MaxReplayAttempts int + StuckRetryEveryRounds int +} + +// PairResult reports what one LogPuller.PullPair call did for one (peer, +// database) pair. +type PairResult struct { + // Applied counts transactions this call actually applied (not merely + // found already covered by an existing marker). + Applied int + + // CaughtUp is true when this call's final cursor reached the peer's + // stable point: the last page had no more entries beyond it and the + // walk covered every entry that page listed. Membership-aware + // "caught up for the database" is a property of every current member, + // computed by the caller from a PairResult per peer. + CaughtUp bool + + // NeedsSnapshot is true when the peer's log no longer covers this + // pair's cursor (its GC has truncated past it): nothing was applied, + // and the pair must be restored from a snapshot instead. + NeedsSnapshot bool + + // DatabaseAbsent is true when the peer does not have this database. + DatabaseAbsent bool + + // Unimplemented is true when the peer does not implement + // ListCommittedLog (a rolling-upgrade peer running an older version). + Unimplemented bool + + // PeerSchemaVersion is the peer's reported schema version for this + // database, from the last ListCommittedLog response this call read. + PeerSchemaVersion uint64 + + // Stuck counts STUCK transactions this call encountered for this pair + // (see LogPuller.StuckTxns). + Stuck int +} + +// StuckTxn describes one transaction a LogPuller has stopped retrying every +// round after DefaultMaxReplayAttempts (or LogPullerConfig.MaxReplayAttempts) +// consecutive replay failures. +type StuckTxn struct { + Database string + PeerNodeID uint64 + TxnID uint64 + Attempts int + LastError string +} + +// pairKey identifies one (peer, database) pull pair, for the per-pair call +// counter that drives the STUCK retry cadence. +type pairKey struct { + peer uint64 + database string +} + +// txnAttemptKey identifies one (peer, database, txn) replay-attempt counter. +type txnAttemptKey struct { + peer uint64 + database string + txnID uint64 +} + +// txnAttemptState is the poison-txn bookkeeping for one txnAttemptKey. +type txnAttemptState struct { + attempts int + lastErr string + stuck bool +} + +// LogPuller pulls one node's per-peer local commit logs, exactly (never +// silently skipping an entry) and exactly-once (via the applied-txn marker, +// db.ApplyReplayedTxn). Every current member's log is pulled independently, +// from a durably persisted cursor, with no shared deadline between pairs, so +// a transaction any member committed reaches this node even when no single +// peer holds every transaction this node lacks. +type LogPuller struct { + nodeID uint64 + client *Client + dbMgr *db.DatabaseManager + pageSize int + maxReplayAttempts int + stuckRetryEveryRounds int + + mu sync.Mutex + attempts map[txnAttemptKey]*txnAttemptState + pairCalls map[pairKey]int + // restoredGen counts, per database, the snapshot restores it has had; + // pairRestoredGen is the count each pair last pulled under. A pair whose + // database was restored since its last pull moves its cursor up to the + // peer's truncation point instead of reporting NeedsSnapshot. + restoredGen map[string]uint64 + pairRestoredGen map[pairKey]uint64 + + warnedUnimplemented sync.Map // peer NodeID -> struct{} +} + +// NewLogPuller builds a LogPuller from cfg, applying the Default* constants +// for any zero-valued tuning field. +func NewLogPuller(cfg LogPullerConfig) *LogPuller { + pageSize := cfg.PageSize + if pageSize <= 0 { + pageSize = DefaultLogPullPageSize + } + maxAttempts := cfg.MaxReplayAttempts + if maxAttempts <= 0 { + maxAttempts = DefaultMaxReplayAttempts + } + stuckEvery := cfg.StuckRetryEveryRounds + if stuckEvery <= 0 { + stuckEvery = DefaultStuckRetryEveryRounds + } + return &LogPuller{ + nodeID: cfg.NodeID, + client: cfg.Client, + dbMgr: cfg.DBManager, + pageSize: pageSize, + maxReplayAttempts: maxAttempts, + stuckRetryEveryRounds: stuckEvery, + attempts: make(map[txnAttemptKey]*txnAttemptState), + pairCalls: make(map[pairKey]int), + restoredGen: make(map[string]uint64), + pairRestoredGen: make(map[pairKey]uint64), + } +} + +// PullPair pulls peer's local commit log for database, from this pair's +// persisted cursor (db.MetaStore.GetPullCursor/SetPullCursor), applying +// every transaction it is missing through ApplyPulledEvent and advancing +// the cursor over the contiguous prefix of covered entries. PairResult's +// fields document each outcome. +// +// PullPair does not share a context or deadline with any other pair: the +// caller bounds ctx per call. An interrupted call (ctx cancelled, or any +// other error) leaves the cursor exactly where it last persisted it - +// PullPair always persists progress after every page it fully or partially +// covers before returning - so the next call resumes from there. +func (lp *LogPuller) PullPair(ctx context.Context, peer PeerRef, database string) (PairResult, error) { + result, err := lp.pullPair(ctx, peer, database) + telemetry.LogPullPairResultsTotal.With(pairOutcome(result, err)).Inc() + if result.Applied > 0 { + telemetry.LogPullTxnsAppliedTotal.Add(float64(result.Applied)) + } + return result, err +} + +// pairOutcome classifies a PullPair result into the fixed outcome label set +// LogPullPairResultsTotal is recorded under. +func pairOutcome(result PairResult, err error) string { + switch { + case err != nil: + return "error" + case result.Unimplemented: + return "unimplemented" + case result.NeedsSnapshot: + return "needs_snapshot" + case result.CaughtUp: + return "caught_up" + default: + return "behind" + } +} + +// pullState is one PullPair call's working state for its (peer, database) +// pair. +type pullState struct { + peer PeerRef + database string + key pairKey + round int + client MarmotServiceClient + mdb *db.ReplicatedDatabase + metaStore db.MetaStore + + // cursor is the pair's persisted position: every entry at or before it + // is covered. listAfter runs ahead of it once an entry cannot be applied + // yet, so the entries after that one are still applied this call + // while the cursor, and the consumed + // position reported to the peer's GC, stay before it. + cursor db.LogPosition + listAfter db.LogPosition + contiguous bool + + // localCommitFailed stops further local commits of prepared entries for + // the rest of this call once one fails: they share its cause (typically + // the SQLite writer held by a pinned session) and each would wait out + // the same busy timeout. + localCommitFailed bool + + result PairResult +} + +func (lp *LogPuller) pullPair(ctx context.Context, peer PeerRef, database string) (PairResult, error) { + st, err := lp.newPullState(peer, database) + if err != nil { + return PairResult{}, err + } + for page := 0; page < maxPagesPerPull; page++ { + if ctxErr := ctx.Err(); ctxErr != nil { + return st.result, ctxErr + } + done, err := lp.pullPage(ctx, st) + if err != nil || done { + return st.result, err + } + } + return st.result, nil +} + +// newPullState resolves the local database, the peer's client and the +// pair's persisted cursor, and counts this call for the STUCK retry cadence. +func (lp *LogPuller) newPullState(peer PeerRef, database string) (*pullState, error) { + mdb, err := lp.dbMgr.GetDatabase(database) + if err != nil { + return nil, fmt.Errorf("get local database %q: %w", database, err) + } + key := pairKey{peer: peer.NodeID, database: database} + lp.mu.Lock() + lp.pairCalls[key]++ + round := lp.pairCalls[key] + lp.mu.Unlock() + + client, err := lp.client.GetClientByAddress(peer.Address) + if err != nil { + return nil, fmt.Errorf("connect to peer %d at %s: %w", peer.NodeID, peer.Address, err) + } + metaStore := mdb.GetMetaStore() + cursor, err := metaStore.GetPullCursor(peer.NodeID) + if err != nil { + return nil, fmt.Errorf("get pull cursor for peer %d database %q: %w", peer.NodeID, database, err) + } + return &pullState{ + peer: peer, database: database, key: key, round: round, + client: client, mdb: mdb, metaStore: metaStore, + cursor: cursor, listAfter: cursor, contiguous: true, + }, nil +} + +// pullPage lists and applies one page of the peer's log. done reports that +// the call is over (caught up, or an outcome recorded in st.result). +func (lp *LogPuller) pullPage(ctx context.Context, st *pullState) (done bool, err error) { + resp, err := st.client.ListCommittedLog(ctx, &LogListRequest{ + Database: st.database, + RequestingNodeId: lp.nodeID, + AfterSeq: st.listAfter.Seq, + AfterTxnId: st.listAfter.TxnID, + Limit: uint32(lp.pageSize), + ConsumedSeq: st.cursor.Seq, + ConsumedTxnId: st.cursor.TxnID, + }) + if err != nil { + if status.Code(err) == codes.Unimplemented { + lp.warnUnimplementedOnce(st.peer.NodeID) + st.result.Unimplemented = true + return true, nil + } + return true, fmt.Errorf("list committed log from peer %d database %q: %w", st.peer.NodeID, st.database, err) + } + if resp.DatabaseAbsent { + st.result.DatabaseAbsent = true + return true, nil + } + st.result.PeerSchemaVersion = resp.SchemaVersion + + truncated := db.LogPosition{Seq: resp.TruncatedSeq, TxnID: resp.TruncatedTxnId} + if lp.takeRestored(st.key) { + // Everything at or below the peer's truncation point was consumed + // by every member, the restore's source included, or is in this + // node's own log for the restore's re-apply. The cursor moves there even when that lowers + // it: a cursor that passed an entry before this restore (covered by + // an earlier restore's marker, say) must see it again, because this + // restore's source may lack it. + st.cursor, st.listAfter = truncated, truncated + return false, lp.persistCursor(st) + } + if st.cursor.Less(truncated) { + st.result.NeedsSnapshot = true + return true, nil + } + if len(resp.Entries) == 0 { + st.result.CaughtUp = st.contiguous && !resp.More + return true, nil + } + return lp.applyListedPage(ctx, st, resp) +} + +// applyListedPage applies one non-empty page, advances the cursor over the +// covered prefix, and moves listAfter past the page. +func (lp *LogPuller) applyListedPage(ctx context.Context, st *pullState, resp *LogListResponse) (done bool, err error) { + coveredPrefixLen, err := lp.applyPage(ctx, st, resp.Entries) + if st.contiguous && coveredPrefixLen > 0 { + last := resp.Entries[coveredPrefixLen-1] + st.cursor = db.LogPosition{Seq: last.Seq, TxnID: last.TxnId} + if setErr := lp.persistCursor(st); setErr != nil { + return true, setErr + } + } + if coveredPrefixLen < len(resp.Entries) { + st.contiguous = false + } + if err != nil { + return true, err + } + if !resp.More { + st.result.CaughtUp = st.contiguous + return true, nil + } + last := resp.Entries[len(resp.Entries)-1] + st.listAfter = db.LogPosition{Seq: last.Seq, TxnID: last.TxnId} + return false, nil +} + +// persistCursor durably stores st.cursor as the pair's pull cursor. +func (lp *LogPuller) persistCursor(st *pullState) error { + if err := st.metaStore.SetPullCursor(st.peer.NodeID, st.cursor); err != nil { + return fmt.Errorf("persist pull cursor for peer %d database %q: %w", st.peer.NodeID, st.database, err) + } + return nil +} + +// applyPage classifies and applies one ListCommittedLog page's entries, in +// order, and returns the length of the page's contiguous prefix now +// covered. It returns an error only for a failure that is not attributable +// to a specific transaction (checking local state). +// +// Every listed entry is COMMITTED in the peer's log, so a transaction this +// node holds PENDING under the same id was decided COMMITTED, and its +// local record is resolved here, in the same call (resolveLocalPending), +// instead of stopping the walk until the stale-transaction GC ends it. +func (lp *LogPuller) applyPage(ctx context.Context, st *pullState, entries []*LogEntry) (int, error) { + ids := make([]uint64, len(entries)) + for i, e := range entries { + ids[i] = e.TxnId + } + appliedMarkers, err := st.mdb.AppliedTxns(ids) + if err != nil { + return 0, fmt.Errorf("check applied markers: %w", err) + } + + covered := make([]bool, len(entries)) + toFetch := make([]uint64, 0, len(entries)) + fetchIdx := make(map[uint64]int, len(entries)) + for i, e := range entries { + if appliedMarkers[e.TxnId] { + covered[i] = true + lp.clearAttempt(st.database, st.peer.NodeID, e.TxnId) + continue + } + if lp.isStuckThisRound(st.database, st.peer.NodeID, e.TxnId, st.round) { + st.result.Stuck++ + continue + } + fetch, err := lp.resolveLocalPending(st, e.TxnId, &covered[i]) + if err != nil { + return 0, err + } + if fetch { + fetchIdx[e.TxnId] = i + toFetch = append(toFetch, e.TxnId) + } + } + + if len(toFetch) > 0 { + lp.fetchAndApply(ctx, st, toFetch, fetchIdx, covered) + } + + prefix := 0 + for prefix < len(covered) && covered[prefix] { + prefix++ + } + return prefix, nil +} + +// resolveLocalPending resolves this node's own record for txnID, which the +// peer's log proves COMMITTED, and reports whether the entry must be fetched +// and replayed from the peer: +// - no PENDING record: fetch; +// - durably prepared: commit it through the local commit path +// (db.DatabaseManager.CommitLocallyPrepared) and mark it covered; on +// failure leave it for the next call, never replaying over it; +// - begun but abandoned (its PREPARE died with an earlier process): it +// promised nothing, so discard it and fetch; +// - begun and live (a PREPARE executing now): leave it for the next call. +func (lp *LogPuller) resolveLocalPending(st *pullState, txnID uint64, covered *bool) (fetch bool, err error) { + kind, err := st.metaStore.ClassifyPending(txnID) + if err != nil { + return false, fmt.Errorf("check local status for txn %d: %w", txnID, err) + } + switch kind { + case db.PendingNone: + return true, nil + case db.PendingPrepared: + if st.localCommitFailed { + return false, nil + } + if err := lp.dbMgr.CommitLocallyPrepared(st.database, txnID); err != nil { + st.localCommitFailed = true + // A busy writer (a pinned session holding it, say) is not this + // transaction's fault and must not push it toward STUCK, whose + // throttled retries would delay it long after the writer frees. + if !db.IsTransientSQLiteError(err) && lp.recordFailedAttempt(st.database, st.peer.NodeID, txnID, err) { + st.result.Stuck++ + } + return false, nil + } + *covered = true + st.result.Applied++ + lp.clearAttempt(st.database, st.peer.NodeID, txnID) + return false, nil + case db.PendingBegunAbandoned: + if err := st.mdb.DiscardAbandonedBegin(txnID); err != nil && !errors.Is(err, db.ErrNotAbandonedBegin) { + return false, fmt.Errorf("discard abandoned local begin of txn %d: %w", txnID, err) + } + return true, nil + default: // db.PendingBegunLive + return false, nil + } +} + +// fetchAndApply fetches every id in toFetch from the peer and applies each +// event through ApplyPulledEvent as it streams in, marking +// covered[fetchIdx[txnID]] true for every one that ends up covered (applied +// or already applied). +// +// The peer serves ids in the order requested and fails the whole call at +// the first id it cannot serve (FetchTransactions), so one bad transaction +// must not cost the healthy ids after it their turn: the id the call +// failed at counts one failed attempt, and the ids after it are fetched +// again in a fresh call. A transport failure (the peer or ctx gone) is not +// any transaction's fault: it ends this call without counting an attempt. +func (lp *LogPuller) fetchAndApply(ctx context.Context, st *pullState, toFetch []uint64, fetchIdx map[uint64]int, covered []bool) { + remaining := toFetch + for len(remaining) > 0 { + received, err := lp.fetchBatch(ctx, st, remaining, fetchIdx, covered) + if err == nil || ctx.Err() != nil || !isPerTxnFetchError(err) { + return + } + rest := remaining[:0:0] + blamed := false + for _, id := range remaining { + if received[id] { + continue + } + if !blamed { + blamed = true + if lp.recordFailedAttempt(st.database, st.peer.NodeID, id, err) { + st.result.Stuck++ + } + continue + } + rest = append(rest, id) + } + remaining = rest + } +} + +// isPerTxnFetchError reports whether a FetchTransactions failure names the +// transaction it stopped at (a missing or unservable entry), rather than +// the call itself failing. +func isPerTxnFetchError(err error) bool { + switch status.Code(err) { + case codes.NotFound, codes.FailedPrecondition, codes.Internal: + return true + default: + return false + } +} + +// fetchBatch runs one FetchTransactions call for ids and applies every event +// it streams. It returns the set of ids an event arrived for, and the call's +// error (nil at a clean end of stream). +func (lp *LogPuller) fetchBatch(ctx context.Context, st *pullState, ids []uint64, fetchIdx map[uint64]int, covered []bool) (map[uint64]bool, error) { + received := make(map[uint64]bool, len(ids)) + stream, err := st.client.FetchTransactions(ctx, &FetchTransactionsRequest{ + Database: st.database, + RequestingNodeId: lp.nodeID, + TxnIds: ids, + }) + if err != nil { + return received, err + } + for { + ev, recvErr := stream.Recv() + if recvErr != nil { + if errors.Is(recvErr, io.EOF) { + return received, nil + } + return received, recvErr + } + received[ev.TxnId] = true + idx, ok := fetchIdx[ev.TxnId] + if !ok { + log.Warn().Uint64("txn_id", ev.TxnId).Str("database", st.database).Uint64("peer_node_id", st.peer.NodeID). + Msg("LogPuller: peer streamed a transaction that was not requested; ignoring") + continue + } + lp.applyFetched(ctx, st, ev, &covered[idx]) + } +} + +// errFetchedWrongDatabase is a fetched event naming a database other than +// the one its pair pulls: applying it would cover the pair's entry with +// another database's transaction. +var errFetchedWrongDatabase = errors.New("peer streamed a transaction of another database") + +// applyFetched applies one fetched event, marking it covered on success. An +// intent conflict, or this node's own record of the txn turning PENDING (a +// late PREPARE's begin landing before the replay), is neither covered nor a +// failed attempt: it is retried on a later call while the walk continues past +// it. An event for another database is a failed attempt, never applied. +func (lp *LogPuller) applyFetched(ctx context.Context, st *pullState, ev *ChangeEvent, covered *bool) { + if ev.Database != st.database { + err := fmt.Errorf("%w: txn %d names %q, pulling %q", errFetchedWrongDatabase, ev.TxnId, ev.Database, st.database) + if lp.recordFailedAttempt(st.database, st.peer.NodeID, ev.TxnId, err) { + st.result.Stuck++ + } + return + } + applyOK, applyErr := ApplyPulledEvent(ctx, lp.dbMgr, ev) + switch { + case applyErr == nil: + *covered = true + lp.clearAttempt(st.database, st.peer.NodeID, ev.TxnId) + if applyOK { + st.result.Applied++ + } + case errors.Is(applyErr, db.ErrReplayIntentConflict), errors.Is(applyErr, db.ErrReplayPending): + default: + if lp.recordFailedAttempt(st.database, st.peer.NodeID, ev.TxnId, applyErr) { + st.result.Stuck++ + } + } +} + +// isStuckThisRound reports whether (peer, database, txnID) is currently +// STUCK and this round is not one of its throttled retry rounds, so +// applyPage should skip fetching it this call. +func (lp *LogPuller) isStuckThisRound(database string, peerNodeID, txnID uint64, round int) bool { + lp.mu.Lock() + st := lp.attempts[txnAttemptKey{peer: peerNodeID, database: database, txnID: txnID}] + lp.mu.Unlock() + if st == nil || !st.stuck { + return false + } + return round%lp.stuckRetryEveryRounds != 0 +} + +// recordFailedAttempt records one failed replay attempt for (peer, +// database, txnID). It returns true if the transaction is STUCK after this +// attempt (whether it just became so, or already was). The first +// transition to STUCK logs one ERROR and raises LogPullStuckTxns. +func (lp *LogPuller) recordFailedAttempt(database string, peerNodeID, txnID uint64, cause error) bool { + key := txnAttemptKey{peer: peerNodeID, database: database, txnID: txnID} + + lp.mu.Lock() + st, ok := lp.attempts[key] + if !ok { + st = &txnAttemptState{} + lp.attempts[key] = st + } + st.attempts++ + st.lastErr = cause.Error() + newlyStuck := !st.stuck && st.attempts >= lp.maxReplayAttempts + if newlyStuck { + st.stuck = true + } + stuck := st.stuck + attempts := st.attempts + lp.mu.Unlock() + + if newlyStuck { + telemetry.LogPullStuckTxns.With(database).Inc() + log.Error().Err(cause).Uint64("txn_id", txnID).Uint64("peer_node_id", peerNodeID). + Str("database", database).Int("attempts", attempts). + Msg("LogPuller: transaction is stuck after repeated replay failures") + } + return stuck +} + +// clearAttempt drops (peer, database, txnID)'s failed-attempt counter once +// it applies successfully, clearing its STUCK status and metric if it had +// one. +func (lp *LogPuller) clearAttempt(database string, peerNodeID, txnID uint64) { + key := txnAttemptKey{peer: peerNodeID, database: database, txnID: txnID} + + lp.mu.Lock() + st, ok := lp.attempts[key] + if ok { + delete(lp.attempts, key) + } + lp.mu.Unlock() + + if ok && st.stuck { + telemetry.LogPullStuckTxns.With(database).Dec() + } +} + +// StuckTxns returns every transaction this LogPuller currently reports +// STUCK, across every pair it has pulled. +func (lp *LogPuller) StuckTxns() []StuckTxn { + lp.mu.Lock() + defer lp.mu.Unlock() + + out := make([]StuckTxn, 0) + for key, st := range lp.attempts { + if !st.stuck { + continue + } + out = append(out, StuckTxn{ + Database: key.database, + PeerNodeID: key.peer, + TxnID: key.txnID, + Attempts: st.attempts, + LastError: st.lastErr, + }) + } + return out +} + +// warnUnimplementedOnce logs, once per peer for this LogPuller's lifetime, +// that peerNodeID does not implement ListCommittedLog. +func (lp *LogPuller) warnUnimplementedOnce(peerNodeID uint64) { + if _, loaded := lp.warnedUnimplemented.LoadOrStore(peerNodeID, struct{}{}); !loaded { + log.Warn().Uint64("peer_node_id", peerNodeID). + Msg("LogPuller: peer does not implement ListCommittedLog (rolling upgrade); skipping") + } +} + +// MarkRestored records that database was just restored from a snapshot. +// The next pull from each peer - every +// peer, including one this node has never pulled from - sets that pair's +// cursor to the peer's truncation point, lowering it if it was past it, +// instead of reporting NeedsSnapshot again: everything at or below that +// point was consumed by every member, the restore's source included, or is +// in this node's own log for the restore's re-apply, while an entry above it +// may be missing from the restored file even if the old cursor had passed +// it, and the marker diff skips whatever the restored file already holds. +// The mark is in memory: a restart before a pair's next pull only costs one +// more restore. +func (lp *LogPuller) MarkRestored(database string) { + lp.mu.Lock() + defer lp.mu.Unlock() + lp.restoredGen[database]++ +} + +// takeRestored reports whether the pair's database was restored since the +// pair last took the mark, and takes it. +func (lp *LogPuller) takeRestored(key pairKey) bool { + lp.mu.Lock() + defer lp.mu.Unlock() + gen := lp.restoredGen[key.database] + if lp.pairRestoredGen[key] >= gen { + return false + } + lp.pairRestoredGen[key] = gen + return true +} diff --git a/grpc/log_puller_gc_race_test.go b/grpc/log_puller_gc_race_test.go new file mode 100644 index 00000000..82571b39 --- /dev/null +++ b/grpc/log_puller_gc_race_test.go @@ -0,0 +1,169 @@ +package grpc + +import ( + "context" + "sync" + "testing" + + "github.com/maxpert/marmot/db" + pb "github.com/maxpert/marmot/grpc/common" + "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" +) + +// TestLogPuller_PreparedPayloadGoneNeverPassesTheCursor: the stale-transaction GC +// deletes a prepared transaction's intents and captured rows, and the log +// puller then reaches that transaction in a peer's log. Its local commit must +// be refused - committing it would record it COMMITTED with no rows, and the +// cursor would pass it for good - so the cursor stays before it; once the +// transaction is aborted, the next pull replays it from the peer. +// +// Mutation: make TransactionManager.verifyPreparedPayload return nil. The +// first pull passes the cursor over txn 2000 and row 2 is never applied. +func TestLogPuller_PreparedPayloadGoneNeverPassesTheCursor(t *testing.T) { + peer := newPullTestNode(t, 2, "app") + peer.serve(t, nil) + local := newPullTestNode(t, 1, "app") + + peer.seed(t, "app", 1000, 2, 1000, 1, "a") + peer.seed(t, "app", 2000, 2, 2000, 2, "b") + peer.seed(t, "app", 3000, 2, 3000, 3, "c") + + mdb, err := local.dm.GetDatabase("app") + require.NoError(t, err) + rh := NewReplicationHandler(1, local.dm, hlc.NewClock(1), db.NewSchemaVersionManager(local.dm)) + resp, err := rh.HandleReplicateTransaction(context.Background(), &TransactionRequest{ + TxnId: 2000, SourceNodeId: 2, Database: "app", Phase: TransactionPhase_PREPARE, + Timestamp: &HLC{WallTime: 2000, NodeId: 2}, RequiredSchemaVersion: mdb.SchemaVersion(), + Statements: []*Statement{{Type: pb.StatementType_INSERT, TableName: "t", Database: "app", + Payload: &Statement_RowChange{RowChange: testInsertRowChange("t", []byte("t:2"), map[string][]byte{ + "id": mustMarshalMsgpack(t, int64(2)), "v": mustMarshalMsgpack(t, "b"), + })}}}, + }) + require.NoError(t, err) + require.True(t, resp.Success, resp.ErrorMessage) + + ms := mdb.GetMetaStore() + require.NoError(t, ms.DeleteIntentsByTxn(2000)) + require.NoError(t, ms.DeleteIntentEntries(2000)) + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm}) + peerRef := PeerRef{NodeID: 2, Address: peer.addr} + res, err := lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.False(t, res.CaughtUp) + + rec, err := ms.GetTransaction(2000) + require.NoError(t, err) + require.NotNil(t, rec) + require.Equal(t, db.TxnStatusPending, rec.Status, "a txn whose prepared payload is gone must not be committed") + cursor, err := ms.GetPullCursor(2) + require.NoError(t, err) + require.Equal(t, uint64(1000), cursor.TxnID, "the cursor must stay before the refused txn") + + require.NoError(t, ms.AbortTransaction(2000)) + res, err = lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.True(t, res.CaughtUp) + require.Equal(t, map[int64]string{1: "a", 2: "b", 3: "c"}, local.rows(t, "app")) +} + +// fetchHookServer serves FetchTransactions like the real server, running +// beforeFetch on the request before anything is streamed and beforeSend on +// each event before it is sent. +type fetchHookServer struct { + *Server + beforeFetch func(req *FetchTransactionsRequest) + beforeSend func(ev *ChangeEvent) +} + +type hookedFetchStream struct { + MarmotService_FetchTransactionsServer + beforeSend func(ev *ChangeEvent) +} + +func (s hookedFetchStream) Send(ev *ChangeEvent) error { + if s.beforeSend != nil { + s.beforeSend(ev) + } + return s.MarmotService_FetchTransactionsServer.Send(ev) +} + +func (f fetchHookServer) FetchTransactions(req *FetchTransactionsRequest, stream MarmotService_FetchTransactionsServer) error { + if f.beforeFetch != nil { + f.beforeFetch(req) + } + return f.Server.FetchTransactions(req, hookedFetchStream{MarmotService_FetchTransactionsServer: stream, beforeSend: f.beforeSend}) +} + +// TestLogPuller_EventForAnotherDatabaseIsAFailedAttempt pins that a fetched +// event naming a database other than the pair's is never applied (it would +// cover the pair's entry with another database's transaction) and counts as +// a failed attempt for that entry. +// +// Mutation: drop the ev.Database check in applyFetched. The event applies, +// the entry is covered and nothing is reported STUCK. +func TestLogPuller_EventForAnotherDatabaseIsAFailedAttempt(t *testing.T) { + peer := newPullTestNode(t, 2, "app", "other") + peer.serve(t, func(s *Server) MarmotServiceServer { + return fetchHookServer{Server: s, beforeSend: func(ev *ChangeEvent) { + if ev.TxnId == 2000 { + ev.Database = "other" + } + }} + }) + local := newPullTestNode(t, 1, "app", "other") + + peer.seed(t, "app", 1000, 2, 1000, 1, "a") + peer.seed(t, "app", 2000, 2, 2000, 2, "b") + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm, MaxReplayAttempts: 1}) + res, err := lp.PullPair(context.Background(), PeerRef{NodeID: 2, Address: peer.addr}, "app") + require.NoError(t, err) + require.False(t, res.CaughtUp) + require.Equal(t, 1, res.Stuck) + require.Equal(t, map[int64]string{1: "a"}, local.rows(t, "app")) + require.Empty(t, local.rows(t, "other"), "the mislabelled event must not be applied anywhere") + stuck := lp.StuckTxns() + require.Len(t, stuck, 1) + require.Equal(t, uint64(2000), stuck[0].TxnID) +} + +// TestLogPuller_LateLocalBeginDuringFetchIsRetriedNotFailed pins that a +// local PREPARE's begin landing for a txn after the walk decided to fetch it +// (so its replay finds the txn PENDING, db.ErrReplayPending) is retried on a +// later call like an intent conflict, never counted as a failed attempt. +// +// Mutation: drop ErrReplayPending from applyFetched's retry case. The entry +// is reported STUCK after one attempt. +func TestLogPuller_LateLocalBeginDuringFetchIsRetriedNotFailed(t *testing.T) { + local := newPullTestNode(t, 1, "app") + mdb, err := local.dm.GetDatabase("app") + require.NoError(t, err) + peer := newPullTestNode(t, 2, "app") + var once sync.Once + begun := make(chan error, 1) + peer.serve(t, func(s *Server) MarmotServiceServer { + return fetchHookServer{Server: s, beforeFetch: func(*FetchTransactionsRequest) { + once.Do(func() { + begun <- mdb.GetMetaStore().BeginTransaction(2000, 2, hlc.Timestamp{WallTime: 1, NodeID: 2}) + }) + }} + }) + peer.seed(t, "app", 2000, 2, 2000, 2, "b") + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm, MaxReplayAttempts: 1}) + peerRef := PeerRef{NodeID: 2, Address: peer.addr} + res, err := lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.NoError(t, <-begun, "the late local begin must land during the fetch") + require.False(t, res.CaughtUp) + require.Zero(t, res.Stuck, "a late local begin must not count as a failed attempt") + require.Empty(t, lp.StuckTxns()) + + require.NoError(t, mdb.GetMetaStore().AbortTransaction(2000)) + res, err = lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.True(t, res.CaughtUp) + require.Equal(t, map[int64]string{2: "b"}, local.rows(t, "app")) +} diff --git a/grpc/log_puller_test.go b/grpc/log_puller_test.go new file mode 100644 index 00000000..1838850e --- /dev/null +++ b/grpc/log_puller_test.go @@ -0,0 +1,562 @@ +package grpc + +import ( + "context" + "net" + "testing" + "time" + + "github.com/maxpert/marmot/db" + pb "github.com/maxpert/marmot/grpc/common" + "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// TestLogPuller_MissedTxnBelowLocalMaxIsDelivered: resuming a pull from +// GetMaxTxnID would skip any missed txn whose id is below the local max. +// PullPair instead pulls a persisted position cursor, so a txn below the +// local max is delivered like any other. +func TestLogPuller_MissedTxnBelowLocalMaxIsDelivered(t *testing.T) { + peer := newPullTestNode(t, 2, "app") + peer.serve(t, nil) + local := newPullTestNode(t, 1, "app") + + peer.seed(t, "app", 1000, 2, 1000, 1, "coordinated-by-2") + peer.seed(t, "app", 2000, 3, 2000, 2, "coordinated-by-3") + peer.seed(t, "app", 3000, 2, 3000, 3, "coordinated-by-2") + local.seed(t, "app", 2000, 3, 2000, 2, "coordinated-by-3") + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm}) + res, err := lp.PullPair(context.Background(), PeerRef{NodeID: 2, Address: peer.addr}, "app") + require.NoError(t, err) + require.True(t, res.CaughtUp) + require.Equal(t, 2, res.Applied, "only the two locally-missing txns are newly applied") + + require.Equal(t, map[int64]string{1: "coordinated-by-2", 2: "coordinated-by-3", 3: "coordinated-by-2"}, local.rows(t, "app")) + + mdb, err := local.dm.GetDatabase("app") + require.NoError(t, err) + cursor, err := mdb.GetMetaStore().GetPullCursor(2) + require.NoError(t, err) + require.Equal(t, uint64(3000), cursor.TxnID, "the cursor must reach the peer's last entry") +} + +// TestLogPuller_TwoCoordinatorsInterleavedIdsDeliveredInOnePull pins that +// entries from multiple origins, missing in an order that does not track +// txn id, all arrive from one PullPair call. +func TestLogPuller_TwoCoordinatorsInterleavedIdsDeliveredInOnePull(t *testing.T) { + peer := newPullTestNode(t, 2, "app") + peer.serve(t, nil) + local := newPullTestNode(t, 1, "app") + + // Interleaved origins 2 and 3; local starts with none of them. + peer.seed(t, "app", 100, 2, 100, 1, "o2-a") + peer.seed(t, "app", 150, 3, 150, 2, "o3-a") + peer.seed(t, "app", 200, 2, 200, 3, "o2-b") + peer.seed(t, "app", 250, 3, 250, 4, "o3-b") + peer.seed(t, "app", 300, 2, 300, 5, "o2-c") + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm}) + res, err := lp.PullPair(context.Background(), PeerRef{NodeID: 2, Address: peer.addr}, "app") + require.NoError(t, err) + require.True(t, res.CaughtUp) + require.Equal(t, 5, res.Applied) + + require.Equal(t, map[int64]string{1: "o2-a", 2: "o3-a", 3: "o2-b", 4: "o3-b", 5: "o2-c"}, local.rows(t, "app")) +} + +// TestLogPuller_InterruptedThenResumedNoDoubleApply pins that an +// interrupted pull persists its cursor mid-log (SetPullCursor after every +// page) and a later PullPair call resumes and finishes, applying nothing +// twice. +func TestLogPuller_InterruptedThenResumedNoDoubleApply(t *testing.T) { + peer := newPullTestNode(t, 2, "app") + peer.serve(t, nil) + local := newPullTestNode(t, 1, "app") + + peer.seed(t, "app", 1000, 2, 1000, 1, "a") + peer.seed(t, "app", 2000, 2, 2000, 2, "b") + peer.seed(t, "app", 3000, 2, 3000, 3, "c") + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm, PageSize: 1}) + peerRef := PeerRef{NodeID: 2, Address: peer.addr} + mdb, err := local.dm.GetDatabase("app") + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _, _ = lp.PullPair(ctx, peerRef, "app") + }() + require.Eventually(t, func() bool { + pos, err := mdb.GetMetaStore().GetPullCursor(2) + return err == nil && pos.TxnID == 1000 + }, 5*time.Second, time.Millisecond, "the first page's cursor was never persisted") + cancel() + <-done + + res, err := lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.True(t, res.CaughtUp, "a resumed pull must finish the peer's log") + + require.Equal(t, map[int64]string{1: "a", 2: "b", 3: "c"}, local.rows(t, "app")) + applied, err := mdb.AppliedTxns([]uint64{1000, 2000, 3000}) + require.NoError(t, err) + require.True(t, applied[1000] && applied[2000] && applied[3000], "every txn must end up applied exactly once") +} + +// TestLogPuller_LiveLocalBeginSkippedLaterEntriesApply pins that a txn +// this node holds begun by a PREPARE still executing (live, not yet durably +// prepared) is left for that PREPARE to finish - neither committed nor +// replayed over - while every entry after it still applies in the same +// call; the cursor stays before it until it resolves. +func TestLogPuller_LiveLocalBeginSkippedLaterEntriesApply(t *testing.T) { + peer := newPullTestNode(t, 2, "app") + peer.serve(t, nil) + local := newPullTestNode(t, 1, "app") + + peer.seed(t, "app", 1000, 2, 1000, 1, "a") + peer.seed(t, "app", 2000, 2, 2000, 2, "b") + peer.seed(t, "app", 3000, 2, 3000, 3, "c") + + mdb, err := local.dm.GetDatabase("app") + require.NoError(t, err) + require.NoError(t, mdb.GetMetaStore().BeginTransaction(2000, 1, hlc.Timestamp{WallTime: 1, NodeID: 1})) + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm}) + peerRef := PeerRef{NodeID: 2, Address: peer.addr} + + res, err := lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.False(t, res.CaughtUp) + require.Equal(t, 2, res.Applied, "the entries before and after the live begin must apply") + require.Equal(t, map[int64]string{1: "a", 3: "c"}, local.rows(t, "app")) + + cursor, err := mdb.GetMetaStore().GetPullCursor(2) + require.NoError(t, err) + require.Equal(t, uint64(1000), cursor.TxnID, "the cursor must stop before the live begin") + + require.NoError(t, mdb.GetMetaStore().AbortTransaction(2000)) + + res, err = lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.True(t, res.CaughtUp) + require.Equal(t, map[int64]string{1: "a", 2: "b", 3: "c"}, local.rows(t, "app"), "once resolved, the entry must apply") +} + +// TestLogPuller_PreparedLocalTxnCommittedLocallyInSamePull pins that a txn +// this node holds durably prepared, which the peer's log proves COMMITTED, +// is committed through the local commit path in the same PullPair call - +// not left PENDING until the stale-transaction GC - and the entries after it +// apply in that call too. +func TestLogPuller_PreparedLocalTxnCommittedLocallyInSamePull(t *testing.T) { + peer := newPullTestNode(t, 2, "app") + peer.serve(t, nil) + local := newPullTestNode(t, 1, "app") + + peer.seed(t, "app", 1000, 2, 1000, 1, "a") + peer.seed(t, "app", 2000, 2, 2000, 2, "b") + peer.seed(t, "app", 3000, 2, 3000, 3, "c") + + mdb, err := local.dm.GetDatabase("app") + require.NoError(t, err) + rh := NewReplicationHandler(1, local.dm, hlc.NewClock(1), db.NewSchemaVersionManager(local.dm)) + resp, err := rh.HandleReplicateTransaction(context.Background(), &TransactionRequest{ + TxnId: 2000, SourceNodeId: 2, Database: "app", Phase: TransactionPhase_PREPARE, + Timestamp: &HLC{WallTime: 2000, NodeId: 2}, RequiredSchemaVersion: mdb.SchemaVersion(), + Statements: []*Statement{{Type: pb.StatementType_INSERT, TableName: "t", Database: "app", + Payload: &Statement_RowChange{RowChange: testInsertRowChange("t", []byte("t:2"), map[string][]byte{ + "id": mustMarshalMsgpack(t, int64(2)), "v": mustMarshalMsgpack(t, "b"), + })}}}, + }) + require.NoError(t, err) + require.True(t, resp.Success, resp.ErrorMessage) + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm}) + res, err := lp.PullPair(context.Background(), PeerRef{NodeID: 2, Address: peer.addr}, "app") + require.NoError(t, err) + require.True(t, res.CaughtUp, "one call must resolve the prepared txn and reach the peer's end") + require.Equal(t, 3, res.Applied) + require.Equal(t, map[int64]string{1: "a", 2: "b", 3: "c"}, local.rows(t, "app")) + + rec, err := mdb.GetMetaStore().GetTransaction(2000) + require.NoError(t, err) + require.NotNil(t, rec) + require.Equal(t, db.TxnStatusCommitted, rec.Status, "the prepared txn must be committed locally") + + cursor, err := mdb.GetMetaStore().GetPullCursor(2) + require.NoError(t, err) + require.Equal(t, uint64(3000), cursor.TxnID) + + // The coordinator's COMMIT arriving afterwards finds the txn already + // committed here and is acknowledged, not refused. + resp, err = rh.HandleReplicateTransaction(context.Background(), &TransactionRequest{ + TxnId: 2000, SourceNodeId: 2, Database: "app", Phase: TransactionPhase_COMMIT, + Timestamp: &HLC{WallTime: 2000, NodeId: 2}, + }) + require.NoError(t, err) + require.True(t, resp.Success, resp.ErrorMessage) + require.Equal(t, 1, countLocalLogEntries(t, mdb, 2000), "the late COMMIT must not log the txn twice") +} + +// countLocalLogEntries counts mdb's local log entries for txnID. +func countLocalLogEntries(t *testing.T, mdb *db.ReplicatedDatabase, txnID uint64) int { + t.Helper() + entries, _, _, err := mdb.GetMetaStore().ListCommittedLog(db.LogPosition{}, 4096) + require.NoError(t, err) + n := 0 + for _, e := range entries { + if e.TxnID == txnID { + n++ + } + } + return n +} + +// TestLogPuller_AbandonedBeginAfterRestartResolvedInOnePull pins that a txn +// this node had only begun (never durably prepared) when its process died +// does not read PENDING forever after the restart: the first PullPair call +// delivers that txn from the peer, and every entry after it. +func TestLogPuller_AbandonedBeginAfterRestartResolvedInOnePull(t *testing.T) { + peer := newPullTestNode(t, 2, "app") + peer.serve(t, nil) + local := newPullTestNode(t, 1, "app") + + peer.seed(t, "app", 1000, 2, 1000, 1, "a") + peer.seed(t, "app", 2000, 2, 2000, 2, "b") + peer.seed(t, "app", 3000, 2, 3000, 3, "c") + + mdb, err := local.dm.GetDatabase("app") + require.NoError(t, err) + require.NoError(t, mdb.GetMetaStore().BeginTransaction(2000, 1, hlc.Timestamp{WallTime: 1, NodeID: 1})) + local.restart(t) + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm}) + res, err := lp.PullPair(context.Background(), PeerRef{NodeID: 2, Address: peer.addr}, "app") + require.NoError(t, err) + require.True(t, res.CaughtUp) + require.Equal(t, 3, res.Applied) + require.Equal(t, map[int64]string{1: "a", 2: "b", 3: "c"}, local.rows(t, "app")) +} + +// TestLogPuller_LocalIntentConflictSkipsEntryContinuesAndResolvesLater +// pins that a row locally held by a foreign write intent refuses that +// entry (ErrReplayIntentConflict), not counted as a failure, while later +// entries in the same page still apply; the cursor stops before the +// conflicting entry until the intent is released. +func TestLogPuller_LocalIntentConflictSkipsEntryContinuesAndResolvesLater(t *testing.T) { + peer := newPullTestNode(t, 2, "app") + peer.serve(t, nil) + local := newPullTestNode(t, 1, "app") + + peer.seed(t, "app", 1000, 2, 1000, 1, "a") + peer.seed(t, "app", 2000, 2, 2000, 2, "b") // row id=2 conflicts locally + peer.seed(t, "app", 3000, 2, 3000, 3, "c") + + mdb, err := local.dm.GetDatabase("app") + require.NoError(t, err) + const foreignTxnID = uint64(9999) + require.NoError(t, mdb.GetMetaStore().WriteIntent(foreignTxnID, db.IntentTypeDML, "t", "t:2", + db.OpTypeInsert, "", nil, hlc.Timestamp{WallTime: 1, NodeID: 1}, 1)) + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm}) + peerRef := PeerRef{NodeID: 2, Address: peer.addr} + + res, err := lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.False(t, res.CaughtUp) + require.Zero(t, res.Stuck, "an intent conflict must not be counted as a failed attempt") + require.Equal(t, 2, res.Applied, "the entry before and after the conflict must both apply") + require.Equal(t, map[int64]string{1: "a", 3: "c"}, local.rows(t, "app")) + + cursor, err := mdb.GetMetaStore().GetPullCursor(2) + require.NoError(t, err) + require.Equal(t, uint64(1000), cursor.TxnID, "the cursor must stop before the conflicting entry") + + require.NoError(t, mdb.GetMetaStore().DeleteIntentsByTxn(foreignTxnID)) + + res, err = lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.True(t, res.CaughtUp) + require.Equal(t, map[int64]string{1: "a", 2: "b", 3: "c"}, local.rows(t, "app")) +} + +// TestLogPuller_PoisonTxnStuckAfterMaxAttempts pins that a +// deterministically-failing entry is reported STUCK exactly once after +// MaxReplayAttempts, exposed via StuckTxns and the LogPullStuckTxns metric, +// while other entries (before and after it) still apply and the cursor +// never passes it. Once STUCK, it is retried only every +// StuckRetryEveryRounds calls of PullPair for that pair. +func TestLogPuller_PoisonTxnStuckAfterMaxAttempts(t *testing.T) { + peer := newPullTestNode(t, 2, "app") + peer.serve(t, nil) + local := newPullTestNode(t, 1, "app") + + peerMdb, err := peer.dm.GetDatabase("app") + require.NoError(t, err) + + peer.seed(t, "app", 1000, 2, 1000, 1, "a") + + // txn 2000 targets a table that exists nowhere, so every replay attempt + // fails deterministically - including on the peer itself, which still + // logs it COMMITTED (log-first) before the SQLite apply fails. + poisonRow := &db.EncodedCapturedRow{ + Table: "missing_table", + Op: uint8(db.OpTypeInsert), + IntentKey: []byte("missing_table:1"), + NewValues: encodeSeedValues(t, map[string]interface{}{"id": int64(1)}), + } + _, err = peerMdb.ApplyReplayedTxn(context.Background(), &db.ReplayTxn{ + TxnID: 2000, OriginNodeID: 2, CommitTS: hlc.Timestamp{WallTime: 2000, NodeID: 2}, + Rows: []*db.EncodedCapturedRow{poisonRow}, + }, true) + require.Error(t, err, "the poison row must fail to apply on the peer too") + + peer.seed(t, "app", 3000, 2, 3000, 3, "c") + + lp := NewLogPuller(LogPullerConfig{ + NodeID: 1, Client: NewClient(1), DBManager: local.dm, + MaxReplayAttempts: 3, StuckRetryEveryRounds: 2, + }) + peerRef := PeerRef{NodeID: 2, Address: peer.addr} + + var res PairResult + for i := 0; i < 3; i++ { + res, err = lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + } + + require.Equal(t, map[int64]string{1: "a", 3: "c"}, local.rows(t, "app"), "entries before and after the poison txn must still apply") + + stuck := lp.StuckTxns() + require.Len(t, stuck, 1) + require.Equal(t, "app", stuck[0].Database) + require.Equal(t, uint64(2), stuck[0].PeerNodeID) + require.Equal(t, uint64(2000), stuck[0].TxnID) + require.Equal(t, 3, stuck[0].Attempts) + require.NotEmpty(t, stuck[0].LastError) + require.Equal(t, 1, res.Stuck) + + mdb, err := local.dm.GetDatabase("app") + require.NoError(t, err) + cursor, err := mdb.GetMetaStore().GetPullCursor(2) + require.NoError(t, err) + require.Equal(t, uint64(1000), cursor.TxnID, "the cursor must never pass a STUCK txn") + + // Round 4 (4 % StuckRetryEveryRounds(2) == 0): retried, fails again. + res, err = lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.Equal(t, 1, res.Stuck) + stuck = lp.StuckTxns() + require.Equal(t, 4, stuck[0].Attempts, "a retry round must re-attempt a STUCK txn") + + // Round 5 (5 % 2 != 0): skipped, attempts unchanged. + res, err = lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.Equal(t, 1, res.Stuck) + stuck = lp.StuckTxns() + require.Equal(t, 4, stuck[0].Attempts, "a non-retry round must not re-attempt a STUCK txn") +} + +// TestLogPuller_NeedsSnapshotWhenPeerTruncatedPastCursor pins that a +// peer whose GC has truncated past this pair's cursor is reported +// NeedsSnapshot with nothing applied, keeps reporting it until the database +// is restored, and after MarkRestored the next pull moves the cursor up to +// the peer's truncation point instead. +func TestLogPuller_NeedsSnapshotWhenPeerTruncatedPastCursor(t *testing.T) { + peer := newPullTestNode(t, 2, "app") + peer.serve(t, nil) + local := newPullTestNode(t, 1, "app") + + peer.seed(t, "app", 1000, 2, 1000, 1, "a") + peer.seed(t, "app", 2000, 2, 2000, 2, "b") + + peerMdb, err := peer.dm.GetDatabase("app") + require.NoError(t, err) + deleted, err := peerMdb.GetMetaStore().CleanupOldTransactionRecords(0, 0, db.LogPosition{Seq: ^uint64(0), TxnID: ^uint64(0)}) + require.NoError(t, err) + require.Equal(t, 2, deleted) + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm}) + peerRef := PeerRef{NodeID: 2, Address: peer.addr} + + res, err := lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.True(t, res.NeedsSnapshot) + require.Zero(t, res.Applied) + require.Empty(t, local.rows(t, "app")) + + res, err = lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.True(t, res.NeedsSnapshot, "without a restore the truncated gap is never skipped") + + lp.MarkRestored("app") + res, err = lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.False(t, res.NeedsSnapshot, "after a restore the pull resumes from the truncation point") + require.True(t, res.CaughtUp) + + localMdb, err := local.dm.GetDatabase("app") + require.NoError(t, err) + cursor, err := localMdb.GetMetaStore().GetPullCursor(2) + require.NoError(t, err) + truncated, err := peerMdb.GetMetaStore().TruncatedThrough() + require.NoError(t, err) + require.Equal(t, truncated, cursor, "MarkRestored must move the cursor to the peer's truncation point") +} + +// TestLogPuller_SecondRestoreLowersCursorToRecoverAnEntryTheFirstCovered +// pins the two-restore loss: a first restore brings txn X's marker without X +// entering this node's log, so the pull covers X by marker and its cursor +// passes X; a second restore, from a source lacking X, removes X again. The +// pull after that restore must move its cursor down to the peer's +// truncation point - not keep a cursor already past it - so X is listed and +// applied again. +func TestLogPuller_SecondRestoreLowersCursorToRecoverAnEntryTheFirstCovered(t *testing.T) { + peer := newPullTestNode(t, 2, "app") + peer.serve(t, nil) + local := newPullTestNode(t, 1, "app") + + peer.seed(t, "app", 1000, 2, 1000, 1, "x") + peer.seed(t, "app", 2000, 2, 2000, 2, "y") + + mdb, err := local.dm.GetDatabase("app") + require.NoError(t, err) + // Restore 1: X's row and marker arrive in the file, not in the log. + applied, err := mdb.ApplyReplayedTxn(context.Background(), &db.ReplayTxn{ + TxnID: 1000, OriginNodeID: 2, CommitTS: hlc.Timestamp{WallTime: 1000, NodeID: 2}, + Rows: []*db.EncodedCapturedRow{{Table: "t", Op: uint8(db.OpTypeInsert), IntentKey: []byte("t:1"), + NewValues: encodeSeedValues(t, map[string]interface{}{"id": int64(1), "v": "x"})}}, + }, false) + require.NoError(t, err) + require.True(t, applied) + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm}) + peerRef := PeerRef{NodeID: 2, Address: peer.addr} + lp.MarkRestored("app") + res, err := lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.True(t, res.CaughtUp) + cursor, err := mdb.GetMetaStore().GetPullCursor(2) + require.NoError(t, err) + require.Equal(t, uint64(2000), cursor.TxnID, "the cursor passes X, covered by its marker") + + // Restore 2, from a source that never had X. + _, err = mdb.GetWriteDB().Exec("DELETE FROM t WHERE id = 1") + require.NoError(t, err) + _, err = mdb.GetWriteDB().Exec("DELETE FROM __marmot_applied_txn WHERE txn_id = 1000") + require.NoError(t, err) + lp.MarkRestored("app") + + res, err = lp.PullPair(context.Background(), peerRef, "app") + require.NoError(t, err) + require.True(t, res.CaughtUp) + require.Equal(t, map[int64]string{1: "x", 2: "y"}, local.rows(t, "app"), "X must be pulled again after the second restore") +} + +// failingFetchServer serves FetchTransactions like the real server, except +// that it fails the whole call, as the real one does for an entry it cannot +// serve, when it reaches badTxnID. +type failingFetchServer struct { + *Server + badTxnID uint64 +} + +func (f failingFetchServer) FetchTransactions(req *FetchTransactionsRequest, stream MarmotService_FetchTransactionsServer) error { + for _, id := range req.TxnIds { + if id == f.badTxnID { + return status.Errorf(codes.FailedPrecondition, "transaction %d: captured rows unreadable", id) + } + one := &FetchTransactionsRequest{Database: req.Database, RequestingNodeId: req.RequestingNodeId, TxnIds: []uint64{id}} + if err := f.Server.FetchTransactions(one, stream); err != nil { + return err + } + } + return nil +} + +// TestLogPuller_OneUnservableTxnDoesNotBlockLaterIdsInItsFetch pins that a +// fetch the peer fails at one transaction still delivers the healthy ids +// after it in the same call, and that only the failing id counts a failed +// attempt. +func TestLogPuller_OneUnservableTxnDoesNotBlockLaterIdsInItsFetch(t *testing.T) { + peer := newPullTestNode(t, 2, "app") + peer.serve(t, func(s *Server) MarmotServiceServer { return failingFetchServer{Server: s, badTxnID: 2000} }) + local := newPullTestNode(t, 1, "app") + + peer.seed(t, "app", 1000, 2, 1000, 1, "a") + peer.seed(t, "app", 2000, 2, 2000, 2, "b") + peer.seed(t, "app", 3000, 2, 3000, 3, "c") + peer.seed(t, "app", 4000, 2, 4000, 4, "d") + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm, MaxReplayAttempts: 1}) + res, err := lp.PullPair(context.Background(), PeerRef{NodeID: 2, Address: peer.addr}, "app") + require.NoError(t, err) + require.False(t, res.CaughtUp) + require.Equal(t, 3, res.Applied, "every healthy id must apply in the same call") + require.Equal(t, map[int64]string{1: "a", 3: "c", 4: "d"}, local.rows(t, "app")) + + stuck := lp.StuckTxns() + require.Len(t, stuck, 1, "only the unservable id may count a failed attempt") + require.Equal(t, uint64(2000), stuck[0].TxnID) + require.Equal(t, 1, res.Stuck) +} + +// TestLogPuller_UnimplementedPeerNoError pins the rolling-upgrade path: +// a peer that does not implement ListCommittedLog answers Unimplemented, +// and PullPair reports that without an error. +func TestLogPuller_UnimplementedPeerNoError(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + gs := grpc.NewServer() + RegisterMarmotServiceServer(gs, UnimplementedMarmotServiceServer{}) + go func() { _ = gs.Serve(listener) }() + t.Cleanup(gs.Stop) + + local := newPullTestNode(t, 1, "app") + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm}) + + res, err := lp.PullPair(context.Background(), PeerRef{NodeID: 2, Address: listener.Addr().String()}, "app") + require.NoError(t, err) + require.True(t, res.Unimplemented) + require.Zero(t, res.Applied) +} + +// TestLogPuller_DDLReplayAdvancesSchemaVersionForPrepareGate: a DDL applied +// through PullPair advances the local schema +// version (bumped inside the same tx as the marker, on the puller's own +// node), and a subsequent PREPARE stamped with that version is accepted. +func TestLogPuller_DDLReplayAdvancesSchemaVersionForPrepareGate(t *testing.T) { + peer := newPullTestNode(t, 2, "app") + peer.serve(t, nil) + local := newPullTestNode(t, 1, "app") + + peer.seedDDL(t, "app", 1000, 2, 1000, "ALTER TABLE t ADD COLUMN extra TEXT") + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm}) + res, err := lp.PullPair(context.Background(), PeerRef{NodeID: 2, Address: peer.addr}, "app") + require.NoError(t, err) + require.True(t, res.CaughtUp) + + mdb, err := local.dm.GetDatabase("app") + require.NoError(t, err) + require.Equal(t, uint64(1), mdb.SchemaVersion(), "the replayed DDL must advance the local schema version") + + svm := db.NewSchemaVersionManager(local.dm) + rh := NewReplicationHandler(1, local.dm, hlc.NewClock(1), svm) + + resp, err := rh.HandleReplicateTransaction(context.Background(), &TransactionRequest{ + TxnId: 2000, SourceNodeId: 2, Database: "app", Phase: TransactionPhase_PREPARE, + Timestamp: &HLC{WallTime: 2000, NodeId: 2}, RequiredSchemaVersion: mdb.SchemaVersion(), + Statements: []*Statement{{Type: pb.StatementType_INSERT, TableName: "t", Database: "app", + Payload: &Statement_RowChange{RowChange: testInsertRowChange("t", []byte("t:1"), map[string][]byte{ + "id": mustMarshalMsgpack(t, int64(1)), "v": mustMarshalMsgpack(t, "prepared"), + })}}}, + }) + require.NoError(t, err) + require.True(t, resp.Success, resp.ErrorMessage) +} diff --git a/grpc/log_server.go b/grpc/log_server.go new file mode 100644 index 00000000..6a2d6743 --- /dev/null +++ b/grpc/log_server.go @@ -0,0 +1,318 @@ +package grpc + +import ( + "context" + "errors" + "fmt" + + "github.com/maxpert/marmot/common" + "github.com/maxpert/marmot/db" + "github.com/rs/zerolog/log" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + // defaultLogListLimit is ListCommittedLog's page size when the request + // leaves Limit unset (0). + defaultLogListLimit = 256 + + // maxLogListLimit caps ListCommittedLog's page size regardless of what + // the request asks for. + maxLogListLimit = 4096 + + // maxFetchTransactionIDs caps how many txn ids one FetchTransactions + // call accepts, so a single call cannot be made to iterate an unbounded + // number of transactions. + maxFetchTransactionIDs = 4096 +) + +// ListCommittedLog serves one page of this node's local commit log for one +// database, strictly after (after_seq, after_txn_id) and up to the store's +// stable point, to a peer pulling for anti-entropy. Before it answers, it durably records the requester's consumed +// position R[self,requester,d] = (consumed_seq, consumed_txn_id) as is - not +// as a max. It is separate from the +// list position because a requester lists past an entry it could not apply +// yet while its consumed position stays +// before that entry, so GC keeps it. +// +// A database this node does not have (never created, or tombstoned) answers +// database_absent=true rather than an error: the caller's database-set +// reconciliation (ListDatabaseRegistry) is what decides whether that is +// expected. +func (s *Server) ListCommittedLog(ctx context.Context, req *LogListRequest) (*LogListResponse, error) { + s.mu.RLock() + dbManager := s.dbManager + s.mu.RUnlock() + if dbManager == nil { + return nil, status.Error(codes.Unavailable, "database manager not initialized") + } + if req.RequestingNodeId != 0 { + s.registry.TouchLastSeen(req.RequestingNodeId) + } + + mdb, err := dbManager.GetDatabase(req.Database) + if err != nil { + if errors.Is(err, db.ErrDatabaseDetached) { + return nil, status.Errorf(codes.Unavailable, "database %s: %v", req.Database, err) + } + return &LogListResponse{DatabaseAbsent: true}, nil + } + metaStore := mdb.GetMetaStore() + + after := db.LogPosition{Seq: req.AfterSeq, TxnID: req.AfterTxnId} + if req.RequestingNodeId != 0 { + consumed := db.LogPosition{Seq: req.ConsumedSeq, TxnID: req.ConsumedTxnId} + if err := metaStore.SetConsumedPosition(req.RequestingNodeId, consumed); err != nil { + return nil, status.Errorf(codes.Internal, "record consumed position: %v", err) + } + } + + limit := int(req.Limit) + if limit <= 0 { + limit = defaultLogListLimit + } + if limit > maxLogListLimit { + limit = maxLogListLimit + } + + entries, stable, more, err := metaStore.ListCommittedLog(after, limit) + if err != nil { + return nil, status.Errorf(codes.Internal, "list committed log: %v", err) + } + truncated, err := metaStore.TruncatedThrough() + if err != nil { + return nil, status.Errorf(codes.Internal, "read truncation point: %v", err) + } + + pbEntries := make([]*LogEntry, 0, len(entries)) + for _, e := range entries { + pbEntries = append(pbEntries, &LogEntry{Seq: e.Seq, TxnId: e.TxnID}) + } + + return &LogListResponse{ + Entries: pbEntries, + StableSeq: stable, + TruncatedSeq: truncated.Seq, + TruncatedTxnId: truncated.TxnID, + More: more, + SchemaVersion: mdb.SchemaVersion(), + }, nil +} + +// FetchTransactions streams the named committed transactions of one +// database, in the order requested, then ends (EOF). Unlike StreamChanges' +// existing lenient semantics (kept unchanged for read replicas), +// it fails the WHOLE call rather than silently skip or send a partial +// transaction: +// - a requested id that is missing, or not COMMITTED (GC may have removed +// it): codes.NotFound; +// - any captured-row iterate or decode error, or a captured row count that +// does not match the commit record's own RowCount: codes.FailedPrecondition. +// +// Each event carries the transaction's immutable NodeID as origin_node_id +// (the DDL incarnation owner, which must not change across replays) and the +// commit record's RowCount, with the HLC timestamp's NodeId also set to the +// origin. +func (s *Server) FetchTransactions(req *FetchTransactionsRequest, stream MarmotService_FetchTransactionsServer) error { + s.mu.RLock() + dbManager := s.dbManager + s.mu.RUnlock() + if dbManager == nil { + return status.Error(codes.Unavailable, "database manager not initialized") + } + if req.RequestingNodeId != 0 { + s.registry.TouchLastSeen(req.RequestingNodeId) + } + if len(req.TxnIds) > maxFetchTransactionIDs { + return status.Errorf(codes.InvalidArgument, "fetch transactions: %d ids exceeds the %d per-call limit", len(req.TxnIds), maxFetchTransactionIDs) + } + + mdb, err := dbManager.GetDatabase(req.Database) + if err != nil { + return status.Errorf(codes.NotFound, "database %s: %v", req.Database, err) + } + metaStore := mdb.GetMetaStore() + + for _, txnID := range req.TxnIds { + rec, err := metaStore.GetTransaction(txnID) + if err != nil { + return status.Errorf(codes.Internal, "read transaction %d: %v", txnID, err) + } + if rec == nil || rec.Status != db.TxnStatusCommitted { + return status.Errorf(codes.NotFound, "transaction %d is not a committed local log entry", txnID) + } + + statements, err := strictStatementsForFetch(req.Database, rec, metaStore) + if err != nil { + return status.Errorf(codes.FailedPrecondition, "transaction %d: %v", txnID, err) + } + + event := &ChangeEvent{ + TxnId: rec.TxnID, + Statements: statements, + Timestamp: &HLC{ + WallTime: rec.CommitTSWall, + Logical: rec.CommitTSLogical, + NodeId: rec.NodeID, + }, + Database: req.Database, + RequiredSchemaVersion: rec.RequiredSchemaVersion, + SeqNum: rec.SeqNum, + OriginNodeId: rec.NodeID, + RowCount: rec.RowCount, + } + if err := stream.Send(event); err != nil { + return err + } + } + return nil +} + +// ListDatabaseRegistry lists this node's database registry, tombstones +// included, so a peer's anti-entropy round can reconcile the database set +// before pulling logs. +func (s *Server) ListDatabaseRegistry(ctx context.Context, req *DatabaseRegistryRequest) (*DatabaseRegistryResponse, error) { + s.mu.RLock() + dbManager := s.dbManager + s.mu.RUnlock() + if dbManager == nil { + return nil, status.Error(codes.Unavailable, "database manager not initialized") + } + if req.RequestingNodeId != 0 { + s.registry.TouchLastSeen(req.RequestingNodeId) + } + + entries, err := dbManager.RegistryEntries() + if err != nil { + return nil, status.Errorf(codes.Internal, "list database registry: %v", err) + } + + pbEntries := make([]*DatabaseRegistryEntry, 0, len(entries)) + for _, e := range entries { + pbEntries = append(pbEntries, &DatabaseRegistryEntry{ + Name: e.Name, + Generation: e.Key.Generation, + Dropped: e.Key.Dropped, + Legacy: e.Legacy, + }) + } + return &DatabaseRegistryResponse{Entries: pbEntries}, nil +} + +// statementFromCapturedRow converts one decoded captured row into the wire +// Statement both sendChangeEvent (StreamChanges) and FetchTransactions send. +// encodedRow is the row's raw msgpack encoding, carried as-is on the DML +// path rather than re-encoded. +func statementFromCapturedRow(dbName string, row *db.EncodedCapturedRow, encodedRow []byte) (*Statement, error) { + switch db.OpType(row.Op) { + case db.OpTypeVectorIndex: + if row.VectorIndexChange == nil { + return nil, fmt.Errorf("vector index CDC row missing payload") + } + return &Statement{ + Type: common.MustToWireType(common.StatementVectorIndexControl), + TableName: row.Table, + Database: dbName, + Payload: &Statement_VectorIndexChange{ + VectorIndexChange: vectorChangeToProto(*row.VectorIndexChange), + }, + }, nil + case db.OpTypeDDL: + return &Statement{ + Type: common.MustToWireType(db.OpTypeToStatementType(db.OpType(row.Op))), + TableName: row.Table, + Database: dbName, + Payload: &Statement_DdlChange{ + DdlChange: &DDLChange{Sql: row.DDLSQL}, + }, + }, nil + case db.OpTypeLoadData: + return &Statement{ + Type: common.MustToWireType(db.OpTypeToStatementType(db.OpType(row.Op))), + TableName: row.Table, + Database: dbName, + Payload: &Statement_LoadDataChange{ + LoadDataChange: &LoadDataChange{Sql: row.LoadSQL, Data: row.LoadData}, + }, + }, nil + default: + return &Statement{ + Type: common.MustToWireType(db.OpTypeToStatementType(db.OpType(row.Op))), + TableName: row.Table, + Database: dbName, + Payload: &Statement_RowChange{ + RowChange: &RowChange{ + EncodedRow: append([]byte(nil), encodedRow...), + EncodedRowCodec: db.EncodedCapturedRowCodecMsgpack(), + }, + }, + }, nil + } +} + +// lenientStatementsFromLog builds rec's wire statements from its captured +// rows, keeping StreamChanges' existing lenient semantics unchanged for read +// replicas: an iterate or decode error is logged and the +// affected row is skipped rather than failing the whole stream. +func lenientStatementsFromLog(rec *db.TransactionRecord, metaStore db.MetaStore) []*Statement { + cursor, err := metaStore.IterateCapturedRows(rec.TxnID) + if err != nil { + log.Warn().Err(err).Uint64("txn_id", rec.TxnID).Msg("Failed to iterate captured rows for streaming") + return nil + } + defer cursor.Close() + + var statements []*Statement + for cursor.Next() { + _, data := cursor.Row() + row, err := db.DecodeRow(data) + if err != nil { + log.Warn().Err(err).Uint64("txn_id", rec.TxnID).Msg("Failed to decode captured row") + continue + } + stmt, err := statementFromCapturedRow(rec.DatabaseName, row, data) + if err != nil { + log.Warn().Err(err).Uint64("txn_id", rec.TxnID).Msg("Failed to convert captured row for streaming") + continue + } + statements = append(statements, stmt) + } + if err := cursor.Err(); err != nil { + log.Warn().Err(err).Uint64("txn_id", rec.TxnID).Msg("Failed to iterate captured rows for streaming") + } + return statements +} + +// strictStatementsForFetch builds rec's wire statements from its captured +// rows for FetchTransactions, refusing to serve a partial transaction: any +// iterate or decode error, or a captured row count that does not +// match rec.RowCount, fails outright instead of skipping the offending row. +func strictStatementsForFetch(database string, rec *db.TransactionRecord, metaStore db.MetaStore) ([]*Statement, error) { + cursor, err := metaStore.IterateCapturedRows(rec.TxnID) + if err != nil { + return nil, fmt.Errorf("iterate captured rows: %w", err) + } + defer cursor.Close() + + statements := make([]*Statement, 0, rec.RowCount) + for cursor.Next() { + _, data := cursor.Row() + row, err := db.DecodeRow(data) + if err != nil { + return nil, fmt.Errorf("decode captured row: %w", err) + } + stmt, err := statementFromCapturedRow(database, row, data) + if err != nil { + return nil, fmt.Errorf("convert captured row: %w", err) + } + statements = append(statements, stmt) + } + if err := cursor.Err(); err != nil { + return nil, fmt.Errorf("iterate captured rows: %w", err) + } + if uint32(len(statements)) != rec.RowCount { + return nil, fmt.Errorf("captured row count mismatch: got %d want %d", len(statements), rec.RowCount) + } + return statements, nil +} diff --git a/grpc/log_server_test.go b/grpc/log_server_test.go new file mode 100644 index 00000000..c6dad859 --- /dev/null +++ b/grpc/log_server_test.go @@ -0,0 +1,247 @@ +package grpc + +import ( + "context" + "io" + "testing" + + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" +) + +// dialLogPullClient connects to addr with no cluster-secret interceptor +// (servePeer's grpc.Server registers none); auth coverage for these RPCs is +// TestLogPullRPCsRequireClusterSecret in auth_test.go. +func dialLogPullClient(t *testing.T, addr string) MarmotServiceClient { + t.Helper() + conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { conn.Close() }) + return NewMarmotServiceClient(conn) +} + +// TestListCommittedLog_PagingStableTruncatedAndConsumedPosition drives +// ListCommittedLog against a real in-process server (servePeer, the pattern +// catch_up_detach_test.go uses) over 5 committed DDL transactions: it must +// page correctly, report every entry as stable (the log-seq tracker marks a +// commit done synchronously, before CommitTransaction returns, so nothing +// pulled from the same goroutine's commits is ever missing), and it must +// record the requester's consumed position as-is - not as a max - +// before it answers. +func TestListCommittedLog_PagingStableTruncatedAndConsumedPosition(t *testing.T) { + dm, err := db.NewDatabaseManager(t.TempDir(), 2, hlc.NewClock(2)) + require.NoError(t, err) + t.Cleanup(func() { dm.Close() }) + require.NoError(t, dm.CreateDatabase("app")) + bumpSchemaVersionForTest(t, dm, "app", 5) + + addr := servePeer(t, dm, 2, nil) + client := dialLogPullClient(t, addr) + + mdb, err := dm.GetDatabase("app") + require.NoError(t, err) + metaStore := mdb.GetMetaStore() + + // database_absent for a database this node does not have. + absentResp, err := client.ListCommittedLog(context.Background(), &LogListRequest{Database: "nope", RequestingNodeId: 42}) + require.NoError(t, err) + require.True(t, absentResp.DatabaseAbsent) + + // Page through the whole log with a small limit and collect every entry. + var all []*LogEntry + after := &LogListRequest{Database: "app", RequestingNodeId: 42, Limit: 2} + for { + resp, err := client.ListCommittedLog(context.Background(), after) + require.NoError(t, err) + require.False(t, resp.DatabaseAbsent) + all = append(all, resp.Entries...) + if !resp.More { + require.LessOrEqual(t, len(resp.Entries), 2) + break + } + require.NotEmpty(t, resp.Entries) + last := resp.Entries[len(resp.Entries)-1] + after = &LogListRequest{Database: "app", RequestingNodeId: 42, AfterSeq: last.Seq, AfterTxnId: last.TxnId, Limit: 2} + } + require.Len(t, all, 5, "expected all 5 committed DDL transactions to be listed") + for i := 1; i < len(all); i++ { + require.True(t, + all[i-1].Seq < all[i].Seq || (all[i-1].Seq == all[i].Seq && all[i-1].TxnId < all[i].TxnId), + "entries must be strictly increasing in position order: %+v then %+v", all[i-1], all[i]) + } + + stable := metaStore.StableSeq() + require.GreaterOrEqual(t, stable, all[len(all)-1].Seq, "every entry returned must be at or below the store's own stable point") + + lastResp, err := client.ListCommittedLog(context.Background(), &LogListRequest{Database: "app", RequestingNodeId: 42}) + require.NoError(t, err) + require.False(t, lastResp.More) + require.Equal(t, uint64(0), lastResp.TruncatedSeq, "GC has not run: nothing is truncated yet") + require.Equal(t, uint64(0), lastResp.TruncatedTxnId) + require.Equal(t, mdb.SchemaVersion(), lastResp.SchemaVersion) + + // The consumed position is the request's consumed field, recorded + // as-is, not as a max, and never the list position: a requester lists + // past an entry it has not applied yet. + highPos := db.LogPosition{Seq: all[len(all)-1].Seq, TxnID: all[len(all)-1].TxnId} + lowPos := db.LogPosition{Seq: all[0].Seq, TxnID: all[0].TxnId} + _, err = client.ListCommittedLog(context.Background(), &LogListRequest{ + Database: "app", RequestingNodeId: 99, AfterSeq: highPos.Seq, AfterTxnId: highPos.TxnID, + ConsumedSeq: highPos.Seq, ConsumedTxnId: highPos.TxnID, + }) + require.NoError(t, err) + positions, err := metaStore.ConsumedPositions() + require.NoError(t, err) + require.Equal(t, highPos, positions[99]) + + _, err = client.ListCommittedLog(context.Background(), &LogListRequest{ + Database: "app", RequestingNodeId: 99, AfterSeq: highPos.Seq, AfterTxnId: highPos.TxnID, + ConsumedSeq: lowPos.Seq, ConsumedTxnId: lowPos.TxnID, + }) + require.NoError(t, err) + positions, err = metaStore.ConsumedPositions() + require.NoError(t, err) + require.Equal(t, lowPos, positions[99], "the consumed field is stored as-is (not a max) and not replaced by the list position") +} + +// TestFetchTransactions_EOFOriginAndRowCount streams every committed +// transaction of a database to EOF and checks origin_node_id and row_count +// match the commit record exactly. +func TestFetchTransactions_EOFOriginAndRowCount(t *testing.T) { + dm, err := db.NewDatabaseManager(t.TempDir(), 3, hlc.NewClock(3)) + require.NoError(t, err) + t.Cleanup(func() { dm.Close() }) + require.NoError(t, dm.CreateDatabase("app")) + bumpSchemaVersionForTest(t, dm, "app", 3) + + mdb, err := dm.GetDatabase("app") + require.NoError(t, err) + metaStore := mdb.GetMetaStore() + entries, _, more, err := metaStore.ListCommittedLog(db.LogPosition{}, 100) + require.NoError(t, err) + require.False(t, more) + require.Len(t, entries, 3) + + addr := servePeer(t, dm, 3, nil) + client := dialLogPullClient(t, addr) + + ids := make([]uint64, len(entries)) + for i, e := range entries { + ids[i] = e.TxnID + } + stream, err := client.FetchTransactions(context.Background(), &FetchTransactionsRequest{ + Database: "app", RequestingNodeId: 42, TxnIds: ids, + }) + require.NoError(t, err) + + var got []*ChangeEvent + for { + ev, err := stream.Recv() + if err == io.EOF { + break + } + require.NoError(t, err) + got = append(got, ev) + } + require.Len(t, got, 3) + for _, ev := range got { + rec, err := metaStore.GetTransaction(ev.TxnId) + require.NoError(t, err) + require.NotNil(t, rec) + require.Equal(t, rec.NodeID, ev.OriginNodeId, "origin_node_id must be the commit record's immutable NodeID") + require.Equal(t, rec.NodeID, ev.Timestamp.NodeId, "timestamp NodeId must also carry the origin") + require.Equal(t, rec.RowCount, ev.RowCount) + require.Len(t, ev.Statements, int(rec.RowCount)) + } +} + +// TestFetchTransactions_NotFoundForMissingTxn fails the whole call with +// codes.NotFound for a requested id this node's log does not hold (missing, +// or GC'd), never serving a partial set. +func TestFetchTransactions_NotFoundForMissingTxn(t *testing.T) { + dm, err := db.NewDatabaseManager(t.TempDir(), 4, hlc.NewClock(4)) + require.NoError(t, err) + t.Cleanup(func() { dm.Close() }) + require.NoError(t, dm.CreateDatabase("app")) + + addr := servePeer(t, dm, 4, nil) + client := dialLogPullClient(t, addr) + + stream, err := client.FetchTransactions(context.Background(), &FetchTransactionsRequest{ + Database: "app", RequestingNodeId: 42, TxnIds: []uint64{999999}, + }) + require.NoError(t, err) + _, err = stream.Recv() + require.Error(t, err) + st, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, codes.NotFound, st.Code()) +} + +// TestFetchTransactions_FailedPreconditionOnCorruptCapturedRow writes a +// captured row directly through MetaStore that db.DecodeRow cannot parse, +// then checks FetchTransactions fails the whole call rather than serve a +// partial or garbage transaction. +func TestFetchTransactions_FailedPreconditionOnCorruptCapturedRow(t *testing.T) { + dm, err := db.NewDatabaseManager(t.TempDir(), 5, hlc.NewClock(5)) + require.NoError(t, err) + t.Cleanup(func() { dm.Close() }) + require.NoError(t, dm.CreateDatabase("app")) + + mdb, err := dm.GetDatabase("app") + require.NoError(t, err) + metaStore := mdb.GetMetaStore() + + const txnID = 12345 + require.NoError(t, metaStore.BeginTransaction(txnID, 5, hlc.Timestamp{WallTime: 1})) + require.NoError(t, metaStore.WriteCapturedRow(txnID, 1, []byte("not a valid msgpack captured row"))) + require.NoError(t, metaStore.SealCapturedRows(txnID)) + require.NoError(t, metaStore.CommitTransaction(txnID, hlc.Timestamp{WallTime: 2}, nil, "app", "", 0, 1)) + + addr := servePeer(t, dm, 5, nil) + client := dialLogPullClient(t, addr) + + stream, err := client.FetchTransactions(context.Background(), &FetchTransactionsRequest{ + Database: "app", RequestingNodeId: 42, TxnIds: []uint64{txnID}, + }) + require.NoError(t, err) + _, err = stream.Recv() + require.Error(t, err) + st, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, codes.FailedPrecondition, st.Code()) +} + +// TestListDatabaseRegistry_IncludesTombstones lists both a live and a +// dropped database, so a peer's anti-entropy round can reconcile a drop it +// missed. +func TestListDatabaseRegistry_IncludesTombstones(t *testing.T) { + dm, err := db.NewDatabaseManager(t.TempDir(), 6, hlc.NewClock(6)) + require.NoError(t, err) + t.Cleanup(func() { dm.Close() }) + require.NoError(t, dm.CreateDatabase("keepme")) + require.NoError(t, dm.CreateDatabase("dropme")) + require.NoError(t, dm.DropDatabase("dropme")) + + addr := servePeer(t, dm, 6, nil) + client := dialLogPullClient(t, addr) + + resp, err := client.ListDatabaseRegistry(context.Background(), &DatabaseRegistryRequest{RequestingNodeId: 42}) + require.NoError(t, err) + + byName := make(map[string]*DatabaseRegistryEntry, len(resp.Entries)) + for _, e := range resp.Entries { + byName[e.Name] = e + } + require.NotNil(t, byName["keepme"]) + require.False(t, byName["keepme"].Dropped) + require.GreaterOrEqual(t, byName["keepme"].Generation, uint64(1)) + + require.NotNil(t, byName["dropme"]) + require.True(t, byName["dropme"].Dropped, "a dropped database must be listed as a tombstone, not omitted") +} diff --git a/grpc/marmot.pb.go b/grpc/marmot.pb.go index d7ac713b..7c2ebd1f 100644 --- a/grpc/marmot.pb.go +++ b/grpc/marmot.pb.go @@ -405,8 +405,15 @@ type NodeState struct { DatabaseSchemaVersions map[string]uint64 `protobuf:"bytes,5,rep,name=database_schema_versions,json=databaseSchemaVersions,proto3" json:"database_schema_versions,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` // Minimum applied sequence number (watermark for GC coordination) MinAppliedSeq uint64 `protobuf:"varint,6,opt,name=min_applied_seq,json=minAppliedSeq,proto3" json:"min_applied_seq,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // log_protocol_version is the node's log-pull anti-entropy protocol + // generation: 0 means an older release (no ListCommittedLog / + // FetchTransactions / ListDatabaseRegistry RPCs, and a per-database DDL + // history counter that is not comparable with this protocol's), 1 + // (LogPullProtocolVersion) means it serves those RPCs. An older release + // never sets this for itself, so it always reports 0 here. + LogProtocolVersion uint32 `protobuf:"varint,7,opt,name=log_protocol_version,json=logProtocolVersion,proto3" json:"log_protocol_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeState) Reset() { @@ -481,6 +488,13 @@ func (x *NodeState) GetMinAppliedSeq() uint64 { return 0 } +func (x *NodeState) GetLogProtocolVersion() uint32 { + if x != nil { + return x.LogProtocolVersion + } + return 0 +} + type JoinRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NodeId uint64 `protobuf:"varint,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` @@ -826,6 +840,10 @@ type Statement struct { // rolling-upgrade gate above, DDLChange is the wrong statement type, and // DMLIntent carries only an intent key. AutoIdClaimPayload []byte `protobuf:"bytes,10,opt,name=auto_id_claim_payload,json=autoIdClaimPayload,proto3" json:"auto_id_claim_payload,omitempty"` + // database_generation stamps a CREATE or DROP DATABASE statement with the + // registry generation the coordinator computed for it; with the statement + // type it forms the key participants order database operations by. + DatabaseGeneration uint64 `protobuf:"varint,11,opt,name=database_generation,json=databaseGeneration,proto3" json:"database_generation,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -947,6 +965,13 @@ func (x *Statement) GetAutoIdClaimPayload() []byte { return nil } +func (x *Statement) GetDatabaseGeneration() uint64 { + if x != nil { + return x.DatabaseGeneration + } + return 0 +} + type isStatement_Payload interface { isStatement_Payload() } @@ -1843,7 +1868,9 @@ type ChangeEvent struct { Timestamp *HLC `protobuf:"bytes,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` Database string `protobuf:"bytes,4,opt,name=database,proto3" json:"database,omitempty"` // Target database for these changes RequiredSchemaVersion uint64 `protobuf:"varint,5,opt,name=required_schema_version,json=requiredSchemaVersion,proto3" json:"required_schema_version,omitempty"` // Minimum schema version required for this transaction - SeqNum uint64 `protobuf:"varint,6,opt,name=seq_num,json=seqNum,proto3" json:"seq_num,omitempty"` // Monotonic sequence number for gap detection + SeqNum uint64 `protobuf:"varint,6,opt,name=seq_num,json=seqNum,proto3" json:"seq_num,omitempty"` // Position of this transaction in the sender's local commit log + OriginNodeId uint64 `protobuf:"varint,7,opt,name=origin_node_id,json=originNodeId,proto3" json:"origin_node_id,omitempty"` // Node that coordinated the transaction + RowCount uint32 `protobuf:"varint,8,opt,name=row_count,json=rowCount,proto3" json:"row_count,omitempty"` // Statement count recorded at commit; a receiver refuses a mismatch unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1920,6 +1947,475 @@ func (x *ChangeEvent) GetSeqNum() uint64 { return 0 } +func (x *ChangeEvent) GetOriginNodeId() uint64 { + if x != nil { + return x.OriginNodeId + } + return 0 +} + +func (x *ChangeEvent) GetRowCount() uint32 { + if x != nil { + return x.RowCount + } + return 0 +} + +// ===== LOG PULL MESSAGES ===== +type LogListRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Database string `protobuf:"bytes,1,opt,name=database,proto3" json:"database,omitempty"` + RequestingNodeId uint64 `protobuf:"varint,2,opt,name=requesting_node_id,json=requestingNodeId,proto3" json:"requesting_node_id,omitempty"` + AfterSeq uint64 `protobuf:"varint,3,opt,name=after_seq,json=afterSeq,proto3" json:"after_seq,omitempty"` // List strictly after the position (after_seq, after_txn_id) + AfterTxnId uint64 `protobuf:"varint,4,opt,name=after_txn_id,json=afterTxnId,proto3" json:"after_txn_id,omitempty"` + Limit uint32 `protobuf:"varint,5,opt,name=limit,proto3" json:"limit,omitempty"` + ConsumedSeq uint64 `protobuf:"varint,6,opt,name=consumed_seq,json=consumedSeq,proto3" json:"consumed_seq,omitempty"` // Position (consumed_seq, consumed_txn_id) the requester has applied every entry through + ConsumedTxnId uint64 `protobuf:"varint,7,opt,name=consumed_txn_id,json=consumedTxnId,proto3" json:"consumed_txn_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LogListRequest) Reset() { + *x = LogListRequest{} + mi := &file_grpc_marmot_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LogListRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LogListRequest) ProtoMessage() {} + +func (x *LogListRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_marmot_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LogListRequest.ProtoReflect.Descriptor instead. +func (*LogListRequest) Descriptor() ([]byte, []int) { + return file_grpc_marmot_proto_rawDescGZIP(), []int{21} +} + +func (x *LogListRequest) GetDatabase() string { + if x != nil { + return x.Database + } + return "" +} + +func (x *LogListRequest) GetRequestingNodeId() uint64 { + if x != nil { + return x.RequestingNodeId + } + return 0 +} + +func (x *LogListRequest) GetAfterSeq() uint64 { + if x != nil { + return x.AfterSeq + } + return 0 +} + +func (x *LogListRequest) GetAfterTxnId() uint64 { + if x != nil { + return x.AfterTxnId + } + return 0 +} + +func (x *LogListRequest) GetLimit() uint32 { + if x != nil { + return x.Limit + } + return 0 +} + +func (x *LogListRequest) GetConsumedSeq() uint64 { + if x != nil { + return x.ConsumedSeq + } + return 0 +} + +func (x *LogListRequest) GetConsumedTxnId() uint64 { + if x != nil { + return x.ConsumedTxnId + } + return 0 +} + +type LogEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + Seq uint64 `protobuf:"varint,1,opt,name=seq,proto3" json:"seq,omitempty"` + TxnId uint64 `protobuf:"varint,2,opt,name=txn_id,json=txnId,proto3" json:"txn_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LogEntry) Reset() { + *x = LogEntry{} + mi := &file_grpc_marmot_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LogEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LogEntry) ProtoMessage() {} + +func (x *LogEntry) ProtoReflect() protoreflect.Message { + mi := &file_grpc_marmot_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LogEntry.ProtoReflect.Descriptor instead. +func (*LogEntry) Descriptor() ([]byte, []int) { + return file_grpc_marmot_proto_rawDescGZIP(), []int{22} +} + +func (x *LogEntry) GetSeq() uint64 { + if x != nil { + return x.Seq + } + return 0 +} + +func (x *LogEntry) GetTxnId() uint64 { + if x != nil { + return x.TxnId + } + return 0 +} + +type LogListResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Entries []*LogEntry `protobuf:"bytes,1,rep,name=entries,proto3" json:"entries,omitempty"` + StableSeq uint64 `protobuf:"varint,2,opt,name=stable_seq,json=stableSeq,proto3" json:"stable_seq,omitempty"` // No entry at or below it can still become visible + TruncatedSeq uint64 `protobuf:"varint,3,opt,name=truncated_seq,json=truncatedSeq,proto3" json:"truncated_seq,omitempty"` // Highest position GC has deleted + TruncatedTxnId uint64 `protobuf:"varint,4,opt,name=truncated_txn_id,json=truncatedTxnId,proto3" json:"truncated_txn_id,omitempty"` + More bool `protobuf:"varint,5,opt,name=more,proto3" json:"more,omitempty"` // More stable entries follow the last one returned + SchemaVersion uint64 `protobuf:"varint,6,opt,name=schema_version,json=schemaVersion,proto3" json:"schema_version,omitempty"` + DatabaseAbsent bool `protobuf:"varint,7,opt,name=database_absent,json=databaseAbsent,proto3" json:"database_absent,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LogListResponse) Reset() { + *x = LogListResponse{} + mi := &file_grpc_marmot_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LogListResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LogListResponse) ProtoMessage() {} + +func (x *LogListResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_marmot_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LogListResponse.ProtoReflect.Descriptor instead. +func (*LogListResponse) Descriptor() ([]byte, []int) { + return file_grpc_marmot_proto_rawDescGZIP(), []int{23} +} + +func (x *LogListResponse) GetEntries() []*LogEntry { + if x != nil { + return x.Entries + } + return nil +} + +func (x *LogListResponse) GetStableSeq() uint64 { + if x != nil { + return x.StableSeq + } + return 0 +} + +func (x *LogListResponse) GetTruncatedSeq() uint64 { + if x != nil { + return x.TruncatedSeq + } + return 0 +} + +func (x *LogListResponse) GetTruncatedTxnId() uint64 { + if x != nil { + return x.TruncatedTxnId + } + return 0 +} + +func (x *LogListResponse) GetMore() bool { + if x != nil { + return x.More + } + return false +} + +func (x *LogListResponse) GetSchemaVersion() uint64 { + if x != nil { + return x.SchemaVersion + } + return 0 +} + +func (x *LogListResponse) GetDatabaseAbsent() bool { + if x != nil { + return x.DatabaseAbsent + } + return false +} + +type FetchTransactionsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Database string `protobuf:"bytes,1,opt,name=database,proto3" json:"database,omitempty"` + RequestingNodeId uint64 `protobuf:"varint,2,opt,name=requesting_node_id,json=requestingNodeId,proto3" json:"requesting_node_id,omitempty"` + TxnIds []uint64 `protobuf:"varint,3,rep,packed,name=txn_ids,json=txnIds,proto3" json:"txn_ids,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FetchTransactionsRequest) Reset() { + *x = FetchTransactionsRequest{} + mi := &file_grpc_marmot_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FetchTransactionsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FetchTransactionsRequest) ProtoMessage() {} + +func (x *FetchTransactionsRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_marmot_proto_msgTypes[24] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FetchTransactionsRequest.ProtoReflect.Descriptor instead. +func (*FetchTransactionsRequest) Descriptor() ([]byte, []int) { + return file_grpc_marmot_proto_rawDescGZIP(), []int{24} +} + +func (x *FetchTransactionsRequest) GetDatabase() string { + if x != nil { + return x.Database + } + return "" +} + +func (x *FetchTransactionsRequest) GetRequestingNodeId() uint64 { + if x != nil { + return x.RequestingNodeId + } + return 0 +} + +func (x *FetchTransactionsRequest) GetTxnIds() []uint64 { + if x != nil { + return x.TxnIds + } + return nil +} + +type DatabaseRegistryRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestingNodeId uint64 `protobuf:"varint,1,opt,name=requesting_node_id,json=requestingNodeId,proto3" json:"requesting_node_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DatabaseRegistryRequest) Reset() { + *x = DatabaseRegistryRequest{} + mi := &file_grpc_marmot_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DatabaseRegistryRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DatabaseRegistryRequest) ProtoMessage() {} + +func (x *DatabaseRegistryRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_marmot_proto_msgTypes[25] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DatabaseRegistryRequest.ProtoReflect.Descriptor instead. +func (*DatabaseRegistryRequest) Descriptor() ([]byte, []int) { + return file_grpc_marmot_proto_rawDescGZIP(), []int{25} +} + +func (x *DatabaseRegistryRequest) GetRequestingNodeId() uint64 { + if x != nil { + return x.RequestingNodeId + } + return 0 +} + +type DatabaseRegistryEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Generation uint64 `protobuf:"varint,2,opt,name=generation,proto3" json:"generation,omitempty"` + Dropped bool `protobuf:"varint,3,opt,name=dropped,proto3" json:"dropped,omitempty"` + // legacy marks a registry row migrated from an older release's registry; a legacy + // live row never creates the database on a peer that lacks it. + Legacy bool `protobuf:"varint,4,opt,name=legacy,proto3" json:"legacy,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DatabaseRegistryEntry) Reset() { + *x = DatabaseRegistryEntry{} + mi := &file_grpc_marmot_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DatabaseRegistryEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DatabaseRegistryEntry) ProtoMessage() {} + +func (x *DatabaseRegistryEntry) ProtoReflect() protoreflect.Message { + mi := &file_grpc_marmot_proto_msgTypes[26] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DatabaseRegistryEntry.ProtoReflect.Descriptor instead. +func (*DatabaseRegistryEntry) Descriptor() ([]byte, []int) { + return file_grpc_marmot_proto_rawDescGZIP(), []int{26} +} + +func (x *DatabaseRegistryEntry) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *DatabaseRegistryEntry) GetGeneration() uint64 { + if x != nil { + return x.Generation + } + return 0 +} + +func (x *DatabaseRegistryEntry) GetDropped() bool { + if x != nil { + return x.Dropped + } + return false +} + +func (x *DatabaseRegistryEntry) GetLegacy() bool { + if x != nil { + return x.Legacy + } + return false +} + +type DatabaseRegistryResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Entries []*DatabaseRegistryEntry `protobuf:"bytes,1,rep,name=entries,proto3" json:"entries,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DatabaseRegistryResponse) Reset() { + *x = DatabaseRegistryResponse{} + mi := &file_grpc_marmot_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DatabaseRegistryResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DatabaseRegistryResponse) ProtoMessage() {} + +func (x *DatabaseRegistryResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_marmot_proto_msgTypes[27] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DatabaseRegistryResponse.ProtoReflect.Descriptor instead. +func (*DatabaseRegistryResponse) Descriptor() ([]byte, []int) { + return file_grpc_marmot_proto_rawDescGZIP(), []int{27} +} + +func (x *DatabaseRegistryResponse) GetEntries() []*DatabaseRegistryEntry { + if x != nil { + return x.Entries + } + return nil +} + type ReplicationStateRequest struct { state protoimpl.MessageState `protogen:"open.v1"` RequestingNodeId uint64 `protobuf:"varint,1,opt,name=requesting_node_id,json=requestingNodeId,proto3" json:"requesting_node_id,omitempty"` @@ -1930,7 +2426,7 @@ type ReplicationStateRequest struct { func (x *ReplicationStateRequest) Reset() { *x = ReplicationStateRequest{} - mi := &file_grpc_marmot_proto_msgTypes[21] + mi := &file_grpc_marmot_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1942,7 +2438,7 @@ func (x *ReplicationStateRequest) String() string { func (*ReplicationStateRequest) ProtoMessage() {} func (x *ReplicationStateRequest) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[21] + mi := &file_grpc_marmot_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1955,7 +2451,7 @@ func (x *ReplicationStateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReplicationStateRequest.ProtoReflect.Descriptor instead. func (*ReplicationStateRequest) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{21} + return file_grpc_marmot_proto_rawDescGZIP(), []int{28} } func (x *ReplicationStateRequest) GetRequestingNodeId() uint64 { @@ -1981,7 +2477,7 @@ type ReplicationStateResponse struct { func (x *ReplicationStateResponse) Reset() { *x = ReplicationStateResponse{} - mi := &file_grpc_marmot_proto_msgTypes[22] + mi := &file_grpc_marmot_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1993,7 +2489,7 @@ func (x *ReplicationStateResponse) String() string { func (*ReplicationStateResponse) ProtoMessage() {} func (x *ReplicationStateResponse) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[22] + mi := &file_grpc_marmot_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2006,7 +2502,7 @@ func (x *ReplicationStateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReplicationStateResponse.ProtoReflect.Descriptor instead. func (*ReplicationStateResponse) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{22} + return file_grpc_marmot_proto_rawDescGZIP(), []int{29} } func (x *ReplicationStateResponse) GetStates() []*DatabaseReplicationState { @@ -2032,7 +2528,7 @@ type DatabaseReplicationState struct { func (x *DatabaseReplicationState) Reset() { *x = DatabaseReplicationState{} - mi := &file_grpc_marmot_proto_msgTypes[23] + mi := &file_grpc_marmot_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2044,7 +2540,7 @@ func (x *DatabaseReplicationState) String() string { func (*DatabaseReplicationState) ProtoMessage() {} func (x *DatabaseReplicationState) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[23] + mi := &file_grpc_marmot_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2057,7 +2553,7 @@ func (x *DatabaseReplicationState) ProtoReflect() protoreflect.Message { // Deprecated: Use DatabaseReplicationState.ProtoReflect.Descriptor instead. func (*DatabaseReplicationState) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{23} + return file_grpc_marmot_proto_rawDescGZIP(), []int{30} } func (x *DatabaseReplicationState) GetDatabaseName() string { @@ -2131,7 +2627,7 @@ type SnapshotInfoRequest struct { func (x *SnapshotInfoRequest) Reset() { *x = SnapshotInfoRequest{} - mi := &file_grpc_marmot_proto_msgTypes[24] + mi := &file_grpc_marmot_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2143,7 +2639,7 @@ func (x *SnapshotInfoRequest) String() string { func (*SnapshotInfoRequest) ProtoMessage() {} func (x *SnapshotInfoRequest) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[24] + mi := &file_grpc_marmot_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2156,7 +2652,7 @@ func (x *SnapshotInfoRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SnapshotInfoRequest.ProtoReflect.Descriptor instead. func (*SnapshotInfoRequest) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{24} + return file_grpc_marmot_proto_rawDescGZIP(), []int{31} } func (x *SnapshotInfoRequest) GetRequestingNodeId() uint64 { @@ -2187,7 +2683,7 @@ type SnapshotInfoResponse struct { func (x *SnapshotInfoResponse) Reset() { *x = SnapshotInfoResponse{} - mi := &file_grpc_marmot_proto_msgTypes[25] + mi := &file_grpc_marmot_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2199,7 +2695,7 @@ func (x *SnapshotInfoResponse) String() string { func (*SnapshotInfoResponse) ProtoMessage() {} func (x *SnapshotInfoResponse) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[25] + mi := &file_grpc_marmot_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2212,7 +2708,7 @@ func (x *SnapshotInfoResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SnapshotInfoResponse.ProtoReflect.Descriptor instead. func (*SnapshotInfoResponse) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{25} + return file_grpc_marmot_proto_rawDescGZIP(), []int{32} } func (x *SnapshotInfoResponse) GetSnapshotTxnId() uint64 { @@ -2269,7 +2765,7 @@ type DatabaseFileInfo struct { func (x *DatabaseFileInfo) Reset() { *x = DatabaseFileInfo{} - mi := &file_grpc_marmot_proto_msgTypes[26] + mi := &file_grpc_marmot_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2281,7 +2777,7 @@ func (x *DatabaseFileInfo) String() string { func (*DatabaseFileInfo) ProtoMessage() {} func (x *DatabaseFileInfo) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[26] + mi := &file_grpc_marmot_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2294,7 +2790,7 @@ func (x *DatabaseFileInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use DatabaseFileInfo.ProtoReflect.Descriptor instead. func (*DatabaseFileInfo) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{26} + return file_grpc_marmot_proto_rawDescGZIP(), []int{33} } func (x *DatabaseFileInfo) GetName() string { @@ -2342,7 +2838,7 @@ type DatabaseSnapshotMetadata struct { func (x *DatabaseSnapshotMetadata) Reset() { *x = DatabaseSnapshotMetadata{} - mi := &file_grpc_marmot_proto_msgTypes[27] + mi := &file_grpc_marmot_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2354,7 +2850,7 @@ func (x *DatabaseSnapshotMetadata) String() string { func (*DatabaseSnapshotMetadata) ProtoMessage() {} func (x *DatabaseSnapshotMetadata) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[27] + mi := &file_grpc_marmot_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2367,7 +2863,7 @@ func (x *DatabaseSnapshotMetadata) ProtoReflect() protoreflect.Message { // Deprecated: Use DatabaseSnapshotMetadata.ProtoReflect.Descriptor instead. func (*DatabaseSnapshotMetadata) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{27} + return file_grpc_marmot_proto_rawDescGZIP(), []int{34} } func (x *DatabaseSnapshotMetadata) GetDatabaseName() string { @@ -2415,7 +2911,7 @@ type SnapshotRequest struct { func (x *SnapshotRequest) Reset() { *x = SnapshotRequest{} - mi := &file_grpc_marmot_proto_msgTypes[28] + mi := &file_grpc_marmot_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2427,7 +2923,7 @@ func (x *SnapshotRequest) String() string { func (*SnapshotRequest) ProtoMessage() {} func (x *SnapshotRequest) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[28] + mi := &file_grpc_marmot_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2440,7 +2936,7 @@ func (x *SnapshotRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SnapshotRequest.ProtoReflect.Descriptor instead. func (*SnapshotRequest) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{28} + return file_grpc_marmot_proto_rawDescGZIP(), []int{35} } func (x *SnapshotRequest) GetRequestingNodeId() uint64 { @@ -2465,13 +2961,19 @@ type SnapshotChunk struct { Checksum string `protobuf:"bytes,4,opt,name=checksum,proto3" json:"checksum,omitempty"` Filename string `protobuf:"bytes,5,opt,name=filename,proto3" json:"filename,omitempty"` // Which database file this chunk belongs to IsLastForFile bool `protobuf:"varint,6,opt,name=is_last_for_file,json=isLastForFile,proto3" json:"is_last_for_file,omitempty"` // True if this is the last chunk for this file + // Set on a file's last chunk: the SHA256 and size of the file exactly as + // this stream sent it. A receiver verifies against these rather than the + // checksums GetSnapshotInfo reported, which describe an earlier, separate + // snapshot of a database that may have been written since. + FileSha256 string `protobuf:"bytes,7,opt,name=file_sha256,json=fileSha256,proto3" json:"file_sha256,omitempty"` + FileSizeBytes int64 `protobuf:"varint,8,opt,name=file_size_bytes,json=fileSizeBytes,proto3" json:"file_size_bytes,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *SnapshotChunk) Reset() { *x = SnapshotChunk{} - mi := &file_grpc_marmot_proto_msgTypes[29] + mi := &file_grpc_marmot_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2483,7 +2985,7 @@ func (x *SnapshotChunk) String() string { func (*SnapshotChunk) ProtoMessage() {} func (x *SnapshotChunk) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[29] + mi := &file_grpc_marmot_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2496,7 +2998,7 @@ func (x *SnapshotChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use SnapshotChunk.ProtoReflect.Descriptor instead. func (*SnapshotChunk) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{29} + return file_grpc_marmot_proto_rawDescGZIP(), []int{36} } func (x *SnapshotChunk) GetChunkIndex() int32 { @@ -2541,6 +3043,20 @@ func (x *SnapshotChunk) GetIsLastForFile() bool { return false } +func (x *SnapshotChunk) GetFileSha256() string { + if x != nil { + return x.FileSha256 + } + return "" +} + +func (x *SnapshotChunk) GetFileSizeBytes() int64 { + if x != nil { + return x.FileSizeBytes + } + return 0 +} + // ===== DELTA SYNC DETECTION ===== type LatestTxnIDsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -2551,7 +3067,7 @@ type LatestTxnIDsRequest struct { func (x *LatestTxnIDsRequest) Reset() { *x = LatestTxnIDsRequest{} - mi := &file_grpc_marmot_proto_msgTypes[30] + mi := &file_grpc_marmot_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2563,7 +3079,7 @@ func (x *LatestTxnIDsRequest) String() string { func (*LatestTxnIDsRequest) ProtoMessage() {} func (x *LatestTxnIDsRequest) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[30] + mi := &file_grpc_marmot_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2576,7 +3092,7 @@ func (x *LatestTxnIDsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LatestTxnIDsRequest.ProtoReflect.Descriptor instead. func (*LatestTxnIDsRequest) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{30} + return file_grpc_marmot_proto_rawDescGZIP(), []int{37} } func (x *LatestTxnIDsRequest) GetRequestingNodeId() uint64 { @@ -2598,7 +3114,7 @@ type DatabaseInfo struct { func (x *DatabaseInfo) Reset() { *x = DatabaseInfo{} - mi := &file_grpc_marmot_proto_msgTypes[31] + mi := &file_grpc_marmot_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2610,7 +3126,7 @@ func (x *DatabaseInfo) String() string { func (*DatabaseInfo) ProtoMessage() {} func (x *DatabaseInfo) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[31] + mi := &file_grpc_marmot_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2623,7 +3139,7 @@ func (x *DatabaseInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use DatabaseInfo.ProtoReflect.Descriptor instead. func (*DatabaseInfo) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{31} + return file_grpc_marmot_proto_rawDescGZIP(), []int{38} } func (x *DatabaseInfo) GetName() string { @@ -2664,7 +3180,7 @@ type LatestTxnIDsResponse struct { func (x *LatestTxnIDsResponse) Reset() { *x = LatestTxnIDsResponse{} - mi := &file_grpc_marmot_proto_msgTypes[32] + mi := &file_grpc_marmot_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2676,7 +3192,7 @@ func (x *LatestTxnIDsResponse) String() string { func (*LatestTxnIDsResponse) ProtoMessage() {} func (x *LatestTxnIDsResponse) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[32] + mi := &file_grpc_marmot_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2689,7 +3205,7 @@ func (x *LatestTxnIDsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use LatestTxnIDsResponse.ProtoReflect.Descriptor instead. func (*LatestTxnIDsResponse) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{32} + return file_grpc_marmot_proto_rawDescGZIP(), []int{39} } func (x *LatestTxnIDsResponse) GetDatabaseTxnIds() map[string]uint64 { @@ -2716,7 +3232,7 @@ type AutoIncBasesRequest struct { func (x *AutoIncBasesRequest) Reset() { *x = AutoIncBasesRequest{} - mi := &file_grpc_marmot_proto_msgTypes[33] + mi := &file_grpc_marmot_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2728,7 +3244,7 @@ func (x *AutoIncBasesRequest) String() string { func (*AutoIncBasesRequest) ProtoMessage() {} func (x *AutoIncBasesRequest) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[33] + mi := &file_grpc_marmot_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2741,7 +3257,7 @@ func (x *AutoIncBasesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AutoIncBasesRequest.ProtoReflect.Descriptor instead. func (*AutoIncBasesRequest) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{33} + return file_grpc_marmot_proto_rawDescGZIP(), []int{40} } func (x *AutoIncBasesRequest) GetRequestingNodeId() uint64 { @@ -2762,7 +3278,7 @@ type AutoIncBase struct { func (x *AutoIncBase) Reset() { *x = AutoIncBase{} - mi := &file_grpc_marmot_proto_msgTypes[34] + mi := &file_grpc_marmot_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2774,7 +3290,7 @@ func (x *AutoIncBase) String() string { func (*AutoIncBase) ProtoMessage() {} func (x *AutoIncBase) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[34] + mi := &file_grpc_marmot_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2787,7 +3303,7 @@ func (x *AutoIncBase) ProtoReflect() protoreflect.Message { // Deprecated: Use AutoIncBase.ProtoReflect.Descriptor instead. func (*AutoIncBase) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{34} + return file_grpc_marmot_proto_rawDescGZIP(), []int{41} } func (x *AutoIncBase) GetDatabase() string { @@ -2823,7 +3339,7 @@ type AutoIncBasesResponse struct { func (x *AutoIncBasesResponse) Reset() { *x = AutoIncBasesResponse{} - mi := &file_grpc_marmot_proto_msgTypes[35] + mi := &file_grpc_marmot_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2835,7 +3351,7 @@ func (x *AutoIncBasesResponse) String() string { func (*AutoIncBasesResponse) ProtoMessage() {} func (x *AutoIncBasesResponse) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[35] + mi := &file_grpc_marmot_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2848,7 +3364,7 @@ func (x *AutoIncBasesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AutoIncBasesResponse.ProtoReflect.Descriptor instead. func (*AutoIncBasesResponse) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{35} + return file_grpc_marmot_proto_rawDescGZIP(), []int{42} } func (x *AutoIncBasesResponse) GetBases() []*AutoIncBase { @@ -2874,7 +3390,7 @@ type AutoIncSyncRequest struct { func (x *AutoIncSyncRequest) Reset() { *x = AutoIncSyncRequest{} - mi := &file_grpc_marmot_proto_msgTypes[36] + mi := &file_grpc_marmot_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2886,7 +3402,7 @@ func (x *AutoIncSyncRequest) String() string { func (*AutoIncSyncRequest) ProtoMessage() {} func (x *AutoIncSyncRequest) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[36] + mi := &file_grpc_marmot_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2899,7 +3415,7 @@ func (x *AutoIncSyncRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AutoIncSyncRequest.ProtoReflect.Descriptor instead. func (*AutoIncSyncRequest) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{36} + return file_grpc_marmot_proto_rawDescGZIP(), []int{43} } func (x *AutoIncSyncRequest) GetRequestingNodeId() uint64 { @@ -2924,7 +3440,7 @@ type AutoIncSyncResponse struct { func (x *AutoIncSyncResponse) Reset() { *x = AutoIncSyncResponse{} - mi := &file_grpc_marmot_proto_msgTypes[37] + mi := &file_grpc_marmot_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2936,7 +3452,7 @@ func (x *AutoIncSyncResponse) String() string { func (*AutoIncSyncResponse) ProtoMessage() {} func (x *AutoIncSyncResponse) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[37] + mi := &file_grpc_marmot_proto_msgTypes[44] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2949,7 +3465,7 @@ func (x *AutoIncSyncResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AutoIncSyncResponse.ProtoReflect.Descriptor instead. func (*AutoIncSyncResponse) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{37} + return file_grpc_marmot_proto_rawDescGZIP(), []int{44} } func (x *AutoIncSyncResponse) GetNodeId() uint64 { @@ -2989,7 +3505,7 @@ type GetClusterNodesRequest struct { func (x *GetClusterNodesRequest) Reset() { *x = GetClusterNodesRequest{} - mi := &file_grpc_marmot_proto_msgTypes[38] + mi := &file_grpc_marmot_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3001,7 +3517,7 @@ func (x *GetClusterNodesRequest) String() string { func (*GetClusterNodesRequest) ProtoMessage() {} func (x *GetClusterNodesRequest) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[38] + mi := &file_grpc_marmot_proto_msgTypes[45] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3014,7 +3530,7 @@ func (x *GetClusterNodesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetClusterNodesRequest.ProtoReflect.Descriptor instead. func (*GetClusterNodesRequest) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{38} + return file_grpc_marmot_proto_rawDescGZIP(), []int{45} } type GetClusterNodesResponse struct { @@ -3026,7 +3542,7 @@ type GetClusterNodesResponse struct { func (x *GetClusterNodesResponse) Reset() { *x = GetClusterNodesResponse{} - mi := &file_grpc_marmot_proto_msgTypes[39] + mi := &file_grpc_marmot_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3038,7 +3554,7 @@ func (x *GetClusterNodesResponse) String() string { func (*GetClusterNodesResponse) ProtoMessage() {} func (x *GetClusterNodesResponse) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[39] + mi := &file_grpc_marmot_proto_msgTypes[46] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3051,7 +3567,7 @@ func (x *GetClusterNodesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetClusterNodesResponse.ProtoReflect.Descriptor instead. func (*GetClusterNodesResponse) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{39} + return file_grpc_marmot_proto_rawDescGZIP(), []int{46} } func (x *GetClusterNodesResponse) GetNodes() []*NodeState { @@ -3075,7 +3591,7 @@ type TransactionChunk struct { func (x *TransactionChunk) Reset() { *x = TransactionChunk{} - mi := &file_grpc_marmot_proto_msgTypes[40] + mi := &file_grpc_marmot_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3087,7 +3603,7 @@ func (x *TransactionChunk) String() string { func (*TransactionChunk) ProtoMessage() {} func (x *TransactionChunk) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[40] + mi := &file_grpc_marmot_proto_msgTypes[47] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3100,7 +3616,7 @@ func (x *TransactionChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use TransactionChunk.ProtoReflect.Descriptor instead. func (*TransactionChunk) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{40} + return file_grpc_marmot_proto_rawDescGZIP(), []int{47} } func (x *TransactionChunk) GetTxnId() uint64 { @@ -3144,7 +3660,7 @@ type TransactionCommit struct { func (x *TransactionCommit) Reset() { *x = TransactionCommit{} - mi := &file_grpc_marmot_proto_msgTypes[41] + mi := &file_grpc_marmot_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3156,7 +3672,7 @@ func (x *TransactionCommit) String() string { func (*TransactionCommit) ProtoMessage() {} func (x *TransactionCommit) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[41] + mi := &file_grpc_marmot_proto_msgTypes[48] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3169,7 +3685,7 @@ func (x *TransactionCommit) ProtoReflect() protoreflect.Message { // Deprecated: Use TransactionCommit.ProtoReflect.Descriptor instead. func (*TransactionCommit) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{41} + return file_grpc_marmot_proto_rawDescGZIP(), []int{48} } func (x *TransactionCommit) GetTxnId() uint64 { @@ -3214,7 +3730,7 @@ type TransactionStreamMessage struct { func (x *TransactionStreamMessage) Reset() { *x = TransactionStreamMessage{} - mi := &file_grpc_marmot_proto_msgTypes[42] + mi := &file_grpc_marmot_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3226,7 +3742,7 @@ func (x *TransactionStreamMessage) String() string { func (*TransactionStreamMessage) ProtoMessage() {} func (x *TransactionStreamMessage) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[42] + mi := &file_grpc_marmot_proto_msgTypes[49] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3239,7 +3755,7 @@ func (x *TransactionStreamMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use TransactionStreamMessage.ProtoReflect.Descriptor instead. func (*TransactionStreamMessage) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{42} + return file_grpc_marmot_proto_rawDescGZIP(), []int{49} } func (x *TransactionStreamMessage) GetPayload() isTransactionStreamMessage_Payload { @@ -3300,7 +3816,7 @@ type ForwardQueryRequest struct { func (x *ForwardQueryRequest) Reset() { *x = ForwardQueryRequest{} - mi := &file_grpc_marmot_proto_msgTypes[43] + mi := &file_grpc_marmot_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3312,7 +3828,7 @@ func (x *ForwardQueryRequest) String() string { func (*ForwardQueryRequest) ProtoMessage() {} func (x *ForwardQueryRequest) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[43] + mi := &file_grpc_marmot_proto_msgTypes[50] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3325,7 +3841,7 @@ func (x *ForwardQueryRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ForwardQueryRequest.ProtoReflect.Descriptor instead. func (*ForwardQueryRequest) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{43} + return file_grpc_marmot_proto_rawDescGZIP(), []int{50} } func (x *ForwardQueryRequest) GetReplicaNodeId() uint64 { @@ -3416,7 +3932,7 @@ type ForwardQueryResponse struct { func (x *ForwardQueryResponse) Reset() { *x = ForwardQueryResponse{} - mi := &file_grpc_marmot_proto_msgTypes[44] + mi := &file_grpc_marmot_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3428,7 +3944,7 @@ func (x *ForwardQueryResponse) String() string { func (*ForwardQueryResponse) ProtoMessage() {} func (x *ForwardQueryResponse) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[44] + mi := &file_grpc_marmot_proto_msgTypes[51] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3441,7 +3957,7 @@ func (x *ForwardQueryResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ForwardQueryResponse.ProtoReflect.Descriptor instead. func (*ForwardQueryResponse) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{44} + return file_grpc_marmot_proto_rawDescGZIP(), []int{51} } func (x *ForwardQueryResponse) GetSuccess() bool { @@ -3516,7 +4032,7 @@ type ForwardLoadDataRequest struct { func (x *ForwardLoadDataRequest) Reset() { *x = ForwardLoadDataRequest{} - mi := &file_grpc_marmot_proto_msgTypes[45] + mi := &file_grpc_marmot_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3528,7 +4044,7 @@ func (x *ForwardLoadDataRequest) String() string { func (*ForwardLoadDataRequest) ProtoMessage() {} func (x *ForwardLoadDataRequest) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[45] + mi := &file_grpc_marmot_proto_msgTypes[52] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3541,7 +4057,7 @@ func (x *ForwardLoadDataRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ForwardLoadDataRequest.ProtoReflect.Descriptor instead. func (*ForwardLoadDataRequest) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{45} + return file_grpc_marmot_proto_rawDescGZIP(), []int{52} } func (x *ForwardLoadDataRequest) GetReplicaNodeId() uint64 { @@ -3612,7 +4128,7 @@ type LoadDataChunkRequest struct { func (x *LoadDataChunkRequest) Reset() { *x = LoadDataChunkRequest{} - mi := &file_grpc_marmot_proto_msgTypes[46] + mi := &file_grpc_marmot_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3624,7 +4140,7 @@ func (x *LoadDataChunkRequest) String() string { func (*LoadDataChunkRequest) ProtoMessage() {} func (x *LoadDataChunkRequest) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[46] + mi := &file_grpc_marmot_proto_msgTypes[53] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3637,7 +4153,7 @@ func (x *LoadDataChunkRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LoadDataChunkRequest.ProtoReflect.Descriptor instead. func (*LoadDataChunkRequest) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{46} + return file_grpc_marmot_proto_rawDescGZIP(), []int{53} } func (x *LoadDataChunkRequest) GetRequestingNodeId() uint64 { @@ -3678,7 +4194,7 @@ type LoadDataChunkResponse struct { func (x *LoadDataChunkResponse) Reset() { *x = LoadDataChunkResponse{} - mi := &file_grpc_marmot_proto_msgTypes[47] + mi := &file_grpc_marmot_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3690,7 +4206,7 @@ func (x *LoadDataChunkResponse) String() string { func (*LoadDataChunkResponse) ProtoMessage() {} func (x *LoadDataChunkResponse) ProtoReflect() protoreflect.Message { - mi := &file_grpc_marmot_proto_msgTypes[47] + mi := &file_grpc_marmot_proto_msgTypes[54] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3703,7 +4219,7 @@ func (x *LoadDataChunkResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use LoadDataChunkResponse.ProtoReflect.Descriptor instead. func (*LoadDataChunkResponse) Descriptor() ([]byte, []int) { - return file_grpc_marmot_proto_rawDescGZIP(), []int{47} + return file_grpc_marmot_proto_rawDescGZIP(), []int{54} } func (x *LoadDataChunkResponse) GetData() []byte { @@ -3730,14 +4246,15 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x05nodes\x18\x02 \x03(\v2\x14.marmot.v2.NodeStateR\x05nodes\x12 \n" + "\vincarnation\x18\x03 \x01(\x04R\vincarnation\"<\n" + "\x0eGossipResponse\x12*\n" + - "\x05nodes\x18\x01 \x03(\v2\x14.marmot.v2.NodeStateR\x05nodes\"\xee\x02\n" + + "\x05nodes\x18\x01 \x03(\v2\x14.marmot.v2.NodeStateR\x05nodes\"\xa0\x03\n" + "\tNodeState\x12\x17\n" + "\anode_id\x18\x01 \x01(\x04R\x06nodeId\x12\x18\n" + "\aaddress\x18\x02 \x01(\tR\aaddress\x12-\n" + "\x06status\x18\x03 \x01(\x0e2\x15.marmot.v2.NodeStatusR\x06status\x12 \n" + "\vincarnation\x18\x04 \x01(\x04R\vincarnation\x12j\n" + "\x18database_schema_versions\x18\x05 \x03(\v20.marmot.v2.NodeState.DatabaseSchemaVersionsEntryR\x16databaseSchemaVersions\x12&\n" + - "\x0fmin_applied_seq\x18\x06 \x01(\x04R\rminAppliedSeq\x1aI\n" + + "\x0fmin_applied_seq\x18\x06 \x01(\x04R\rminAppliedSeq\x120\n" + + "\x14log_protocol_version\x18\a \x01(\rR\x12logProtocolVersion\x1aI\n" + "\x1bDatabaseSchemaVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\x04R\x05value:\x028\x01\"@\n" + @@ -3762,7 +4279,7 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x05phase\x18\x05 \x01(\x0e2\x1b.marmot.v2.TransactionPhaseR\x05phase\x12=\n" + "\vconsistency\x18\x06 \x01(\x0e2\x1b.marmot.v2.ConsistencyLevelR\vconsistency\x12\x1a\n" + "\bdatabase\x18\a \x01(\tR\bdatabase\x126\n" + - "\x17required_schema_version\x18\t \x01(\x04R\x15requiredSchemaVersionJ\x04\b\b\x10\t\"\x96\x04\n" + + "\x17required_schema_version\x18\t \x01(\x04R\x15requiredSchemaVersionJ\x04\b\b\x10\t\"\xc7\x04\n" + "\tStatement\x120\n" + "\x04type\x18\x01 \x01(\x0e2\x1c.marmot.common.StatementTypeR\x04type\x12\x1d\n" + "\n" + @@ -3778,7 +4295,8 @@ const file_grpc_marmot_proto_rawDesc = "" + "dml_intent\x18\b \x01(\v2\x14.marmot.v2.DMLIntentH\x00R\tdmlIntent\x12\"\n" + "\rauto_id_claim\x18\t \x01(\bR\vautoIdClaim\x121\n" + "\x15auto_id_claim_payload\x18\n" + - " \x01(\fR\x12autoIdClaimPayloadB\t\n" + + " \x01(\fR\x12autoIdClaimPayload\x12/\n" + + "\x13database_generation\x18\v \x01(\x04R\x12databaseGenerationB\t\n" + "\apayload\"\xe4\x05\n" + "\x11VectorIndexChange\x124\n" + "\x06action\x18\x01 \x01(\x0e2\x1c.marmot.v2.VectorIndexActionR\x06action\x12\x1a\n" + @@ -3860,7 +4378,7 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x12requesting_node_id\x18\x02 \x01(\x04R\x10requestingNodeId\x12\x1a\n" + "\bdatabase\x18\x03 \x01(\tR\bdatabase\x12 \n" + "\ffrom_seq_num\x18\x04 \x01(\x04R\n" + - "fromSeqNum\"\xf5\x01\n" + + "fromSeqNum\"\xb8\x02\n" + "\vChangeEvent\x12\x15\n" + "\x06txn_id\x18\x01 \x01(\x04R\x05txnId\x124\n" + "\n" + @@ -3869,7 +4387,45 @@ const file_grpc_marmot_proto_rawDesc = "" + "\ttimestamp\x18\x03 \x01(\v2\x0e.marmot.v2.HLCR\ttimestamp\x12\x1a\n" + "\bdatabase\x18\x04 \x01(\tR\bdatabase\x126\n" + "\x17required_schema_version\x18\x05 \x01(\x04R\x15requiredSchemaVersion\x12\x17\n" + - "\aseq_num\x18\x06 \x01(\x04R\x06seqNum\"c\n" + + "\aseq_num\x18\x06 \x01(\x04R\x06seqNum\x12$\n" + + "\x0eorigin_node_id\x18\a \x01(\x04R\foriginNodeId\x12\x1b\n" + + "\trow_count\x18\b \x01(\rR\browCount\"\xfa\x01\n" + + "\x0eLogListRequest\x12\x1a\n" + + "\bdatabase\x18\x01 \x01(\tR\bdatabase\x12,\n" + + "\x12requesting_node_id\x18\x02 \x01(\x04R\x10requestingNodeId\x12\x1b\n" + + "\tafter_seq\x18\x03 \x01(\x04R\bafterSeq\x12 \n" + + "\fafter_txn_id\x18\x04 \x01(\x04R\n" + + "afterTxnId\x12\x14\n" + + "\x05limit\x18\x05 \x01(\rR\x05limit\x12!\n" + + "\fconsumed_seq\x18\x06 \x01(\x04R\vconsumedSeq\x12&\n" + + "\x0fconsumed_txn_id\x18\a \x01(\x04R\rconsumedTxnId\"3\n" + + "\bLogEntry\x12\x10\n" + + "\x03seq\x18\x01 \x01(\x04R\x03seq\x12\x15\n" + + "\x06txn_id\x18\x02 \x01(\x04R\x05txnId\"\x92\x02\n" + + "\x0fLogListResponse\x12-\n" + + "\aentries\x18\x01 \x03(\v2\x13.marmot.v2.LogEntryR\aentries\x12\x1d\n" + + "\n" + + "stable_seq\x18\x02 \x01(\x04R\tstableSeq\x12#\n" + + "\rtruncated_seq\x18\x03 \x01(\x04R\ftruncatedSeq\x12(\n" + + "\x10truncated_txn_id\x18\x04 \x01(\x04R\x0etruncatedTxnId\x12\x12\n" + + "\x04more\x18\x05 \x01(\bR\x04more\x12%\n" + + "\x0eschema_version\x18\x06 \x01(\x04R\rschemaVersion\x12'\n" + + "\x0fdatabase_absent\x18\a \x01(\bR\x0edatabaseAbsent\"}\n" + + "\x18FetchTransactionsRequest\x12\x1a\n" + + "\bdatabase\x18\x01 \x01(\tR\bdatabase\x12,\n" + + "\x12requesting_node_id\x18\x02 \x01(\x04R\x10requestingNodeId\x12\x17\n" + + "\atxn_ids\x18\x03 \x03(\x04R\x06txnIds\"G\n" + + "\x17DatabaseRegistryRequest\x12,\n" + + "\x12requesting_node_id\x18\x01 \x01(\x04R\x10requestingNodeId\"}\n" + + "\x15DatabaseRegistryEntry\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1e\n" + + "\n" + + "generation\x18\x02 \x01(\x04R\n" + + "generation\x12\x18\n" + + "\adropped\x18\x03 \x01(\bR\adropped\x12\x16\n" + + "\x06legacy\x18\x04 \x01(\bR\x06legacy\"V\n" + + "\x18DatabaseRegistryResponse\x12:\n" + + "\aentries\x18\x01 \x03(\v2 .marmot.v2.DatabaseRegistryEntryR\aentries\"c\n" + "\x17ReplicationStateRequest\x12,\n" + "\x12requesting_node_id\x18\x01 \x01(\x04R\x10requestingNodeId\x12\x1a\n" + "\bdatabase\x18\x02 \x01(\tR\bdatabase\"W\n" + @@ -3910,7 +4466,7 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x0eschema_version\x18\x05 \x01(\x04R\rschemaVersion\"[\n" + "\x0fSnapshotRequest\x12,\n" + "\x12requesting_node_id\x18\x01 \x01(\x04R\x10requestingNodeId\x12\x1a\n" + - "\bdatabase\x18\x02 \x01(\tR\bdatabase\"\xc8\x01\n" + + "\bdatabase\x18\x02 \x01(\tR\bdatabase\"\x91\x02\n" + "\rSnapshotChunk\x12\x1f\n" + "\vchunk_index\x18\x01 \x01(\x05R\n" + "chunkIndex\x12!\n" + @@ -3918,7 +4474,10 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x04data\x18\x03 \x01(\fR\x04data\x12\x1a\n" + "\bchecksum\x18\x04 \x01(\tR\bchecksum\x12\x1a\n" + "\bfilename\x18\x05 \x01(\tR\bfilename\x12'\n" + - "\x10is_last_for_file\x18\x06 \x01(\bR\risLastForFile\"C\n" + + "\x10is_last_for_file\x18\x06 \x01(\bR\risLastForFile\x12\x1f\n" + + "\vfile_sha256\x18\a \x01(\tR\n" + + "fileSha256\x12&\n" + + "\x0ffile_size_bytes\x18\b \x01(\x03R\rfileSizeBytes\"C\n" + "\x13LatestTxnIDsRequest\x12,\n" + "\x12requesting_node_id\x18\x01 \x01(\x04R\x10requestingNodeId\"~\n" + "\fDatabaseInfo\x12\x12\n" + @@ -4047,8 +4606,7 @@ const file_grpc_marmot_proto_rawDesc = "" + "\fFWD_TXN_NONE\x10\x00\x12\x11\n" + "\rFWD_TXN_BEGIN\x10\x01\x12\x12\n" + "\x0eFWD_TXN_COMMIT\x10\x02\x12\x14\n" + - "\x10FWD_TXN_ROLLBACK\x10\x032\xc3\n" + - "\n" + + "\x10FWD_TXN_ROLLBACK\x10\x032\xc3\f\n" + "\rMarmotService\x12=\n" + "\x06Gossip\x12\x18.marmot.v2.GossipRequest\x1a\x19.marmot.v2.GossipResponse\x127\n" + "\x04Join\x12\x16.marmot.v2.JoinRequest\x1a\x17.marmot.v2.JoinResponse\x127\n" + @@ -4065,7 +4623,10 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x0fForwardLoadData\x12!.marmot.v2.ForwardLoadDataRequest\x1a\x1f.marmot.v2.ForwardQueryResponse\x12U\n" + "\x10GetLoadDataChunk\x12\x1f.marmot.v2.LoadDataChunkRequest\x1a .marmot.v2.LoadDataChunkResponse\x12R\n" + "\x0fGetAutoIncBases\x12\x1e.marmot.v2.AutoIncBasesRequest\x1a\x1f.marmot.v2.AutoIncBasesResponse\x12Q\n" + - "\x10SyncAutoIncBases\x12\x1d.marmot.v2.AutoIncSyncRequest\x1a\x1e.marmot.v2.AutoIncSyncResponse\x12Z\n" + + "\x10SyncAutoIncBases\x12\x1d.marmot.v2.AutoIncSyncRequest\x1a\x1e.marmot.v2.AutoIncSyncResponse\x12I\n" + + "\x10ListCommittedLog\x12\x19.marmot.v2.LogListRequest\x1a\x1a.marmot.v2.LogListResponse\x12R\n" + + "\x11FetchTransactions\x12#.marmot.v2.FetchTransactionsRequest\x1a\x16.marmot.v2.ChangeEvent0\x01\x12_\n" + + "\x14ListDatabaseRegistry\x12\".marmot.v2.DatabaseRegistryRequest\x1a#.marmot.v2.DatabaseRegistryResponse\x12Z\n" + "\x11TransactionStream\x12#.marmot.v2.TransactionStreamMessage\x1a\x1e.marmot.v2.TransactionResponse(\x01B Z\x1egithub.com/maxpert/marmot/grpcb\x06proto3" var ( @@ -4081,7 +4642,7 @@ func file_grpc_marmot_proto_rawDescGZIP() []byte { } var file_grpc_marmot_proto_enumTypes = make([]protoimpl.EnumInfo, 5) -var file_grpc_marmot_proto_msgTypes = make([]protoimpl.MessageInfo, 51) +var file_grpc_marmot_proto_msgTypes = make([]protoimpl.MessageInfo, 58) var file_grpc_marmot_proto_goTypes = []any{ (NodeStatus)(0), // 0: marmot.v2.NodeStatus (TransactionPhase)(0), // 1: marmot.v2.TransactionPhase @@ -4109,50 +4670,57 @@ var file_grpc_marmot_proto_goTypes = []any{ (*Row)(nil), // 23: marmot.v2.Row (*StreamRequest)(nil), // 24: marmot.v2.StreamRequest (*ChangeEvent)(nil), // 25: marmot.v2.ChangeEvent - (*ReplicationStateRequest)(nil), // 26: marmot.v2.ReplicationStateRequest - (*ReplicationStateResponse)(nil), // 27: marmot.v2.ReplicationStateResponse - (*DatabaseReplicationState)(nil), // 28: marmot.v2.DatabaseReplicationState - (*SnapshotInfoRequest)(nil), // 29: marmot.v2.SnapshotInfoRequest - (*SnapshotInfoResponse)(nil), // 30: marmot.v2.SnapshotInfoResponse - (*DatabaseFileInfo)(nil), // 31: marmot.v2.DatabaseFileInfo - (*DatabaseSnapshotMetadata)(nil), // 32: marmot.v2.DatabaseSnapshotMetadata - (*SnapshotRequest)(nil), // 33: marmot.v2.SnapshotRequest - (*SnapshotChunk)(nil), // 34: marmot.v2.SnapshotChunk - (*LatestTxnIDsRequest)(nil), // 35: marmot.v2.LatestTxnIDsRequest - (*DatabaseInfo)(nil), // 36: marmot.v2.DatabaseInfo - (*LatestTxnIDsResponse)(nil), // 37: marmot.v2.LatestTxnIDsResponse - (*AutoIncBasesRequest)(nil), // 38: marmot.v2.AutoIncBasesRequest - (*AutoIncBase)(nil), // 39: marmot.v2.AutoIncBase - (*AutoIncBasesResponse)(nil), // 40: marmot.v2.AutoIncBasesResponse - (*AutoIncSyncRequest)(nil), // 41: marmot.v2.AutoIncSyncRequest - (*AutoIncSyncResponse)(nil), // 42: marmot.v2.AutoIncSyncResponse - (*GetClusterNodesRequest)(nil), // 43: marmot.v2.GetClusterNodesRequest - (*GetClusterNodesResponse)(nil), // 44: marmot.v2.GetClusterNodesResponse - (*TransactionChunk)(nil), // 45: marmot.v2.TransactionChunk - (*TransactionCommit)(nil), // 46: marmot.v2.TransactionCommit - (*TransactionStreamMessage)(nil), // 47: marmot.v2.TransactionStreamMessage - (*ForwardQueryRequest)(nil), // 48: marmot.v2.ForwardQueryRequest - (*ForwardQueryResponse)(nil), // 49: marmot.v2.ForwardQueryResponse - (*ForwardLoadDataRequest)(nil), // 50: marmot.v2.ForwardLoadDataRequest - (*LoadDataChunkRequest)(nil), // 51: marmot.v2.LoadDataChunkRequest - (*LoadDataChunkResponse)(nil), // 52: marmot.v2.LoadDataChunkResponse - nil, // 53: marmot.v2.NodeState.DatabaseSchemaVersionsEntry - nil, // 54: marmot.v2.Row.ColumnsEntry - nil, // 55: marmot.v2.LatestTxnIDsResponse.DatabaseTxnIdsEntry - (common.StatementType)(0), // 56: marmot.common.StatementType + (*LogListRequest)(nil), // 26: marmot.v2.LogListRequest + (*LogEntry)(nil), // 27: marmot.v2.LogEntry + (*LogListResponse)(nil), // 28: marmot.v2.LogListResponse + (*FetchTransactionsRequest)(nil), // 29: marmot.v2.FetchTransactionsRequest + (*DatabaseRegistryRequest)(nil), // 30: marmot.v2.DatabaseRegistryRequest + (*DatabaseRegistryEntry)(nil), // 31: marmot.v2.DatabaseRegistryEntry + (*DatabaseRegistryResponse)(nil), // 32: marmot.v2.DatabaseRegistryResponse + (*ReplicationStateRequest)(nil), // 33: marmot.v2.ReplicationStateRequest + (*ReplicationStateResponse)(nil), // 34: marmot.v2.ReplicationStateResponse + (*DatabaseReplicationState)(nil), // 35: marmot.v2.DatabaseReplicationState + (*SnapshotInfoRequest)(nil), // 36: marmot.v2.SnapshotInfoRequest + (*SnapshotInfoResponse)(nil), // 37: marmot.v2.SnapshotInfoResponse + (*DatabaseFileInfo)(nil), // 38: marmot.v2.DatabaseFileInfo + (*DatabaseSnapshotMetadata)(nil), // 39: marmot.v2.DatabaseSnapshotMetadata + (*SnapshotRequest)(nil), // 40: marmot.v2.SnapshotRequest + (*SnapshotChunk)(nil), // 41: marmot.v2.SnapshotChunk + (*LatestTxnIDsRequest)(nil), // 42: marmot.v2.LatestTxnIDsRequest + (*DatabaseInfo)(nil), // 43: marmot.v2.DatabaseInfo + (*LatestTxnIDsResponse)(nil), // 44: marmot.v2.LatestTxnIDsResponse + (*AutoIncBasesRequest)(nil), // 45: marmot.v2.AutoIncBasesRequest + (*AutoIncBase)(nil), // 46: marmot.v2.AutoIncBase + (*AutoIncBasesResponse)(nil), // 47: marmot.v2.AutoIncBasesResponse + (*AutoIncSyncRequest)(nil), // 48: marmot.v2.AutoIncSyncRequest + (*AutoIncSyncResponse)(nil), // 49: marmot.v2.AutoIncSyncResponse + (*GetClusterNodesRequest)(nil), // 50: marmot.v2.GetClusterNodesRequest + (*GetClusterNodesResponse)(nil), // 51: marmot.v2.GetClusterNodesResponse + (*TransactionChunk)(nil), // 52: marmot.v2.TransactionChunk + (*TransactionCommit)(nil), // 53: marmot.v2.TransactionCommit + (*TransactionStreamMessage)(nil), // 54: marmot.v2.TransactionStreamMessage + (*ForwardQueryRequest)(nil), // 55: marmot.v2.ForwardQueryRequest + (*ForwardQueryResponse)(nil), // 56: marmot.v2.ForwardQueryResponse + (*ForwardLoadDataRequest)(nil), // 57: marmot.v2.ForwardLoadDataRequest + (*LoadDataChunkRequest)(nil), // 58: marmot.v2.LoadDataChunkRequest + (*LoadDataChunkResponse)(nil), // 59: marmot.v2.LoadDataChunkResponse + nil, // 60: marmot.v2.NodeState.DatabaseSchemaVersionsEntry + nil, // 61: marmot.v2.Row.ColumnsEntry + nil, // 62: marmot.v2.LatestTxnIDsResponse.DatabaseTxnIdsEntry + (common.StatementType)(0), // 63: marmot.common.StatementType } var file_grpc_marmot_proto_depIdxs = []int32{ 7, // 0: marmot.v2.GossipRequest.nodes:type_name -> marmot.v2.NodeState 7, // 1: marmot.v2.GossipResponse.nodes:type_name -> marmot.v2.NodeState 0, // 2: marmot.v2.NodeState.status:type_name -> marmot.v2.NodeStatus - 53, // 3: marmot.v2.NodeState.database_schema_versions:type_name -> marmot.v2.NodeState.DatabaseSchemaVersionsEntry + 60, // 3: marmot.v2.NodeState.database_schema_versions:type_name -> marmot.v2.NodeState.DatabaseSchemaVersionsEntry 7, // 4: marmot.v2.JoinResponse.cluster_nodes:type_name -> marmot.v2.NodeState 0, // 5: marmot.v2.PingResponse.status:type_name -> marmot.v2.NodeStatus 13, // 6: marmot.v2.TransactionRequest.statements:type_name -> marmot.v2.Statement 19, // 7: marmot.v2.TransactionRequest.timestamp:type_name -> marmot.v2.HLC 1, // 8: marmot.v2.TransactionRequest.phase:type_name -> marmot.v2.TransactionPhase 2, // 9: marmot.v2.TransactionRequest.consistency:type_name -> marmot.v2.ConsistencyLevel - 56, // 10: marmot.v2.Statement.type:type_name -> marmot.common.StatementType + 63, // 10: marmot.v2.Statement.type:type_name -> marmot.common.StatementType 15, // 11: marmot.v2.Statement.row_change:type_name -> marmot.v2.RowChange 17, // 12: marmot.v2.Statement.ddl_change:type_name -> marmot.v2.DDLChange 18, // 13: marmot.v2.Statement.load_data_change:type_name -> marmot.v2.LoadDataChange @@ -4164,62 +4732,70 @@ var file_grpc_marmot_proto_depIdxs = []int32{ 2, // 19: marmot.v2.ReadRequest.consistency:type_name -> marmot.v2.ConsistencyLevel 23, // 20: marmot.v2.ReadResponse.rows:type_name -> marmot.v2.Row 19, // 21: marmot.v2.ReadResponse.timestamp:type_name -> marmot.v2.HLC - 54, // 22: marmot.v2.Row.columns:type_name -> marmot.v2.Row.ColumnsEntry + 61, // 22: marmot.v2.Row.columns:type_name -> marmot.v2.Row.ColumnsEntry 13, // 23: marmot.v2.ChangeEvent.statements:type_name -> marmot.v2.Statement 19, // 24: marmot.v2.ChangeEvent.timestamp:type_name -> marmot.v2.HLC - 28, // 25: marmot.v2.ReplicationStateResponse.states:type_name -> marmot.v2.DatabaseReplicationState - 19, // 26: marmot.v2.DatabaseReplicationState.last_applied_timestamp:type_name -> marmot.v2.HLC - 19, // 27: marmot.v2.SnapshotInfoResponse.timestamp:type_name -> marmot.v2.HLC - 31, // 28: marmot.v2.SnapshotInfoResponse.databases:type_name -> marmot.v2.DatabaseFileInfo - 32, // 29: marmot.v2.SnapshotInfoResponse.database_metadata:type_name -> marmot.v2.DatabaseSnapshotMetadata - 55, // 30: marmot.v2.LatestTxnIDsResponse.database_txn_ids:type_name -> marmot.v2.LatestTxnIDsResponse.DatabaseTxnIdsEntry - 36, // 31: marmot.v2.LatestTxnIDsResponse.database_info:type_name -> marmot.v2.DatabaseInfo - 39, // 32: marmot.v2.AutoIncBasesResponse.bases:type_name -> marmot.v2.AutoIncBase - 7, // 33: marmot.v2.GetClusterNodesResponse.nodes:type_name -> marmot.v2.NodeState - 13, // 34: marmot.v2.TransactionChunk.statements:type_name -> marmot.v2.Statement - 19, // 35: marmot.v2.TransactionCommit.timestamp:type_name -> marmot.v2.HLC - 45, // 36: marmot.v2.TransactionStreamMessage.chunk:type_name -> marmot.v2.TransactionChunk - 46, // 37: marmot.v2.TransactionStreamMessage.commit:type_name -> marmot.v2.TransactionCommit - 4, // 38: marmot.v2.ForwardQueryRequest.txn_control:type_name -> marmot.v2.ForwardTxnControl - 5, // 39: marmot.v2.MarmotService.Gossip:input_type -> marmot.v2.GossipRequest - 8, // 40: marmot.v2.MarmotService.Join:input_type -> marmot.v2.JoinRequest - 10, // 41: marmot.v2.MarmotService.Ping:input_type -> marmot.v2.PingRequest - 12, // 42: marmot.v2.MarmotService.ReplicateTransaction:input_type -> marmot.v2.TransactionRequest - 21, // 43: marmot.v2.MarmotService.Read:input_type -> marmot.v2.ReadRequest - 24, // 44: marmot.v2.MarmotService.StreamChanges:input_type -> marmot.v2.StreamRequest - 26, // 45: marmot.v2.MarmotService.GetReplicationState:input_type -> marmot.v2.ReplicationStateRequest - 29, // 46: marmot.v2.MarmotService.GetSnapshotInfo:input_type -> marmot.v2.SnapshotInfoRequest - 33, // 47: marmot.v2.MarmotService.StreamSnapshot:input_type -> marmot.v2.SnapshotRequest - 35, // 48: marmot.v2.MarmotService.GetLatestTxnIDs:input_type -> marmot.v2.LatestTxnIDsRequest - 43, // 49: marmot.v2.MarmotService.GetClusterNodes:input_type -> marmot.v2.GetClusterNodesRequest - 48, // 50: marmot.v2.MarmotService.ForwardQuery:input_type -> marmot.v2.ForwardQueryRequest - 50, // 51: marmot.v2.MarmotService.ForwardLoadData:input_type -> marmot.v2.ForwardLoadDataRequest - 51, // 52: marmot.v2.MarmotService.GetLoadDataChunk:input_type -> marmot.v2.LoadDataChunkRequest - 38, // 53: marmot.v2.MarmotService.GetAutoIncBases:input_type -> marmot.v2.AutoIncBasesRequest - 41, // 54: marmot.v2.MarmotService.SyncAutoIncBases:input_type -> marmot.v2.AutoIncSyncRequest - 47, // 55: marmot.v2.MarmotService.TransactionStream:input_type -> marmot.v2.TransactionStreamMessage - 6, // 56: marmot.v2.MarmotService.Gossip:output_type -> marmot.v2.GossipResponse - 9, // 57: marmot.v2.MarmotService.Join:output_type -> marmot.v2.JoinResponse - 11, // 58: marmot.v2.MarmotService.Ping:output_type -> marmot.v2.PingResponse - 20, // 59: marmot.v2.MarmotService.ReplicateTransaction:output_type -> marmot.v2.TransactionResponse - 22, // 60: marmot.v2.MarmotService.Read:output_type -> marmot.v2.ReadResponse - 25, // 61: marmot.v2.MarmotService.StreamChanges:output_type -> marmot.v2.ChangeEvent - 27, // 62: marmot.v2.MarmotService.GetReplicationState:output_type -> marmot.v2.ReplicationStateResponse - 30, // 63: marmot.v2.MarmotService.GetSnapshotInfo:output_type -> marmot.v2.SnapshotInfoResponse - 34, // 64: marmot.v2.MarmotService.StreamSnapshot:output_type -> marmot.v2.SnapshotChunk - 37, // 65: marmot.v2.MarmotService.GetLatestTxnIDs:output_type -> marmot.v2.LatestTxnIDsResponse - 44, // 66: marmot.v2.MarmotService.GetClusterNodes:output_type -> marmot.v2.GetClusterNodesResponse - 49, // 67: marmot.v2.MarmotService.ForwardQuery:output_type -> marmot.v2.ForwardQueryResponse - 49, // 68: marmot.v2.MarmotService.ForwardLoadData:output_type -> marmot.v2.ForwardQueryResponse - 52, // 69: marmot.v2.MarmotService.GetLoadDataChunk:output_type -> marmot.v2.LoadDataChunkResponse - 40, // 70: marmot.v2.MarmotService.GetAutoIncBases:output_type -> marmot.v2.AutoIncBasesResponse - 42, // 71: marmot.v2.MarmotService.SyncAutoIncBases:output_type -> marmot.v2.AutoIncSyncResponse - 20, // 72: marmot.v2.MarmotService.TransactionStream:output_type -> marmot.v2.TransactionResponse - 56, // [56:73] is the sub-list for method output_type - 39, // [39:56] is the sub-list for method input_type - 39, // [39:39] is the sub-list for extension type_name - 39, // [39:39] is the sub-list for extension extendee - 0, // [0:39] is the sub-list for field type_name + 27, // 25: marmot.v2.LogListResponse.entries:type_name -> marmot.v2.LogEntry + 31, // 26: marmot.v2.DatabaseRegistryResponse.entries:type_name -> marmot.v2.DatabaseRegistryEntry + 35, // 27: marmot.v2.ReplicationStateResponse.states:type_name -> marmot.v2.DatabaseReplicationState + 19, // 28: marmot.v2.DatabaseReplicationState.last_applied_timestamp:type_name -> marmot.v2.HLC + 19, // 29: marmot.v2.SnapshotInfoResponse.timestamp:type_name -> marmot.v2.HLC + 38, // 30: marmot.v2.SnapshotInfoResponse.databases:type_name -> marmot.v2.DatabaseFileInfo + 39, // 31: marmot.v2.SnapshotInfoResponse.database_metadata:type_name -> marmot.v2.DatabaseSnapshotMetadata + 62, // 32: marmot.v2.LatestTxnIDsResponse.database_txn_ids:type_name -> marmot.v2.LatestTxnIDsResponse.DatabaseTxnIdsEntry + 43, // 33: marmot.v2.LatestTxnIDsResponse.database_info:type_name -> marmot.v2.DatabaseInfo + 46, // 34: marmot.v2.AutoIncBasesResponse.bases:type_name -> marmot.v2.AutoIncBase + 7, // 35: marmot.v2.GetClusterNodesResponse.nodes:type_name -> marmot.v2.NodeState + 13, // 36: marmot.v2.TransactionChunk.statements:type_name -> marmot.v2.Statement + 19, // 37: marmot.v2.TransactionCommit.timestamp:type_name -> marmot.v2.HLC + 52, // 38: marmot.v2.TransactionStreamMessage.chunk:type_name -> marmot.v2.TransactionChunk + 53, // 39: marmot.v2.TransactionStreamMessage.commit:type_name -> marmot.v2.TransactionCommit + 4, // 40: marmot.v2.ForwardQueryRequest.txn_control:type_name -> marmot.v2.ForwardTxnControl + 5, // 41: marmot.v2.MarmotService.Gossip:input_type -> marmot.v2.GossipRequest + 8, // 42: marmot.v2.MarmotService.Join:input_type -> marmot.v2.JoinRequest + 10, // 43: marmot.v2.MarmotService.Ping:input_type -> marmot.v2.PingRequest + 12, // 44: marmot.v2.MarmotService.ReplicateTransaction:input_type -> marmot.v2.TransactionRequest + 21, // 45: marmot.v2.MarmotService.Read:input_type -> marmot.v2.ReadRequest + 24, // 46: marmot.v2.MarmotService.StreamChanges:input_type -> marmot.v2.StreamRequest + 33, // 47: marmot.v2.MarmotService.GetReplicationState:input_type -> marmot.v2.ReplicationStateRequest + 36, // 48: marmot.v2.MarmotService.GetSnapshotInfo:input_type -> marmot.v2.SnapshotInfoRequest + 40, // 49: marmot.v2.MarmotService.StreamSnapshot:input_type -> marmot.v2.SnapshotRequest + 42, // 50: marmot.v2.MarmotService.GetLatestTxnIDs:input_type -> marmot.v2.LatestTxnIDsRequest + 50, // 51: marmot.v2.MarmotService.GetClusterNodes:input_type -> marmot.v2.GetClusterNodesRequest + 55, // 52: marmot.v2.MarmotService.ForwardQuery:input_type -> marmot.v2.ForwardQueryRequest + 57, // 53: marmot.v2.MarmotService.ForwardLoadData:input_type -> marmot.v2.ForwardLoadDataRequest + 58, // 54: marmot.v2.MarmotService.GetLoadDataChunk:input_type -> marmot.v2.LoadDataChunkRequest + 45, // 55: marmot.v2.MarmotService.GetAutoIncBases:input_type -> marmot.v2.AutoIncBasesRequest + 48, // 56: marmot.v2.MarmotService.SyncAutoIncBases:input_type -> marmot.v2.AutoIncSyncRequest + 26, // 57: marmot.v2.MarmotService.ListCommittedLog:input_type -> marmot.v2.LogListRequest + 29, // 58: marmot.v2.MarmotService.FetchTransactions:input_type -> marmot.v2.FetchTransactionsRequest + 30, // 59: marmot.v2.MarmotService.ListDatabaseRegistry:input_type -> marmot.v2.DatabaseRegistryRequest + 54, // 60: marmot.v2.MarmotService.TransactionStream:input_type -> marmot.v2.TransactionStreamMessage + 6, // 61: marmot.v2.MarmotService.Gossip:output_type -> marmot.v2.GossipResponse + 9, // 62: marmot.v2.MarmotService.Join:output_type -> marmot.v2.JoinResponse + 11, // 63: marmot.v2.MarmotService.Ping:output_type -> marmot.v2.PingResponse + 20, // 64: marmot.v2.MarmotService.ReplicateTransaction:output_type -> marmot.v2.TransactionResponse + 22, // 65: marmot.v2.MarmotService.Read:output_type -> marmot.v2.ReadResponse + 25, // 66: marmot.v2.MarmotService.StreamChanges:output_type -> marmot.v2.ChangeEvent + 34, // 67: marmot.v2.MarmotService.GetReplicationState:output_type -> marmot.v2.ReplicationStateResponse + 37, // 68: marmot.v2.MarmotService.GetSnapshotInfo:output_type -> marmot.v2.SnapshotInfoResponse + 41, // 69: marmot.v2.MarmotService.StreamSnapshot:output_type -> marmot.v2.SnapshotChunk + 44, // 70: marmot.v2.MarmotService.GetLatestTxnIDs:output_type -> marmot.v2.LatestTxnIDsResponse + 51, // 71: marmot.v2.MarmotService.GetClusterNodes:output_type -> marmot.v2.GetClusterNodesResponse + 56, // 72: marmot.v2.MarmotService.ForwardQuery:output_type -> marmot.v2.ForwardQueryResponse + 56, // 73: marmot.v2.MarmotService.ForwardLoadData:output_type -> marmot.v2.ForwardQueryResponse + 59, // 74: marmot.v2.MarmotService.GetLoadDataChunk:output_type -> marmot.v2.LoadDataChunkResponse + 47, // 75: marmot.v2.MarmotService.GetAutoIncBases:output_type -> marmot.v2.AutoIncBasesResponse + 49, // 76: marmot.v2.MarmotService.SyncAutoIncBases:output_type -> marmot.v2.AutoIncSyncResponse + 28, // 77: marmot.v2.MarmotService.ListCommittedLog:output_type -> marmot.v2.LogListResponse + 25, // 78: marmot.v2.MarmotService.FetchTransactions:output_type -> marmot.v2.ChangeEvent + 32, // 79: marmot.v2.MarmotService.ListDatabaseRegistry:output_type -> marmot.v2.DatabaseRegistryResponse + 20, // 80: marmot.v2.MarmotService.TransactionStream:output_type -> marmot.v2.TransactionResponse + 61, // [61:81] is the sub-list for method output_type + 41, // [41:61] is the sub-list for method input_type + 41, // [41:41] is the sub-list for extension type_name + 41, // [41:41] is the sub-list for extension extendee + 0, // [0:41] is the sub-list for field type_name } func init() { file_grpc_marmot_proto_init() } @@ -4234,7 +4810,7 @@ func file_grpc_marmot_proto_init() { (*Statement_VectorIndexChange)(nil), (*Statement_DmlIntent)(nil), } - file_grpc_marmot_proto_msgTypes[42].OneofWrappers = []any{ + file_grpc_marmot_proto_msgTypes[49].OneofWrappers = []any{ (*TransactionStreamMessage_Chunk)(nil), (*TransactionStreamMessage_Commit)(nil), } @@ -4244,7 +4820,7 @@ func file_grpc_marmot_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_grpc_marmot_proto_rawDesc), len(file_grpc_marmot_proto_rawDesc)), NumEnums: 5, - NumMessages: 51, + NumMessages: 58, NumExtensions: 0, NumServices: 1, }, diff --git a/grpc/marmot.proto b/grpc/marmot.proto index 7443bde8..396b9d47 100644 --- a/grpc/marmot.proto +++ b/grpc/marmot.proto @@ -68,6 +68,19 @@ service MarmotService { // maximum every alive peer reports, and reports which members it reached. rpc SyncAutoIncBases(AutoIncSyncRequest) returns (AutoIncSyncResponse); + // ===== LOG PULL (anti-entropy) ===== + // Lists this node's local commit log for one database strictly after a + // position, up to its stable point, and records that position as how far + // the requester has consumed this log. + rpc ListCommittedLog(LogListRequest) returns (LogListResponse); + + // Streams the named committed transactions of one database, then ends. + // Fails the call rather than send a transaction whose rows cannot be read. + rpc FetchTransactions(FetchTransactionsRequest) returns (stream ChangeEvent); + + // Lists this node's database registry, tombstones included. + rpc ListDatabaseRegistry(DatabaseRegistryRequest) returns (DatabaseRegistryResponse); + // ===== CDC STREAMING ===== // Client-streaming RPC for large CDC payloads (≥128KB) // Sends TransactionChunks followed by TransactionCommit @@ -95,6 +108,13 @@ message NodeState { map database_schema_versions = 5; // Minimum applied sequence number (watermark for GC coordination) uint64 min_applied_seq = 6; + // log_protocol_version is the node's log-pull anti-entropy protocol + // generation: 0 means an older release (no ListCommittedLog / + // FetchTransactions / ListDatabaseRegistry RPCs, and a per-database DDL + // history counter that is not comparable with this protocol's), 1 + // (LogPullProtocolVersion) means it serves those RPCs. An older release + // never sets this for itself, so it always reports 0 here. + uint32 log_protocol_version = 7; } enum NodeStatus { @@ -195,6 +215,10 @@ message Statement { // rolling-upgrade gate above, DDLChange is the wrong statement type, and // DMLIntent carries only an intent key. bytes auto_id_claim_payload = 10; + // database_generation stamps a CREATE or DROP DATABASE statement with the + // registry generation the coordinator computed for it; with the statement + // type it forms the key participants order database operations by. + uint64 database_generation = 11; } enum VectorIndexAction { @@ -326,7 +350,58 @@ message ChangeEvent { HLC timestamp = 3; string database = 4; // Target database for these changes uint64 required_schema_version = 5; // Minimum schema version required for this transaction - uint64 seq_num = 6; // Monotonic sequence number for gap detection + uint64 seq_num = 6; // Position of this transaction in the sender's local commit log + uint64 origin_node_id = 7; // Node that coordinated the transaction + uint32 row_count = 8; // Statement count recorded at commit; a receiver refuses a mismatch +} + +// ===== LOG PULL MESSAGES ===== +message LogListRequest { + string database = 1; + uint64 requesting_node_id = 2; + uint64 after_seq = 3; // List strictly after the position (after_seq, after_txn_id) + uint64 after_txn_id = 4; + uint32 limit = 5; + uint64 consumed_seq = 6; // Position (consumed_seq, consumed_txn_id) the requester has applied every entry through + uint64 consumed_txn_id = 7; +} + +message LogEntry { + uint64 seq = 1; + uint64 txn_id = 2; +} + +message LogListResponse { + repeated LogEntry entries = 1; + uint64 stable_seq = 2; // No entry at or below it can still become visible + uint64 truncated_seq = 3; // Highest position GC has deleted + uint64 truncated_txn_id = 4; + bool more = 5; // More stable entries follow the last one returned + uint64 schema_version = 6; + bool database_absent = 7; +} + +message FetchTransactionsRequest { + string database = 1; + uint64 requesting_node_id = 2; + repeated uint64 txn_ids = 3; +} + +message DatabaseRegistryRequest { + uint64 requesting_node_id = 1; +} + +message DatabaseRegistryEntry { + string name = 1; + uint64 generation = 2; + bool dropped = 3; + // legacy marks a registry row migrated from an older release's registry; a legacy + // live row never creates the database on a peer that lacks it. + bool legacy = 4; +} + +message DatabaseRegistryResponse { + repeated DatabaseRegistryEntry entries = 1; } message ReplicationStateRequest { @@ -399,6 +474,12 @@ message SnapshotChunk { string checksum = 4; string filename = 5; // Which database file this chunk belongs to bool is_last_for_file = 6; // True if this is the last chunk for this file + // Set on a file's last chunk: the SHA256 and size of the file exactly as + // this stream sent it. A receiver verifies against these rather than the + // checksums GetSnapshotInfo reported, which describe an earlier, separate + // snapshot of a database that may have been written since. + string file_sha256 = 7; + int64 file_size_bytes = 8; } // ===== DELTA SYNC DETECTION ===== diff --git a/grpc/marmot_grpc.pb.go b/grpc/marmot_grpc.pb.go index 7df594f1..6298fb36 100644 --- a/grpc/marmot_grpc.pb.go +++ b/grpc/marmot_grpc.pb.go @@ -35,6 +35,9 @@ const ( MarmotService_GetLoadDataChunk_FullMethodName = "/marmot.v2.MarmotService/GetLoadDataChunk" MarmotService_GetAutoIncBases_FullMethodName = "/marmot.v2.MarmotService/GetAutoIncBases" MarmotService_SyncAutoIncBases_FullMethodName = "/marmot.v2.MarmotService/SyncAutoIncBases" + MarmotService_ListCommittedLog_FullMethodName = "/marmot.v2.MarmotService/ListCommittedLog" + MarmotService_FetchTransactions_FullMethodName = "/marmot.v2.MarmotService/FetchTransactions" + MarmotService_ListDatabaseRegistry_FullMethodName = "/marmot.v2.MarmotService/ListDatabaseRegistry" MarmotService_TransactionStream_FullMethodName = "/marmot.v2.MarmotService/TransactionStream" ) @@ -83,6 +86,16 @@ type MarmotServiceClient interface { // Runs one claim base sync on this node now: it raises its bases to the // maximum every alive peer reports, and reports which members it reached. SyncAutoIncBases(ctx context.Context, in *AutoIncSyncRequest, opts ...grpc.CallOption) (*AutoIncSyncResponse, error) + // ===== LOG PULL (anti-entropy) ===== + // Lists this node's local commit log for one database strictly after a + // position, up to its stable point, and records that position as how far + // the requester has consumed this log. + ListCommittedLog(ctx context.Context, in *LogListRequest, opts ...grpc.CallOption) (*LogListResponse, error) + // Streams the named committed transactions of one database, then ends. + // Fails the call rather than send a transaction whose rows cannot be read. + FetchTransactions(ctx context.Context, in *FetchTransactionsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ChangeEvent], error) + // Lists this node's database registry, tombstones included. + ListDatabaseRegistry(ctx context.Context, in *DatabaseRegistryRequest, opts ...grpc.CallOption) (*DatabaseRegistryResponse, error) // ===== CDC STREAMING ===== // Client-streaming RPC for large CDC payloads (≥128KB) // Sends TransactionChunks followed by TransactionCommit @@ -275,9 +288,48 @@ func (c *marmotServiceClient) SyncAutoIncBases(ctx context.Context, in *AutoIncS return out, nil } +func (c *marmotServiceClient) ListCommittedLog(ctx context.Context, in *LogListRequest, opts ...grpc.CallOption) (*LogListResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(LogListResponse) + err := c.cc.Invoke(ctx, MarmotService_ListCommittedLog_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *marmotServiceClient) FetchTransactions(ctx context.Context, in *FetchTransactionsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ChangeEvent], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &MarmotService_ServiceDesc.Streams[2], MarmotService_FetchTransactions_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[FetchTransactionsRequest, ChangeEvent]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type MarmotService_FetchTransactionsClient = grpc.ServerStreamingClient[ChangeEvent] + +func (c *marmotServiceClient) ListDatabaseRegistry(ctx context.Context, in *DatabaseRegistryRequest, opts ...grpc.CallOption) (*DatabaseRegistryResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(DatabaseRegistryResponse) + err := c.cc.Invoke(ctx, MarmotService_ListDatabaseRegistry_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *marmotServiceClient) TransactionStream(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[TransactionStreamMessage, TransactionResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &MarmotService_ServiceDesc.Streams[2], MarmotService_TransactionStream_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &MarmotService_ServiceDesc.Streams[3], MarmotService_TransactionStream_FullMethodName, cOpts...) if err != nil { return nil, err } @@ -333,6 +385,16 @@ type MarmotServiceServer interface { // Runs one claim base sync on this node now: it raises its bases to the // maximum every alive peer reports, and reports which members it reached. SyncAutoIncBases(context.Context, *AutoIncSyncRequest) (*AutoIncSyncResponse, error) + // ===== LOG PULL (anti-entropy) ===== + // Lists this node's local commit log for one database strictly after a + // position, up to its stable point, and records that position as how far + // the requester has consumed this log. + ListCommittedLog(context.Context, *LogListRequest) (*LogListResponse, error) + // Streams the named committed transactions of one database, then ends. + // Fails the call rather than send a transaction whose rows cannot be read. + FetchTransactions(*FetchTransactionsRequest, grpc.ServerStreamingServer[ChangeEvent]) error + // Lists this node's database registry, tombstones included. + ListDatabaseRegistry(context.Context, *DatabaseRegistryRequest) (*DatabaseRegistryResponse, error) // ===== CDC STREAMING ===== // Client-streaming RPC for large CDC payloads (≥128KB) // Sends TransactionChunks followed by TransactionCommit @@ -395,6 +457,15 @@ func (UnimplementedMarmotServiceServer) GetAutoIncBases(context.Context, *AutoIn func (UnimplementedMarmotServiceServer) SyncAutoIncBases(context.Context, *AutoIncSyncRequest) (*AutoIncSyncResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method SyncAutoIncBases not implemented") } +func (UnimplementedMarmotServiceServer) ListCommittedLog(context.Context, *LogListRequest) (*LogListResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListCommittedLog not implemented") +} +func (UnimplementedMarmotServiceServer) FetchTransactions(*FetchTransactionsRequest, grpc.ServerStreamingServer[ChangeEvent]) error { + return status.Errorf(codes.Unimplemented, "method FetchTransactions not implemented") +} +func (UnimplementedMarmotServiceServer) ListDatabaseRegistry(context.Context, *DatabaseRegistryRequest) (*DatabaseRegistryResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListDatabaseRegistry not implemented") +} func (UnimplementedMarmotServiceServer) TransactionStream(grpc.ClientStreamingServer[TransactionStreamMessage, TransactionResponse]) error { return status.Errorf(codes.Unimplemented, "method TransactionStream not implemented") } @@ -693,6 +764,53 @@ func _MarmotService_SyncAutoIncBases_Handler(srv interface{}, ctx context.Contex return interceptor(ctx, in, info, handler) } +func _MarmotService_ListCommittedLog_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LogListRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MarmotServiceServer).ListCommittedLog(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: MarmotService_ListCommittedLog_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MarmotServiceServer).ListCommittedLog(ctx, req.(*LogListRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _MarmotService_FetchTransactions_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(FetchTransactionsRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(MarmotServiceServer).FetchTransactions(m, &grpc.GenericServerStream[FetchTransactionsRequest, ChangeEvent]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type MarmotService_FetchTransactionsServer = grpc.ServerStreamingServer[ChangeEvent] + +func _MarmotService_ListDatabaseRegistry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DatabaseRegistryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MarmotServiceServer).ListDatabaseRegistry(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: MarmotService_ListDatabaseRegistry_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MarmotServiceServer).ListDatabaseRegistry(ctx, req.(*DatabaseRegistryRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _MarmotService_TransactionStream_Handler(srv interface{}, stream grpc.ServerStream) error { return srv.(MarmotServiceServer).TransactionStream(&grpc.GenericServerStream[TransactionStreamMessage, TransactionResponse]{ServerStream: stream}) } @@ -763,6 +881,14 @@ var MarmotService_ServiceDesc = grpc.ServiceDesc{ MethodName: "SyncAutoIncBases", Handler: _MarmotService_SyncAutoIncBases_Handler, }, + { + MethodName: "ListCommittedLog", + Handler: _MarmotService_ListCommittedLog_Handler, + }, + { + MethodName: "ListDatabaseRegistry", + Handler: _MarmotService_ListDatabaseRegistry_Handler, + }, }, Streams: []grpc.StreamDesc{ { @@ -775,6 +901,11 @@ var MarmotService_ServiceDesc = grpc.ServiceDesc{ Handler: _MarmotService_StreamSnapshot_Handler, ServerStreams: true, }, + { + StreamName: "FetchTransactions", + Handler: _MarmotService_FetchTransactions_Handler, + ServerStreams: true, + }, { StreamName: "TransactionStream", Handler: _MarmotService_TransactionStream_Handler, diff --git a/grpc/node_registry.go b/grpc/node_registry.go index 6ce7fad5..7ba47dd3 100644 --- a/grpc/node_registry.go +++ b/grpc/node_registry.go @@ -13,6 +13,14 @@ import ( "github.com/rs/zerolog/log" ) +// LogPullProtocolVersion is the log-pull anti-entropy protocol generation this +// binary serves (ListCommittedLog / FetchTransactions / ListDatabaseRegistry, +// and a per-database DDL history counter comparable across the cluster). A +// release before this protocol never sets NodeState.LogProtocolVersion, so it +// is always observed as 0 for such a node, and DDL is refused cluster-wide +// while any member reports less than this (LegacyLogProtocolMembers). +const LogPullProtocolVersion uint32 = 1 + // copySchemaVersionMap creates a deep copy of a schema version map func copySchemaVersionMap(m map[string]uint64) map[string]uint64 { if m == nil { @@ -34,6 +42,7 @@ func copyNodeState(node *NodeState) *NodeState { Incarnation: node.Incarnation, DatabaseSchemaVersions: copySchemaVersionMap(node.DatabaseSchemaVersions), MinAppliedSeq: node.MinAppliedSeq, + LogProtocolVersion: node.LogProtocolVersion, } } @@ -91,10 +100,11 @@ func NewNodeRegistryWithDataDir(localNodeID uint64, advertiseAddress string, dat // Add self to registry as ALIVE now := time.Now() nr.nodes[localNodeID] = &NodeState{ - NodeId: localNodeID, - Address: advertiseAddress, - Status: NodeStatus_ALIVE, - Incarnation: 0, + NodeId: localNodeID, + Address: advertiseAddress, + Status: NodeStatus_ALIVE, + Incarnation: 0, + LogProtocolVersion: LogPullProtocolVersion, } nr.lastSeen[localNodeID] = now @@ -227,10 +237,21 @@ func (nr *NodeRegistry) Update(node *NodeState) { Uint64("incarnation", node.Incarnation). Msg("REGISTRY: Escalating node status (same incarnation)") existing.Status = node.Status + if node.LogProtocolVersion > existing.LogProtocolVersion { + existing.LogProtocolVersion = node.LogProtocolVersion + } // Record state transition telemetry.NodeStateTransitionsTotal.With(oldStatus.String(), node.Status.String()).Inc() stateChanged = true + } else if node.Incarnation == existing.Incarnation && node.LogProtocolVersion > existing.LogProtocolVersion { + // Same incarnation, no status escalation: a relayed gossip copy of a + // node's own state can lack the version an earlier copy carried (for + // example a seed's Join response, which builds a bare NodeState for + // the joining node - see grpc/server.go Join). Never downgrade a + // known version; take the max for the same incarnation. + existing.LogProtocolVersion = node.LogProtocolVersion + stateChanged = true } else { log.Debug(). Uint64("local_node", nr.localNodeID). @@ -241,7 +262,8 @@ func (nr *NodeRegistry) Update(node *NodeState) { Uint64("incoming_inc", node.Incarnation). Msg("REGISTRY: Ignoring update (stale or invalid)") } - // Ignore updates with same/older incarnation that don't escalate + // Ignore updates with same/older incarnation that don't escalate and + // don't raise the known log protocol version // Refresh metrics and persist when the record changed at all. The persist // hook fingerprints exactly the fields the snapshot stores, so a call here @@ -313,8 +335,21 @@ func (nr *NodeRegistry) handleSelfUpdateLocked(node *NodeState) { return } - // If someone claims we're not ALIVE, refute by incrementing incarnation - if node.Status != NodeStatus_ALIVE && node.Incarnation >= self.Incarnation { + // If someone claims we're not ALIVE, refute by incrementing incarnation. + // + // A view of us with an older log protocol version (the one we had before + // a restart onto a newer binary, which does not bump our incarnation) is + // refuted only when it carries a HIGHER incarnation than ours: SWIM never + // lets a lower incarnation replace it, so without the refutation it would + // stay stale forever and keep DDL refused as if we had not been upgraded + // (LegacyLogProtocolMembers). At our own incarnation it is left alone: + // every current peer keeps the max version per incarnation (Update), so + // our own gossip corrects it, and a peer on an older release relays our + // state with the version stripped - refuting every such relay would climb our incarnation for as + // long as that peer runs. + stale := node.Status != NodeStatus_ALIVE && node.Incarnation >= self.Incarnation + staleVersion := node.LogProtocolVersion < self.LogProtocolVersion && node.Incarnation > self.Incarnation + if stale || staleVersion { self.Incarnation = node.Incarnation + 1 // Only set to ALIVE if we're not JOINING // JOINING nodes stay JOINING until explicitly promoted @@ -325,7 +360,7 @@ func (nr *NodeRegistry) handleSelfUpdateLocked(node *NodeState) { Uint64("refuted_incarnation", node.Incarnation). Uint64("new_incarnation", self.Incarnation). Str("current_status", self.Status.String()). - Msg("SWIM refutation: rejecting SUSPECT/DEAD claim") + Msg("SWIM refutation: rejecting a stale view of this node") } } @@ -531,6 +566,29 @@ func (nr *NodeRegistry) MemberIDs() []uint64 { return ids } +// LegacyLogProtocolMembers returns the ids of every current member (any +// status except REMOVED, self included) whose LogProtocolVersion is below +// LogPullProtocolVersion, sorted ascending. A non-empty result means at least +// one member runs an older release: DDL and CREATE/DROP DATABASE must be +// refused cluster-wide until every member reports LogPullProtocolVersion +// (coordinator.LegacyMembersDDLRefusal). +func (nr *NodeRegistry) LegacyLogProtocolMembers() []uint64 { + nr.mu.RLock() + defer nr.mu.RUnlock() + + ids := make([]uint64, 0) + for id, node := range nr.nodes { + if node.Status == NodeStatus_REMOVED { + continue + } + if node.LogProtocolVersion < LogPullProtocolVersion { + ids = append(ids, id) + } + } + slices.Sort(ids) + return ids +} + // Count returns the number of nodes in membership (excludes REMOVED and LEAVING nodes) // Used for quorum calculation to prevent split-brain. LEAVING nodes are excluded // because they are intentionally departing and should not count toward quorum. @@ -1085,69 +1143,3 @@ func (nr *NodeRegistry) QuorumInfo() (totalMembership int, aliveCount int, quoru quorumSize = (totalMembership / 2) + 1 return } - -// ======================= -// WATERMARK PROTOCOL -// ======================= - -// UpdateLocalWatermark updates the minimum applied sequence number for the local node -// This watermark is gossiped to all peers and used for GC coordination -func (nr *NodeRegistry) UpdateLocalWatermark(minSeq uint64) { - nr.mu.Lock() - defer nr.mu.Unlock() - - node, exists := nr.nodes[nr.localNodeID] - if !exists { - log.Error(). - Uint64("node_id", nr.localNodeID). - Msg("BUG: Local node not found in registry during watermark update") - return - } - - // Only update if the new watermark is higher (watermarks only advance) - if minSeq > node.MinAppliedSeq { - node.MinAppliedSeq = minSeq - log.Debug(). - Uint64("node_id", nr.localNodeID). - Uint64("watermark", minSeq). - Msg("Updated local watermark") - } -} - -// GetLocalWatermark returns the local node's watermark -func (nr *NodeRegistry) GetLocalWatermark() uint64 { - nr.mu.RLock() - defer nr.mu.RUnlock() - - if node, exists := nr.nodes[nr.localNodeID]; exists { - return node.MinAppliedSeq - } - return 0 -} - -// GetClusterMinWatermark returns the minimum watermark across all ALIVE nodes -// This is the safe point for garbage collection - transactions with seq_num below -// this value have been applied by all nodes and can be safely cleaned up -func (nr *NodeRegistry) GetClusterMinWatermark() uint64 { - nr.mu.RLock() - defer nr.mu.RUnlock() - - var minWatermark uint64 = ^uint64(0) // Max uint64 - hasAliveNodes := false - - for _, node := range nr.nodes { - if node.Status == NodeStatus_ALIVE { - hasAliveNodes = true - if node.MinAppliedSeq < minWatermark { - minWatermark = node.MinAppliedSeq - } - } - } - - // If no alive nodes (shouldn't happen, we're always alive), return 0 - if !hasAliveNodes { - return 0 - } - - return minWatermark -} diff --git a/grpc/node_registry_test.go b/grpc/node_registry_test.go index b471efdd..13ac2f74 100644 --- a/grpc/node_registry_test.go +++ b/grpc/node_registry_test.go @@ -790,72 +790,12 @@ func TestNodeRegistry_GetMembershipInfo(t *testing.T) { } // ======================= -// WATERMARK PROTOCOL TESTS +// WATERMARK FIELD GOSSIP TEST // ======================= - -func TestNodeRegistry_UpdateLocalWatermark(t *testing.T) { - nr := NewNodeRegistry(1, "localhost:8081") - - // Initial watermark should be 0 - if wm := nr.GetLocalWatermark(); wm != 0 { - t.Errorf("Expected initial watermark 0, got %d", wm) - } - - // Update watermark - nr.UpdateLocalWatermark(100) - if wm := nr.GetLocalWatermark(); wm != 100 { - t.Errorf("Expected watermark 100, got %d", wm) - } - - // Update to higher value - nr.UpdateLocalWatermark(200) - if wm := nr.GetLocalWatermark(); wm != 200 { - t.Errorf("Expected watermark 200, got %d", wm) - } - - // Update to lower value should be ignored (watermarks only advance) - nr.UpdateLocalWatermark(150) - if wm := nr.GetLocalWatermark(); wm != 200 { - t.Errorf("Watermark should not decrease, expected 200, got %d", wm) - } -} - -func TestNodeRegistry_GetClusterMinWatermark(t *testing.T) { - nr := NewNodeRegistry(1, "localhost:8081") - - // Set local watermark - nr.UpdateLocalWatermark(100) - - // Add other nodes with different watermarks - nr.Add(&NodeState{NodeId: 2, Address: "localhost:8082", Status: NodeStatus_ALIVE, MinAppliedSeq: 50}) - nr.Add(&NodeState{NodeId: 3, Address: "localhost:8083", Status: NodeStatus_ALIVE, MinAppliedSeq: 75}) - - // Minimum should be 50 (node 2) - min := nr.GetClusterMinWatermark() - if min != 50 { - t.Errorf("Expected min watermark 50, got %d", min) - } - - // Add a dead node with lower watermark - should not affect minimum - nr.Add(&NodeState{NodeId: 4, Address: "localhost:8084", Status: NodeStatus_DEAD, MinAppliedSeq: 10}) - - // Minimum should still be 50 (dead nodes excluded) - min = nr.GetClusterMinWatermark() - if min != 50 { - t.Errorf("Expected min watermark 50 (dead excluded), got %d", min) - } -} - -func TestNodeRegistry_GetClusterMinWatermark_SingleNode(t *testing.T) { - nr := NewNodeRegistry(1, "localhost:8081") - nr.UpdateLocalWatermark(500) - - // Single node cluster, min should be local watermark - min := nr.GetClusterMinWatermark() - if min != 500 { - t.Errorf("Expected min watermark 500, got %d", min) - } -} +// NodeState.MinAppliedSeq stays on the wire for compat, but the local watermark plumbing that used to read and write +// it (UpdateLocalWatermark, GetLocalWatermark, GetClusterMinWatermark) is +// removed: GC's safe position now comes from GCSafePositionFunc +// (db/transaction.go), driven by consumed positions, not this field. func TestNodeRegistry_WatermarkGossipPropagation(t *testing.T) { nr := NewNodeRegistry(1, "localhost:8081") @@ -881,12 +821,12 @@ func TestNodeRegistry_WatermarkGossipPropagation(t *testing.T) { func TestNodeRegistry_CopyNodeStateIncludesWatermark(t *testing.T) { nr := NewNodeRegistry(1, "localhost:8081") - nr.UpdateLocalWatermark(999) + nr.Add(&NodeState{NodeId: 2, Address: "localhost:8082", Status: NodeStatus_ALIVE, MinAppliedSeq: 999}) // Get returns a copy - node, _ := nr.Get(1) + node, _ := nr.Get(2) - // Verify watermark is included in copy + // Verify the wire-compat field is included in copyNodeState's copy if node.MinAppliedSeq != 999 { t.Errorf("Expected watermark 999 in copy, got %d", node.MinAppliedSeq) } @@ -1212,8 +1152,8 @@ func TestLeavingThenCrashesSuspect(t *testing.T) { } } -// TestNodeRegistry_DiscoveringAnAliveNodeConnectsToIt pins R3c-14 (reviewer -// A's F-L1): the ALIVE callback, which opens this node's connection to a +// TestNodeRegistry_DiscoveringAnAliveNodeConnectsToIt pins that the ALIVE +// callback, which opens this node's connection to a // peer, fires when gossip first tells us of a peer that is already ALIVE, not // only when a known peer turns ALIVE. A wiped node that learned of its // restarted seed as ALIVE, from another member, never saw that seed change diff --git a/grpc/replication_focused_test.go b/grpc/replication_focused_test.go index 5ee9f01b..922cb5a6 100644 --- a/grpc/replication_focused_test.go +++ b/grpc/replication_focused_test.go @@ -254,7 +254,7 @@ func TestReplicationHandlerPrepareConflictAbortRecovery(t *testing.T) { } func TestReplicationHandlerReplayFailureCanRetryWithoutAdvancingWatermark(t *testing.T) { - tmpDir, dbMgr, schemaVersionMgr := setupTestEnvironment(t, "focused_replay_failure_retry") + tmpDir, dbMgr, _ := setupTestEnvironment(t, "focused_replay_failure_retry") t.Cleanup(func() { dbMgr.Close() os.RemoveAll(tmpDir) @@ -268,14 +268,13 @@ func TestReplicationHandlerReplayFailureCanRetryWithoutAdvancingWatermark(t *tes if err != nil { t.Fatalf("GetDatabase: %v", err) } - handler := NewReplicationHandler(1, dbMgr, hlc.NewClock(1), schemaVersionMgr) clock := hlc.NewClock(2) - req := &TransactionRequest{ - TxnId: 3001, - SourceNodeId: 2, + const txnID = 3001 + ev := &ChangeEvent{ + TxnId: txnID, + OriginNodeId: 2, Database: dbName, - Phase: TransactionPhase_REPLAY, Timestamp: focusedHLC(clock), Statements: []*Statement{ focusedRowStatement(dbName, pb.StatementType_INSERT, testInsertRowChange("docs", []byte("docs:1"), map[string][]byte{ @@ -283,17 +282,22 @@ func TestReplicationHandlerReplayFailureCanRetryWithoutAdvancingWatermark(t *tes "title": mustMarshalMsgpack(t, "late table"), })), }, + RowCount: 1, } - resp, err := handler.HandleReplicateTransaction(context.Background(), req) - if err != nil { - t.Fatalf("first replay call: %v", err) - } - if resp.Success { + if _, err := ApplyPulledEvent(context.Background(), dbMgr, ev); err == nil { t.Fatal("replay against a missing table should fail") } - if rec, err := mdb.GetMetaStore().GetTransaction(req.TxnId); err == nil && rec != nil && rec.Status == db.TxnStatusCommitted { - t.Fatalf("failed replay advanced committed transaction: %+v", rec) + // The local log entry is written log-first, before the SQLite + // apply is attempted, so it is expected to exist and read COMMITTED even + // though the apply below failed; what must NOT exist yet is the + // __marmot_applied_txn marker. + applied, err := mdb.AppliedTxns([]uint64{txnID}) + if err != nil { + t.Fatalf("AppliedTxns: %v", err) + } + if applied[txnID] { + t.Fatalf("failed replay wrote the applied marker for txn %d", txnID) } if _, err := mdb.GetDB().Exec(`CREATE TABLE docs (id INTEGER PRIMARY KEY, title TEXT)`); err != nil { @@ -303,12 +307,12 @@ func TestReplicationHandlerReplayFailureCanRetryWithoutAdvancingWatermark(t *tes t.Fatalf("ReloadSchema: %v", err) } - resp, err = handler.HandleReplicateTransaction(context.Background(), req) + replayed, err := ApplyPulledEvent(context.Background(), dbMgr, ev) if err != nil { t.Fatalf("retry replay call: %v", err) } - if !resp.Success { - t.Fatalf("retry replay failed: %s", resp.ErrorMessage) + if !replayed { + t.Fatal("retry replay reported not applied") } var title string if err := mdb.GetDB().QueryRow(`SELECT title FROM docs WHERE id = 1`).Scan(&title); err != nil { @@ -317,17 +321,24 @@ func TestReplicationHandlerReplayFailureCanRetryWithoutAdvancingWatermark(t *tes if title != "late table" { t.Fatalf("replayed title=%q, want late table", title) } - rec, err := mdb.GetMetaStore().GetTransaction(req.TxnId) + rec, err := mdb.GetMetaStore().GetTransaction(txnID) if err != nil { t.Fatalf("GetTransaction after retry: %v", err) } if rec == nil || rec.Status != db.TxnStatusCommitted { t.Fatalf("replay retry did not record committed transaction: %+v", rec) } + applied, err = mdb.AppliedTxns([]uint64{txnID}) + if err != nil { + t.Fatalf("AppliedTxns after retry: %v", err) + } + if !applied[txnID] { + t.Fatalf("retry replay did not write the applied marker for txn %d", txnID) + } } func TestReplicationHandlerReplayVectorCDCFailureMarksDirtyButSucceeds(t *testing.T) { - tmpDir, dbMgr, schemaVersionMgr := setupTestEnvironment(t, "focused_replay_vector_dirty") + tmpDir, dbMgr, _ := setupTestEnvironment(t, "focused_replay_vector_dirty") t.Cleanup(func() { dbMgr.Close() os.RemoveAll(tmpDir) @@ -367,12 +378,11 @@ func TestReplicationHandlerReplayVectorCDCFailureMarksDirtyButSucceeds(t *testin t.Fatalf("ApplyVectorControl: %v", err) } - handler := NewReplicationHandler(1, dbMgr, hlc.NewClock(1), schemaVersionMgr) - req := &TransactionRequest{ - TxnId: 3101, - SourceNodeId: 2, + const txnID = 3101 + ev := &ChangeEvent{ + TxnId: txnID, + OriginNodeId: 2, Database: dbName, - Phase: TransactionPhase_REPLAY, Timestamp: focusedHLC(hlc.NewClock(2)), Statements: []*Statement{ focusedRowStatement(dbName, pb.StatementType_INSERT, testInsertRowChange("docs", []byte("docs:1"), map[string][]byte{ @@ -381,14 +391,11 @@ func TestReplicationHandlerReplayVectorCDCFailureMarksDirtyButSucceeds(t *testin "title": mustMarshalMsgpack(t, "vector row"), })), }, + RowCount: 1, } - resp, err := handler.HandleReplicateTransaction(context.Background(), req) - if err != nil { - t.Fatalf("replay call: %v", err) - } - if !resp.Success { - t.Fatalf("vector CDC failure should not fail row replay, got: %s", resp.ErrorMessage) + if _, err := ApplyPulledEvent(context.Background(), dbMgr, ev); err != nil { + t.Fatalf("vector CDC failure should not fail row replay, got: %v", err) } var title string @@ -408,7 +415,7 @@ func TestReplicationHandlerReplayVectorCDCFailureMarksDirtyButSucceeds(t *testin if status != "dirty" { t.Fatalf("vector status=%q, want dirty", status) } - rec, err := mdb.GetMetaStore().GetTransaction(req.TxnId) + rec, err := mdb.GetMetaStore().GetTransaction(txnID) if err != nil { t.Fatalf("GetTransaction: %v", err) } @@ -435,11 +442,10 @@ func BenchmarkReplicationHandlerReplayRowCDC(b *testing.B) { } defer dbMgr.Close() - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) + _, err = dbMgr.GetDatabase(db.SystemDatabaseName) if err != nil { b.Fatalf("system database: %v", err) } - schemaVersionMgr := db.NewSchemaVersionManager(systemDB.GetMetaStore()) const dbName = "bench_replay" if err := dbMgr.CreateDatabase(dbName); err != nil { @@ -456,7 +462,7 @@ func BenchmarkReplicationHandlerReplayRowCDC(b *testing.B) { b.Fatalf("ReloadSchema: %v", err) } - handler := NewReplicationHandler(1, dbMgr, clock, schemaVersionMgr) + _ = clock sourceClock := hlc.NewClock(2) ctx := context.Background() @@ -464,11 +470,10 @@ func BenchmarkReplicationHandlerReplayRowCDC(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { id := int64(i + 1) - req := &TransactionRequest{ + ev := &ChangeEvent{ TxnId: uint64(i + 1), - SourceNodeId: 2, + OriginNodeId: 2, Database: dbName, - Phase: TransactionPhase_REPLAY, Timestamp: focusedHLC(sourceClock), Statements: []*Statement{ focusedRowStatement(dbName, pb.StatementType_INSERT, testInsertRowChange("docs", []byte(fmt.Sprintf("docs:%d", id)), map[string][]byte{ @@ -477,13 +482,10 @@ func BenchmarkReplicationHandlerReplayRowCDC(b *testing.B) { "score": mustMarshalMsgpack(b, id%100), })), }, + RowCount: 1, } - resp, err := handler.HandleReplicateTransaction(ctx, req) - if err != nil { - b.Fatalf("HandleReplicateTransaction: %v", err) - } - if !resp.Success { - b.Fatalf("replay failed: %s", resp.ErrorMessage) + if _, err := ApplyPulledEvent(ctx, dbMgr, ev); err != nil { + b.Fatalf("ApplyPulledEvent: %v", err) } } } diff --git a/grpc/replication_handler.go b/grpc/replication_handler.go index e7abbb37..b5bf987c 100644 --- a/grpc/replication_handler.go +++ b/grpc/replication_handler.go @@ -5,10 +5,11 @@ import ( "fmt" "time" - "github.com/maxpert/marmot/common" + "github.com/maxpert/marmot/coordinator" "github.com/maxpert/marmot/db" "github.com/maxpert/marmot/hlc" "github.com/maxpert/marmot/protocol" + "github.com/maxpert/marmot/protocol/mysqlcode" "github.com/maxpert/marmot/telemetry" "github.com/rs/zerolog/log" ) @@ -63,8 +64,6 @@ func (rh *ReplicationHandler) HandleReplicateTransaction(ctx context.Context, re return rh.handleCommit(ctx, req) case TransactionPhase_ABORT: return rh.handleAbort(ctx, req) - case TransactionPhase_REPLAY: - return rh.handleReplay(ctx, req) default: return &TransactionResponse{ Success: false, @@ -141,6 +140,23 @@ func (rh *ReplicationHandler) handlePrepare(ctx context.Context, req *Transactio statements = append(statements, internalStmt) } + // Rolling-upgrade gate, participant half: refuse a PREPARE carrying DDL or + // a CREATE/DROP DATABASE while any member does not serve the commit-log + // pull protocol yet, so a coordinator running an older release, which has + // no gate of its own, cannot reach a quorum for DDL either + // (coordinator.LegacyMembersDDLRefusal). + if rh.registry != nil && statementsCarryDDLOrDatabaseOp(statements) { + if legacy := rh.registry.LegacyLogProtocolMembers(); len(legacy) > 0 { + telemetry.ReplicationRequestsTotal.With("prepare", "failed").Inc() + return &TransactionResponse{ + Success: false, + Rejected: true, + ErrorMessage: coordinator.NewLegacyMembersDDLRefusal(legacy).Error(), + ErrorCode: uint32(mysqlcode.ErrCodeDeadlock), + }, nil + } + } + // Build engine request startTS := hlc.Timestamp{ WallTime: req.Timestamp.WallTime, @@ -181,6 +197,19 @@ func (rh *ReplicationHandler) handlePrepare(ctx context.Context, req *Transactio return resp, nil } +// statementsCarryDDLOrDatabaseOp reports whether any statement is DDL or a +// CREATE/DROP DATABASE, matching the isDDL classification in +// coordinator.CoordinatorHandler.handleMutation. +func statementsCarryDDLOrDatabaseOp(statements []protocol.Statement) bool { + for _, stmt := range statements { + switch stmt.Type { + case protocol.StatementDDL, protocol.StatementCreateDatabase, protocol.StatementDropDatabase: + return true + } + } + return false +} + // handleCommit processes Phase 2 of 2PC: Commit transaction func (rh *ReplicationHandler) handleCommit(ctx context.Context, req *TransactionRequest) (*TransactionResponse, error) { commitStart := time.Now() @@ -219,14 +248,10 @@ func (rh *ReplicationHandler) handleCommit(ctx context.Context, req *Transaction }, nil } - // Schema version increment for DDL transactions (MUST happen after successful commit) - // DDL information is returned by the engine (extracted from committed intents) - if rh.schemaVersionMgr != nil && result.DDLSQL != "" { - _, verErr := rh.schemaVersionMgr.IncrementSchemaVersion(req.Database, result.DDLSQL, req.TxnId) - if verErr != nil { - log.Error().Err(verErr).Str("database", req.Database).Uint64("txn_id", req.TxnId).Msg("Failed to increment schema version after DDL replication") - } - } + // The schema version bump for a DDL transaction is already durable, atomic + // with the DDL itself, in the database's own SQLite file: the + // engine's commit path bumped and cached it as part of committing above. + // There is nothing left to do here. telemetry.ReplicationRequestsTotal.With("commit", "success").Inc() return &TransactionResponse{ @@ -293,286 +318,6 @@ func (rh *ReplicationHandler) handleAbort(ctx context.Context, req *TransactionR }, nil } -// handleReplay processes anti-entropy replay: Apply already-committed transactions directly. -// This bypasses 2PC state tracking since these transactions are already committed on the source. -// Used by delta sync to repair divergent nodes without requiring PREPARE phase. -func (rh *ReplicationHandler) handleReplay(ctx context.Context, req *TransactionRequest) (*TransactionResponse, error) { - replayStart := time.Now() - defer func() { - telemetry.ReplicaReplaySeconds.Observe(time.Since(replayStart).Seconds()) - }() - - log.Debug(). - Uint64("node_id", rh.nodeID). - Str("database", req.Database). - Uint64("txn_id", req.TxnId). - Int("num_statements", len(req.Statements)). - Msg("handleReplay called - applying already-committed transaction") - - // Get the target database from request (database name is required) - dbName := req.Database - if dbName == "" { - telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() - return &TransactionResponse{ - Success: false, - ErrorMessage: "database name is required in replay request", - }, nil - } - - // Get database instance - dbInstance, err := rh.dbMgr.GetDatabase(dbName) - if err != nil { - telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() - return &TransactionResponse{ - Success: false, - ErrorMessage: fmt.Sprintf("database %s not found: %v", dbName, err), - }, nil - } - if len(req.Statements) == 1 { - if change := req.Statements[0].GetVectorIndexChange(); change != nil { - if err := rh.applyVectorIndexChange(ctx, vectorChangeFromProto(change)); err != nil { - telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() - return &TransactionResponse{ - Success: false, - ErrorMessage: fmt.Sprintf("failed to apply vector index control: %v", err), - }, nil - } - if _, err := StoreAppliedChangeEvent(dbInstance.GetMetaStore(), req.TxnId, req.Timestamp, dbName, req.Statements); err != nil { - telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() - return &TransactionResponse{ - Success: false, - ErrorMessage: fmt.Sprintf("failed to store vector index control rows: %v", err), - }, nil - } - now := rh.clock.Now() - telemetry.ReplicationRequestsTotal.With("replay", "success").Inc() - return &TransactionResponse{ - Success: true, - AppliedAt: &HLC{ - WallTime: now.WallTime, - Logical: now.Logical, - NodeId: rh.nodeID, - }, - }, nil - } - } - - sqliteDB := dbInstance.GetDB() - - // Execute statements directly in a SQLite transaction - tx, err := sqliteDB.BeginTx(ctx, nil) - if err != nil { - telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() - return &TransactionResponse{ - Success: false, - ErrorMessage: fmt.Sprintf("failed to begin transaction: %v", err), - }, nil - } - defer tx.Rollback() - - // Create schema adapter for CDC operations - schemaAdapter := &replicationSchemaAdapter{dbMgr: rh.dbMgr, dbName: dbName} - var schemaChange db.SchemaChange - - for _, stmt := range req.Statements { - // Check for CDC data (RowChange payload) - if rowChange := stmt.GetRowChange(); rowChange != nil && len(rowChange.EncodedRow) > 0 { - // CDC path: apply row data directly using unified applier - opType, opErr := wireDMLToOp(stmt.Type) - if opErr != nil { - telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() - return &TransactionResponse{ - Success: false, - ErrorMessage: opErr.Error(), - }, nil - } - row, rowErr := decodeRowChange(stmt) - if rowErr != nil { - telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() - return &TransactionResponse{Success: false, ErrorMessage: rowErr.Error()}, nil - } - if err := db.ApplyCDCValues(tx, schemaAdapter, opType, stmt.TableName, row.OldValues, row.NewValues); err != nil { - telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() - return &TransactionResponse{ - Success: false, - ErrorMessage: fmt.Sprintf("failed to apply CDC statement: %v", err), - }, nil - } - continue - } - - // DDL path: execute SQL directly - if ddl := stmt.GetDdlChange(); ddl != nil && ddl.Sql != "" { - if err := dbInstance.ApplyReplayedDDL(ctx, tx, ddl.Sql, stmt.TableName, req.SourceNodeId, &schemaChange); err != nil { - telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() - return &TransactionResponse{ - Success: false, - ErrorMessage: fmt.Sprintf("failed to execute DDL: %v", err), - }, nil - } - continue - } - - // LOAD DATA path: apply via shared bulk-load executor. - if loadData := stmt.GetLoadDataChange(); loadData != nil { - if _, err := db.ApplyLoadDataInTx(tx, loadData.Sql, loadData.Data); err != nil { - telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() - return &TransactionResponse{ - Success: false, - ErrorMessage: fmt.Sprintf("failed to apply LOAD DATA: %v", err), - }, nil - } - continue - } - - // No CDC data and no DDL - this shouldn't happen for replay - log.Warn(). - Str("table", stmt.TableName). - Int32("type", int32(stmt.Type)). - Msg("handleReplay: statement has no CDC data or DDL") - } - - // Replayed DDL changes table incarnations exactly as a committed one does. - if err := dbInstance.FinishReplayedSchemaChange(tx, &schemaChange); err != nil { - telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() - return &TransactionResponse{ - Success: false, - ErrorMessage: fmt.Sprintf("failed to finish replayed DDL: %v", err), - }, nil - } - - // Commit the transaction - if err := db.MarkSQLiteTxnApplied(tx, req.TxnId, HLCToTimestamp(req.Timestamp)); err != nil { - telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() - return &TransactionResponse{ - Success: false, - ErrorMessage: fmt.Sprintf("failed to mark applied txn: %v", err), - }, nil - } - if err := tx.Commit(); err != nil { - telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() - return &TransactionResponse{ - Success: false, - ErrorMessage: fmt.Sprintf("failed to commit: %v", err), - }, nil - } - if !schemaChange.Empty() { - if err := dbInstance.ReloadSchema(); err != nil { - log.Warn().Err(err).Str("database", dbName).Msg("handleReplay: failed to reload schema after DDL") - } - } - - // Store TransactionRecord in MetaStore so GetCommittedTxnCount/GetMaxTxnID return correct values. - // Without this, anti-entropy keeps thinking we're behind because these metrics read from PebbleDB. - seqNum, err := StoreAppliedChangeEvent(dbInstance.GetMetaStore(), req.TxnId, req.Timestamp, dbName, req.Statements) - if err != nil { - telemetry.ReplicationRequestsTotal.With("replay", "failed").Inc() - return &TransactionResponse{ - Success: false, - ErrorMessage: fmt.Sprintf("failed to store replay captured rows: %v", err), - }, nil - } - if err := rh.applyVectorCDCFromStatements(ctx, dbName, req.TxnId, seqNum, req.Statements); err != nil { - log.Error(). - Err(err). - Str("database", dbName). - Uint64("txn_id", req.TxnId). - Uint64("seq_num", seqNum). - Msg("handleReplay: vector CDC failed after row commit; local vector index is dirty") - } - - log.Debug(). - Uint64("txn_id", req.TxnId). - Str("database", dbName). - Int("statements", len(req.Statements)). - Msg("handleReplay: transaction applied successfully") - - telemetry.ReplicationRequestsTotal.With("replay", "success").Inc() - now := rh.clock.Now() - return &TransactionResponse{ - Success: true, - AppliedAt: &HLC{ - WallTime: now.WallTime, - Logical: now.Logical, - NodeId: rh.nodeID, - }, - }, nil -} - -func (rh *ReplicationHandler) applyVectorIndexChange(ctx context.Context, change common.VectorIndexChange) error { - vecMgr := rh.dbMgr.GetVectorIndexManager() - if vecMgr == nil { - return fmt.Errorf("vector index manager not configured") - } - applier, ok := vecMgr.(interface { - ApplyVectorControl(context.Context, common.VectorIndexChange) error - }) - if !ok { - return fmt.Errorf("vector index manager cannot apply replicated control metadata") - } - return applier.ApplyVectorControl(ctx, change) -} - -func (rh *ReplicationHandler) applyVectorCDCFromStatements(ctx context.Context, database string, txnID, seqNum uint64, statements []*Statement) error { - entries := make([]common.CDCEntry, 0, len(statements)) - for _, stmt := range statements { - rowChange := stmt.GetRowChange() - if rowChange == nil || len(rowChange.EncodedRow) == 0 { - continue - } - row, err := decodeRowChange(stmt) - if err != nil { - return err - } - entries = append(entries, common.CDCEntry{ - Table: stmt.TableName, - IntentKey: row.IntentKey, - Operation: row.Op, - OldValues: row.OldValues, - NewValues: row.NewValues, - EncodedRow: rowChange.EncodedRow, - EncodedCodec: rowChange.EncodedRowCodec, - CommitTxnID: txnID, - CommitSeqNum: seqNum, - }) - } - if len(entries) == 0 { - return nil - } - vecMgr := rh.dbMgr.GetVectorIndexManager() - if vecMgr == nil { - return nil - } - applier, ok := vecMgr.(interface { - ApplyCommittedVectorCDC(context.Context, string, uint64, uint64, []common.CDCEntry) error - }) - if !ok { - return nil - } - if err := applier.ApplyCommittedVectorCDC(ctx, database, txnID, seqNum, entries); err != nil { - return fmt.Errorf("handleReplay: apply vector CDC: %w", err) - } - return nil -} - -// replicationSchemaAdapter adapts DatabaseManager schema access to CDCSchemaProvider -type replicationSchemaAdapter struct { - dbMgr *db.DatabaseManager - dbName string -} - -func (a *replicationSchemaAdapter) GetPrimaryKeys(tableName string) ([]string, error) { - dbInstance, err := a.dbMgr.GetDatabase(a.dbName) - if err != nil { - return nil, fmt.Errorf("database %s not found: %w", a.dbName, err) - } - schema, err := dbInstance.GetCachedTableSchema(tableName) - if err != nil { - return nil, fmt.Errorf("schema not found for table %s: %w", tableName, err) - } - return schema.PrimaryKeys, nil -} - // HandleRead handles incoming read requests with MVCC snapshot isolation func (rh *ReplicationHandler) HandleRead(ctx context.Context, req *ReadRequest) (*ReadResponse, error) { // Update local clock with incoming timestamp diff --git a/grpc/replication_handler_test.go b/grpc/replication_handler_test.go index b271d79d..1903e332 100644 --- a/grpc/replication_handler_test.go +++ b/grpc/replication_handler_test.go @@ -35,20 +35,12 @@ func TestSchemaVersionRejection(t *testing.T) { t.Fatalf("Failed to create test database: %v", err) } - // Get system database (already created by DatabaseManager) - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) - if err != nil { - t.Fatalf("Failed to get system database: %v", err) - } - schemaVersionMgr := db.NewSchemaVersionManager(systemDB.GetMetaStore()) + schemaVersionMgr := db.NewSchemaVersionManager(dbMgr) handler := NewReplicationHandler(1, dbMgr, clock, schemaVersionMgr) // Set local schema version to 5 - err = schemaVersionMgr.SetSchemaVersion(testDB, 5, "CREATE TABLE test (id INT)", 100) - if err != nil { - t.Fatalf("Failed to set schema version: %v", err) - } + bumpSchemaVersionForTest(t, dbMgr, testDB, 5) // Transaction with RequiredSchemaVersion = 10 should be rejected (10 > 5) req := &TransactionRequest{ @@ -107,11 +99,10 @@ func TestReplicationHandler_ReplayDDLIdempotent(t *testing.T) { t.Fatalf("Failed to create test database: %v", err) } - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) + _, err = dbMgr.GetDatabase(db.SystemDatabaseName) if err != nil { t.Fatalf("Failed to get system database: %v", err) } - handler := NewReplicationHandler(1, dbMgr, clock, db.NewSchemaVersionManager(systemDB.GetMetaStore())) ddlStmt := &Statement{ Type: pb.StatementType_DDL, @@ -124,25 +115,21 @@ func TestReplicationHandler_ReplayDDLIdempotent(t *testing.T) { } for i := 0; i < 2; i++ { - req := &TransactionRequest{ + ev := &ChangeEvent{ TxnId: uint64(100 + i), - SourceNodeId: 2, + OriginNodeId: 2, Database: testDB, - Phase: TransactionPhase_REPLAY, Timestamp: &HLC{ WallTime: clock.Now().WallTime, Logical: clock.Now().Logical, NodeId: 2, }, Statements: []*Statement{ddlStmt}, + RowCount: 1, } - resp, err := handler.HandleReplicateTransaction(context.Background(), req) - if err != nil { - t.Fatalf("Replay DDL call %d failed: %v", i+1, err) - } - if !resp.Success { - t.Fatalf("Replay DDL call %d should be idempotent, got error: %s", i+1, resp.ErrorMessage) + if _, err := ApplyPulledEvent(context.Background(), dbMgr, ev); err != nil { + t.Fatalf("Replay DDL call %d should be idempotent, got error: %v", i+1, err) } } } @@ -166,17 +153,15 @@ func TestReplicationHandler_ReplayReloadsSchemaAfterDDL(t *testing.T) { t.Fatalf("Failed to create test database: %v", err) } - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) + _, err = dbMgr.GetDatabase(db.SystemDatabaseName) if err != nil { t.Fatalf("Failed to get system database: %v", err) } - handler := NewReplicationHandler(1, dbMgr, clock, db.NewSchemaVersionManager(systemDB.GetMetaStore())) - ddlReq := &TransactionRequest{ + ddlEvent := &ChangeEvent{ TxnId: 200, - SourceNodeId: 2, + OriginNodeId: 2, Database: testDB, - Phase: TransactionPhase_REPLAY, Timestamp: &HLC{ WallTime: clock.Now().WallTime, Logical: clock.Now().Logical, @@ -193,20 +178,16 @@ func TestReplicationHandler_ReplayReloadsSchemaAfterDDL(t *testing.T) { }, }, }, + RowCount: 1, } - resp, err := handler.HandleReplicateTransaction(context.Background(), ddlReq) - if err != nil { + if _, err := ApplyPulledEvent(context.Background(), dbMgr, ddlEvent); err != nil { t.Fatalf("Replay DDL failed: %v", err) } - if !resp.Success { - t.Fatalf("Replay DDL failed with response error: %s", resp.ErrorMessage) - } - insertReq := &TransactionRequest{ + insertEvent := &ChangeEvent{ TxnId: 201, - SourceNodeId: 2, + OriginNodeId: 2, Database: testDB, - Phase: TransactionPhase_REPLAY, Timestamp: &HLC{ WallTime: clock.Now().WallTime, Logical: clock.Now().Logical, @@ -225,20 +206,16 @@ func TestReplicationHandler_ReplayReloadsSchemaAfterDDL(t *testing.T) { }, }, }, + RowCount: 1, } - resp, err = handler.HandleReplicateTransaction(context.Background(), insertReq) - if err != nil { + if _, err := ApplyPulledEvent(context.Background(), dbMgr, insertEvent); err != nil { t.Fatalf("Replay insert failed: %v", err) } - if !resp.Success { - t.Fatalf("Replay insert failed with response error: %s", resp.ErrorMessage) - } - updateReq := &TransactionRequest{ + updateEvent := &ChangeEvent{ TxnId: 202, - SourceNodeId: 2, + OriginNodeId: 2, Database: testDB, - Phase: TransactionPhase_REPLAY, Timestamp: &HLC{ WallTime: clock.Now().WallTime, Logical: clock.Now().Logical, @@ -260,14 +237,11 @@ func TestReplicationHandler_ReplayReloadsSchemaAfterDDL(t *testing.T) { }, }, }, + RowCount: 1, } - resp, err = handler.HandleReplicateTransaction(context.Background(), updateReq) - if err != nil { + if _, err := ApplyPulledEvent(context.Background(), dbMgr, updateEvent); err != nil { t.Fatalf("Replay update failed: %v", err) } - if !resp.Success { - t.Fatalf("Replay update failed with response error: %s", resp.ErrorMessage) - } } func mustMarshalMsgpack(t testing.TB, v interface{}) []byte { @@ -311,12 +285,12 @@ func newLeavingHandlerFixture(t *testing.T) (*ReplicationHandler, *NodeRegistry, t.Fatalf("Failed to reload test schema: %v", err) } - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) + _, err = dbMgr.GetDatabase(db.SystemDatabaseName) if err != nil { t.Fatalf("Failed to get system database: %v", err) } - handler := NewReplicationHandler(localNodeID, dbMgr, clock, db.NewSchemaVersionManager(systemDB.GetMetaStore())) + handler := NewReplicationHandler(localNodeID, dbMgr, clock, db.NewSchemaVersionManager(dbMgr)) registry := NewNodeRegistry(localNodeID, "localhost:9000") handler.SetRegistry(registry) @@ -505,7 +479,7 @@ func (r *incarnationRecorder) take() []string { return tables } -// TestReplicationHandler_ReplayedDDLEndsIncarnations pins R3c-10 end to end +// TestReplicationHandler_ReplayedDDLEndsIncarnations checks end to end, // on the anti-entropy replay RPC: a DDL a node catches up on through REPLAY // ends table incarnations, inherits and seeds claim bases exactly as the // same DDL's 2PC COMMIT would. @@ -519,20 +493,18 @@ func TestReplicationHandler_ReplayedDDLEndsIncarnations(t *testing.T) { defer dbMgr.Close() const testDB = "replay_incarnations" require.NoError(t, dbMgr.CreateDatabase(testDB)) - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) + _, err = dbMgr.GetDatabase(db.SystemDatabaseName) require.NoError(t, err) - handler := NewReplicationHandler(1, dbMgr, clock, db.NewSchemaVersionManager(systemDB.GetMetaStore())) rec := &incarnationRecorder{} dbMgr.SetAutoIncIncarnationListener(rec) replay := func(txnID uint64, table, ddl string) { t.Helper() now := clock.Now() - resp, err := handler.HandleReplicateTransaction(context.Background(), &TransactionRequest{ + _, err := ApplyPulledEvent(context.Background(), dbMgr, &ChangeEvent{ TxnId: txnID, - SourceNodeId: 2, + OriginNodeId: 2, Database: testDB, - Phase: TransactionPhase_REPLAY, Timestamp: &HLC{WallTime: now.WallTime, Logical: now.Logical, NodeId: 2}, Statements: []*Statement{{ Type: pb.StatementType_DDL, @@ -540,9 +512,9 @@ func TestReplicationHandler_ReplayedDDLEndsIncarnations(t *testing.T) { Database: testDB, Payload: &Statement_DdlChange{DdlChange: &DDLChange{Sql: ddl}}, }}, + RowCount: 1, }) - require.NoError(t, err) - require.True(t, resp.Success, "replay of %q: %s", ddl, resp.ErrorMessage) + require.NoError(t, err, "replay of %q", ddl) } claims := db.NewAutoIncClaimStore(dbMgr.GetSystemDatabase()) diff --git a/grpc/replicator_adapter.go b/grpc/replicator_adapter.go index 1901911f..d3bee2af 100644 --- a/grpc/replicator_adapter.go +++ b/grpc/replicator_adapter.go @@ -112,6 +112,7 @@ func convertStatementsToProto(stmts []protocol.Statement, database string, txnID protoStmt.AutoIdClaim = stmt.AutoIDClaim protoStmt.AutoIdClaimPayload = stmt.AutoIDClaimPayload + protoStmt.DatabaseGeneration = stmt.DatabaseGeneration isDML := stmt.Type == protocol.StatementInsert || stmt.Type == protocol.StatementUpdate || diff --git a/grpc/replicator_adapter_test.go b/grpc/replicator_adapter_test.go index e58438d5..853e8672 100644 --- a/grpc/replicator_adapter_test.go +++ b/grpc/replicator_adapter_test.go @@ -53,9 +53,9 @@ func TestGRPCReplicator_PrepareRejectionCarriesOverRealGRPCConnection(t *testing _, err = dbInstance.GetDB().Exec(`CREATE TABLE groups (group_id INTEGER PRIMARY KEY, creation_date datetime)`) require.NoError(t, err) - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) + _, err = dbMgr.GetDatabase(db.SystemDatabaseName) require.NoError(t, err) - handler := NewReplicationHandler(1, dbMgr, clock, db.NewSchemaVersionManager(systemDB.GetMetaStore())) + handler := NewReplicationHandler(1, dbMgr, clock, db.NewSchemaVersionManager(dbMgr)) listener, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) diff --git a/grpc/rolling_upgrade_prepare_test.go b/grpc/rolling_upgrade_prepare_test.go new file mode 100644 index 00000000..c42e97cf --- /dev/null +++ b/grpc/rolling_upgrade_prepare_test.go @@ -0,0 +1,115 @@ +package grpc + +import ( + "context" + "os" + "testing" + + "github.com/maxpert/marmot/coordinator" + "github.com/maxpert/marmot/db" + pb "github.com/maxpert/marmot/grpc/common" + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol/mysqlcode" + "github.com/stretchr/testify/require" +) + +// TestHandlePrepare_RefusesDDLWhileALegacyMemberIsPresent proves the +// participant half of the rolling-upgrade DDL gate: a PREPARE carrying DDL +// (or CREATE/DROP DATABASE) must be refused, with the retryable 1213, while +// any cluster member does not serve the commit-log pull protocol yet. The +// coordinator-side gate alone would not stop a coordinator running an older +// release, which has no such gate. +func TestHandlePrepare_RefusesDDLWhileALegacyMemberIsPresent(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "marmot_test_upgrade_prepare") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + clock := hlc.NewClock(1) + dbMgr, err := db.NewDatabaseManager(tmpDir, 1, clock) + require.NoError(t, err) + defer dbMgr.Close() + + testDB := "test_db" + require.NoError(t, dbMgr.CreateDatabase(testDB)) + + schemaVersionMgr := db.NewSchemaVersionManager(dbMgr) + handler := NewReplicationHandler(1, dbMgr, clock, schemaVersionMgr) + + registry := NewNodeRegistry(1, "localhost:8081") + registry.Add(&NodeState{NodeId: 2, Address: "localhost:8082", Status: NodeStatus_ALIVE, LogProtocolVersion: 0}) // an older release + handler.SetRegistry(registry) + + req := &TransactionRequest{ + TxnId: 9, + SourceNodeId: 2, + Database: testDB, + Phase: TransactionPhase_PREPARE, + Timestamp: &HLC{ + WallTime: clock.Now().WallTime, + Logical: clock.Now().Logical, + NodeId: 2, + }, + Statements: []*Statement{ + { + Type: pb.StatementType_DDL, + TableName: "t", + Database: testDB, + Payload: &Statement_DdlChange{DdlChange: &DDLChange{Sql: "CREATE TABLE t (id INTEGER PRIMARY KEY)"}}, + }, + }, + } + + resp, err := handler.HandleReplicateTransaction(context.Background(), req) + require.NoError(t, err, "refusal is a response field, not a transport error") + require.False(t, resp.Success, "PREPARE carrying DDL must be refused while a legacy member is present") + require.True(t, resp.Rejected) + require.Equal(t, uint32(mysqlcode.ErrCodeDeadlock), resp.ErrorCode) + require.Equal(t, coordinator.NewLegacyMembersDDLRefusal([]uint64{2}).Error(), resp.ErrorMessage) +} + +// TestHandlePrepare_AllowsDDLWhenEveryMemberServesLogPull proves the gate does not +// misfire once every member reports LogPullProtocolVersion. +func TestHandlePrepare_AllowsDDLWhenEveryMemberServesLogPull(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "marmot_test_upgrade_prepare_ok") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + clock := hlc.NewClock(1) + dbMgr, err := db.NewDatabaseManager(tmpDir, 1, clock) + require.NoError(t, err) + defer dbMgr.Close() + + testDB := "test_db" + require.NoError(t, dbMgr.CreateDatabase(testDB)) + + schemaVersionMgr := db.NewSchemaVersionManager(dbMgr) + handler := NewReplicationHandler(1, dbMgr, clock, schemaVersionMgr) + + registry := NewNodeRegistry(1, "localhost:8081") + registry.Add(&NodeState{NodeId: 2, Address: "localhost:8082", Status: NodeStatus_ALIVE, LogProtocolVersion: LogPullProtocolVersion}) + handler.SetRegistry(registry) + + req := &TransactionRequest{ + TxnId: 10, + SourceNodeId: 2, + Database: testDB, + Phase: TransactionPhase_PREPARE, + Timestamp: &HLC{ + WallTime: clock.Now().WallTime, + Logical: clock.Now().Logical, + NodeId: 2, + }, + Statements: []*Statement{ + { + Type: pb.StatementType_DDL, + TableName: "t", + Database: testDB, + Payload: &Statement_DdlChange{DdlChange: &DDLChange{Sql: "CREATE TABLE t (id INTEGER PRIMARY KEY)"}}, + }, + }, + } + + resp, err := handler.HandleReplicateTransaction(context.Background(), req) + require.NoError(t, err) + require.True(t, resp.Success, "PREPARE carrying DDL must succeed once every member reports LogPullProtocolVersion: %s", resp.ErrorMessage) +} diff --git a/grpc/server.go b/grpc/server.go index 0ae6e75d..83cc83d7 100644 --- a/grpc/server.go +++ b/grpc/server.go @@ -15,7 +15,6 @@ import ( "time" "github.com/maxpert/marmot/cfg" - "github.com/maxpert/marmot/common" "github.com/maxpert/marmot/coordinator" "github.com/maxpert/marmot/db" "github.com/rs/zerolog/log" @@ -55,6 +54,11 @@ type Server struct { // CDC signal-based change streaming cdcSubscriber db.CDCSubscriber + // antiEntropy reports each local database's caught-up/stuck status. + // nil until SetAntiEntropy is called + // (marmot.go, once AntiEntropyService is constructed). + antiEntropy *AntiEntropyService + // Per-database snapshot exports served within snapshot_cache_ttl_seconds snapshotExports snapshotExportCache @@ -451,97 +455,27 @@ func (s *Server) Read(ctx context.Context, req *ReadRequest) (*ReadResponse, err return &ReadResponse{}, nil } -// sendChangeEvent converts a TransactionRecord to ChangeEvent and sends it on the stream. -// Handles both CDC (Change Data Capture) path with row data and DDL path with SQL statements. -// Reads CDC entries from MetaStore for the transaction and converts them to Statement objects. +// sendChangeEvent converts a TransactionRecord to ChangeEvent and sends it on +// the stream, using the same captured-row-to-Statement conversion +// FetchTransactions shares (statementFromCapturedRow, grpc/log_server.go). +// StreamChanges keeps its existing lenient semantics for read replicas: an +// iterate or decode error is logged and that row is skipped +// rather than failing the whole stream (lenientStatementsFromLog) - unlike +// FetchTransactions, which fails the call outright. func (s *Server) sendChangeEvent(rec *db.TransactionRecord, metaStore db.MetaStore, stream MarmotService_StreamChangesServer) error { - // Build statements from CDC entries in MetaStore - var statements []*Statement - - cursor, err := metaStore.IterateCapturedRows(rec.TxnID) - if err != nil { - log.Warn().Err(err).Uint64("txn_id", rec.TxnID).Msg("Failed to iterate captured rows for streaming") - } else { - defer cursor.Close() - for cursor.Next() { - _, data := cursor.Row() - row, err := db.DecodeRow(data) - if err != nil { - log.Warn().Err(err).Uint64("txn_id", rec.TxnID).Msg("Failed to decode captured row") - continue - } - - // Convert OpType to wire StatementType - stmtCode := db.OpTypeToStatementType(db.OpType(row.Op)) - wireType := common.MustToWireType(stmtCode) - - var stmt *Statement - switch db.OpType(row.Op) { - case db.OpTypeVectorIndex: - if row.VectorIndexChange == nil { - log.Warn().Uint64("txn_id", rec.TxnID).Msg("Vector index CDC row missing payload") - continue - } - stmt = &Statement{ - Type: common.MustToWireType(common.StatementVectorIndexControl), - TableName: row.Table, - Database: rec.DatabaseName, - Payload: &Statement_VectorIndexChange{ - VectorIndexChange: vectorChangeToProto(*row.VectorIndexChange), - }, - } - case db.OpTypeDDL: - // DDL statement - use DDLChange payload - stmt = &Statement{ - Type: wireType, - TableName: row.Table, - Database: rec.DatabaseName, - Payload: &Statement_DdlChange{ - DdlChange: &DDLChange{ - Sql: row.DDLSQL, - }, - }, - } - case db.OpTypeLoadData: - stmt = &Statement{ - Type: wireType, - TableName: row.Table, - Database: rec.DatabaseName, - Payload: &Statement_LoadDataChange{ - LoadDataChange: &LoadDataChange{ - Sql: row.LoadSQL, - Data: row.LoadData, - }, - }, - } - default: - // DML statement - use RowChange payload - stmt = &Statement{ - Type: wireType, - TableName: row.Table, - Database: rec.DatabaseName, - Payload: &Statement_RowChange{ - RowChange: &RowChange{ - EncodedRow: append([]byte(nil), data...), - EncodedRowCodec: db.EncodedCapturedRowCodecMsgpack(), - }, - }, - } - } - statements = append(statements, stmt) - } - } - event := &ChangeEvent{ TxnId: rec.TxnID, SeqNum: rec.SeqNum, Timestamp: &HLC{ WallTime: rec.CommitTSWall, Logical: rec.CommitTSLogical, + NodeId: rec.NodeID, }, - Statements: statements, + Statements: lenientStatementsFromLog(rec, metaStore), Database: rec.DatabaseName, RequiredSchemaVersion: rec.RequiredSchemaVersion, + OriginNodeId: rec.NodeID, + RowCount: rec.RowCount, } return stream.Send(event) @@ -736,11 +670,25 @@ func (s *Server) streamHistoricalTransactions( return nil } -// GetReplicationState returns current replication state for anti-entropy -// Returns per-database replication progress for the requesting peer +// GetReplicationState is kept only for rolling upgrades: a peer on a release +// before the commit-log pull protocol still calls it, during its own +// anti-entropy negotiation. Its wire shape is frozen - no field is ever added +// to it again - so it reports the log-pull replication state mapped into the +// old fields rather than the removed MetaStore.ReplicationState table: +// - LastAppliedTxnId is req.RequestingNodeId's last-consumed position in +// this database's log (MetaStore.ConsumedPositions), i.e. how far that +// peer has pulled from this node - the closest equivalent of "last +// applied txn id from this peer". +// - SyncStatus is derived from AntiEntropyService: STUCK if the database +// has any stuck transaction (LogPuller.StuckTxns), SYNCED if +// AntiEntropyService.CaughtUp, CATCHING_UP otherwise. +// - LastAppliedTimestamp has no equivalent (log positions are (seq, +// txnID), not HLC) and is always the zero timestamp. +// - LastSyncTime has no equivalent and is always zero. func (s *Server) GetReplicationState(ctx context.Context, req *ReplicationStateRequest) (*ReplicationStateResponse, error) { s.mu.RLock() dbManager := s.dbManager + antiEntropy := s.antiEntropy s.mu.RUnlock() if dbManager == nil { @@ -752,80 +700,68 @@ func (s *Server) GetReplicationState(ctx context.Context, req *ReplicationStateR Str("database_filter", req.Database). Msg("Replication state requested") - var states []*DatabaseReplicationState - - // Get list of databases to query var databases []string if req.Database != "" { - // Specific database requested databases = []string{req.Database} } else { - // All databases databases = dbManager.ListDatabases() } - // Query replication state for each database + states := make([]*DatabaseReplicationState, 0, len(databases)) for _, dbName := range databases { - // Get last applied txn_id from replication_state table for this peer - repState, err := dbManager.GetReplicationState(req.RequestingNodeId, dbName) - var lastAppliedTxnID uint64 - var lastAppliedTS *HLC - var lastSyncTime int64 - var syncStatus string - - if err != nil || repState == nil { - // No replication state yet for this peer/database - use defaults - lastAppliedTxnID = 0 - lastAppliedTS = &HLC{WallTime: 0, Logical: 0, NodeId: s.nodeID} - lastSyncTime = 0 - syncStatus = "SYNCED" - } else { - lastAppliedTxnID = repState.LastAppliedTxnID - lastAppliedTS = &HLC{ - WallTime: repState.LastAppliedTSWall, - Logical: repState.LastAppliedTSLog, - NodeId: s.nodeID, - } - lastSyncTime = repState.LastSyncTime - syncStatus = repState.SyncStatus - } + states = append(states, s.legacyReplicationStateFor(dbName, req.RequestingNodeId, dbManager, antiEntropy)) + } - // Get current max txn_id in this database - maxTxnID, err := dbManager.GetMaxTxnID(dbName) - if err != nil { - log.Warn().Err(err).Str("database", dbName).Msg("Failed to get max txn_id") - maxTxnID = 0 - } + return &ReplicationStateResponse{ + States: states, + }, nil +} - // Get committed transaction count for data completeness comparison - txnCount, err := dbManager.GetCommittedTxnCount(dbName) - if err != nil { - log.Warn().Err(err).Str("database", dbName).Msg("Failed to get committed txn count") - txnCount = 0 +// legacyReplicationStateFor builds one database's entry for the +// GetReplicationState shim (see its doc comment). +func (s *Server) legacyReplicationStateFor(dbName string, requestingNodeID uint64, dbManager *db.DatabaseManager, antiEntropy *AntiEntropyService) *DatabaseReplicationState { + var lastAppliedTxnID uint64 + if mdb, err := dbManager.GetDatabase(dbName); err == nil { + if metaStore := mdb.GetMetaStore(); metaStore != nil { + if positions, err := metaStore.ConsumedPositions(); err == nil { + lastAppliedTxnID = positions[requestingNodeID].TxnID + } } + } - // Get max sequence number for gap detection - maxSeqNum, err := dbManager.GetMaxSeqNum(dbName) - if err != nil { - log.Warn().Err(err).Str("database", dbName).Msg("Failed to get max seq_num") - maxSeqNum = 0 + syncStatus := "CATCHING_UP" + if antiEntropy != nil { + switch { + case antiEntropy.StuckTxnCount(dbName) > 0: + syncStatus = "STUCK" + case antiEntropy.CaughtUp(dbName): + syncStatus = "SYNCED" } + } - states = append(states, &DatabaseReplicationState{ - DatabaseName: dbName, - LastAppliedTxnId: lastAppliedTxnID, - LastAppliedTimestamp: lastAppliedTS, - LastSyncTime: lastSyncTime, - SyncStatus: syncStatus, - CurrentMaxTxnId: maxTxnID, - CommittedTxnCount: txnCount, - MaxSeqNum: maxSeqNum, - }) + maxTxnID, err := dbManager.GetMaxTxnID(dbName) + if err != nil { + log.Warn().Err(err).Str("database", dbName).Msg("Failed to get max txn_id") + } + txnCount, err := dbManager.GetCommittedTxnCount(dbName) + if err != nil { + log.Warn().Err(err).Str("database", dbName).Msg("Failed to get committed txn count") + } + maxSeqNum, err := dbManager.GetMaxSeqNum(dbName) + if err != nil { + log.Warn().Err(err).Str("database", dbName).Msg("Failed to get max seq_num") } - return &ReplicationStateResponse{ - States: states, - }, nil + return &DatabaseReplicationState{ + DatabaseName: dbName, + LastAppliedTxnId: lastAppliedTxnID, + LastAppliedTimestamp: &HLC{WallTime: 0, Logical: 0, NodeId: s.nodeID}, + LastSyncTime: 0, + SyncStatus: syncStatus, + CurrentMaxTxnId: maxTxnID, + CommittedTxnCount: txnCount, + MaxSeqNum: maxSeqNum, + } } // ======================= @@ -871,11 +807,6 @@ func (s *Server) GetSnapshotInfo(ctx context.Context, req *SnapshotInfoRequest) return nil, snapshotError("failed to get snapshot info", err) } - // Schema versions travel with the snapshot: they live in the MetaStore, which - // is not part of the transferred files, and a receiver that cannot recover - // them refuses every transaction requiring a newer schema. - schemaVersions := snapshotSchemaVersions(dbManager) - // Calculate total size and chunks (estimates) var totalSize int64 var dbInfos []*DatabaseFileInfo @@ -902,7 +833,6 @@ func (s *Server) GetSnapshotInfo(ctx context.Context, req *SnapshotInfoRequest) SnapshotTxnId: txnID, SizeBytes: snap.Size, Sha256Checksum: snap.SHA256, - SchemaVersion: schemaVersions[snap.Name], }) } @@ -940,7 +870,6 @@ func (s *Server) databaseSnapshotInfo(dbManager *db.DatabaseManager, database st SnapshotTxnId: txnID, SizeBytes: snap.Size, Sha256Checksum: snap.SHA256, - SchemaVersion: snapshotSchemaVersions(dbManager)[snap.Name], }}, }, nil } @@ -974,23 +903,6 @@ func (s *Server) StreamSnapshot(req *SnapshotRequest, stream MarmotService_Strea snapshots := export.snapshots maxTxnID := export.maxTxnID - // Advertise the schema versions captured with these exact files as stream - // trailer metadata. GetSnapshotInfo's earlier estimate can go stale if a - // DDL commits between that call and this one; receivers prefer this value - // (see grpc.SnapshotVersionsForRestore) so they never restore a version - // older than the files they actually received. - schemaVersions := export.schemaVersions - if req.Database != "" { - // Single-database snapshots don't capture schema versions in the - // same atomic step as the file copy (unlike TakeSnapshotToDir), so - // read the current value here instead - it is the best available - // estimate for this legacy path. - schemaVersions = snapshotSchemaVersions(dbManager) - } - if trailer := snapshotSchemaVersionsTrailer(schemaVersions); trailer != nil { - stream.SetTrailer(trailer) - } - log.Info(). Uint64("requesting_node", req.RequestingNodeId). Uint64("snapshot_txn_id", maxTxnID). @@ -1049,6 +961,10 @@ func (s *Server) StreamSnapshot(req *SnapshotRequest, stream MarmotService_Strea Filename: snap.Filename, IsLastForFile: isLastForFile, } + if isLastForFile { + chunk.FileSha256 = snap.SHA256 + chunk.FileSizeBytes = snap.Size + } if err := stream.Send(chunk); err != nil { return fmt.Errorf("failed to send chunk: %w", err) @@ -1095,7 +1011,7 @@ func (s *Server) exportSnapshot(dbManager *db.DatabaseManager, dataDir, database } if database == "" { - export.snapshots, export.maxTxnID, export.schemaVersions, err = dbManager.TakeSnapshotToDir(export.dir) + export.snapshots, export.maxTxnID, err = dbManager.TakeSnapshotToDir(export.dir) if err != nil { export.release() return nil, snapshotError("failed to take snapshot", err) @@ -1196,6 +1112,17 @@ func (s *Server) SetCDCSubscriber(subscriber db.CDCSubscriber) { s.cdcSubscriber = subscriber } +// SetAntiEntropy wires the running AntiEntropyService into the server, so +// checkPromotionCriteria can additionally require PromotionReady for every +// local database and GetReplicationState can report real +// caught-up/stuck status. marmot.go calls this once anti-entropy is +// constructed. +func (s *Server) SetAntiEntropy(ae *AntiEntropyService) { + s.mu.Lock() + defer s.mu.Unlock() + s.antiEntropy = ae +} + // ======================= // PROMOTION CHECKER // ======================= @@ -1276,6 +1203,7 @@ func (s *Server) checkPromotionCriteria() bool { s.mu.RLock() dbManager := s.dbManager replicationHandler := s.replicationHandler + antiEntropy := s.antiEntropy s.mu.RUnlock() if dbManager == nil { @@ -1295,20 +1223,42 @@ func (s *Server) checkPromotionCriteria() bool { return false } + // Schema versions matching is not enough - a database can be current on schema + // but still missing committed rows anti-entropy has not pulled yet. + // Require every local database to be PromotionReady (AntiEntropyService, + // evaluated over alive peers only) before promoting. + if antiEntropy != nil { + for _, dbName := range localDatabases { + if dbName == db.SystemDatabaseName { + continue + } + if !antiEntropy.PromotionReady(dbName) { + log.Debug().Str("database", dbName).Msg("Database not promotion-ready, delaying promotion") + return false + } + } + } + // Verify schema versions match or exceed all ALIVE peers // This prevents premature promotion before DDL replication completes if replicationHandler == nil { return false } + return schemaVersionsMatchOrExceedPeers(replicationHandler, localDatabases, aliveNodes) +} +// schemaVersionsMatchOrExceedPeers reports whether every ALIVE peer's +// reported schema version, for every database it and this node both have, is +// no higher than this node's own cached version - the second half of +// checkPromotionCriteria's promotion gate. +func schemaVersionsMatchOrExceedPeers(replicationHandler *ReplicationHandler, localDatabases []string, aliveNodes []*NodeState) bool { localSchemaVersions, err := replicationHandler.GetAllSchemaVersions() if err != nil { log.Warn().Err(err).Msg("Failed to get local schema versions for promotion check") return false } - // Build a set of local database names for quick lookup - localDBSet := make(map[string]bool) + localDBSet := make(map[string]bool, len(localDatabases)) for _, dbName := range localDatabases { localDBSet[dbName] = true } @@ -1322,34 +1272,13 @@ func (s *Server) checkPromotionCriteria() bool { // For each database the peer has, check if our schema is at least as recent for dbName, peerVersion := range peerSchemaVersions { - // Skip system database - it's not subject to DDL replication - if dbName == db.SystemDatabaseName { - continue - } - - localVersion, hasSchemaEntry := localSchemaVersions[dbName] - hasLocalDB := localDBSet[dbName] - - if !hasLocalDB { - // Peer has a database we don't have yet - we'll get it via replication - // This is acceptable, continue checking + if dbName == db.SystemDatabaseName || !localDBSet[dbName] { + // System database is not subject to DDL replication; a + // database we don't have yet arrives via replication - both + // are acceptable, continue checking. continue } - - // If we have the database locally but no schema version entry, - // treat it as version 0 (fresh database, no DDL applied yet) - if !hasSchemaEntry { - localVersion = 0 - } - - // If peer has higher schema version, we're behind on DDL - if localVersion < peerVersion { - log.Debug(). - Str("database", dbName). - Uint64("local_version", localVersion). - Uint64("peer_version", peerVersion). - Uint64("peer_id", peer.NodeId). - Msg("Schema version behind peer, delaying promotion") + if schemaVersionBehindPeer(localSchemaVersions[dbName], peerVersion, dbName, peer.NodeId) { return false } } @@ -1358,6 +1287,22 @@ func (s *Server) checkPromotionCriteria() bool { return true } +// schemaVersionBehindPeer reports (and logs) whether localVersion is behind +// peerVersion for dbName. A missing local schema version entry is treated as +// version 0 (fresh database, no DDL applied yet). +func schemaVersionBehindPeer(localVersion, peerVersion uint64, dbName string, peerNodeID uint64) bool { + if localVersion >= peerVersion { + return false + } + log.Debug(). + Str("database", dbName). + Uint64("local_version", localVersion). + Uint64("peer_version", peerVersion). + Uint64("peer_id", peerNodeID). + Msg("Schema version behind peer, delaying promotion") + return true +} + // ======================= // SNAPSHOT CACHE CLEANUP // ======================= diff --git a/grpc/server_promotion_test.go b/grpc/server_promotion_test.go index e77e4e75..2c870504 100644 --- a/grpc/server_promotion_test.go +++ b/grpc/server_promotion_test.go @@ -21,10 +21,7 @@ func TestCheckPromotionCriteria_NoAliveNodes(t *testing.T) { t.Fatalf("Failed to create test database: %v", err) } - err = schemaVersionMgr.SetSchemaVersion("test_db", 1, "CREATE TABLE test (id INT)", 1) - if err != nil { - t.Fatalf("Failed to set schema version: %v", err) - } + bumpSchemaVersionForTest(t, dbMgr, "test_db", 1) clock := hlc.NewClock(1) replicationHandler := NewReplicationHandler(1, dbMgr, clock, schemaVersionMgr) @@ -56,10 +53,7 @@ func TestCheckPromotionCriteria_SchemaMatches(t *testing.T) { t.Fatalf("Failed to create test database: %v", err) } - err = schemaVersionMgr.SetSchemaVersion("test_db", 5, "CREATE TABLE test (id INT)", 1) - if err != nil { - t.Fatalf("Failed to set schema version: %v", err) - } + bumpSchemaVersionForTest(t, dbMgr, "test_db", 5) // Add an ALIVE peer with same schema version registry.Add(&NodeState{ @@ -101,10 +95,7 @@ func TestCheckPromotionCriteria_SchemaAhead(t *testing.T) { t.Fatalf("Failed to create test database: %v", err) } - err = schemaVersionMgr.SetSchemaVersion("test_db", 5, "CREATE TABLE test (id INT)", 1) - if err != nil { - t.Fatalf("Failed to set schema version: %v", err) - } + bumpSchemaVersionForTest(t, dbMgr, "test_db", 5) // Add an ALIVE peer with lower schema version registry.Add(&NodeState{ @@ -146,10 +137,7 @@ func TestCheckPromotionCriteria_SchemaBehind(t *testing.T) { t.Fatalf("Failed to create test database: %v", err) } - err = schemaVersionMgr.SetSchemaVersion("test_db", 5, "CREATE TABLE test (id INT)", 1) - if err != nil { - t.Fatalf("Failed to set schema version: %v", err) - } + bumpSchemaVersionForTest(t, dbMgr, "test_db", 5) // Add an ALIVE peer with higher schema version registry.Add(&NodeState{ @@ -191,10 +179,7 @@ func TestCheckPromotionCriteria_PeerHasNewDatabase(t *testing.T) { t.Fatalf("Failed to create test database: %v", err) } - err = schemaVersionMgr.SetSchemaVersion("test_db", 5, "CREATE TABLE test (id INT)", 1) - if err != nil { - t.Fatalf("Failed to set schema version: %v", err) - } + bumpSchemaVersionForTest(t, dbMgr, "test_db", 5) // Add an ALIVE peer with an additional database we don't have registry.Add(&NodeState{ @@ -241,14 +226,8 @@ func TestCheckPromotionCriteria_MultipleDatabase(t *testing.T) { t.Fatalf("Failed to create db2: %v", err) } - err = schemaVersionMgr.SetSchemaVersion("db1", 5, "CREATE TABLE test (id INT)", 1) - if err != nil { - t.Fatalf("Failed to set schema version for db1: %v", err) - } - err = schemaVersionMgr.SetSchemaVersion("db2", 3, "CREATE TABLE test (id INT)", 2) - if err != nil { - t.Fatalf("Failed to set schema version for db2: %v", err) - } + bumpSchemaVersionForTest(t, dbMgr, "db1", 5) + bumpSchemaVersionForTest(t, dbMgr, "db2", 3) // Add an ALIVE peer with multiple databases registry.Add(&NodeState{ @@ -295,14 +274,8 @@ func TestCheckPromotionCriteria_MultipleDatabaseOneBehind(t *testing.T) { t.Fatalf("Failed to create db2: %v", err) } - err = schemaVersionMgr.SetSchemaVersion("db1", 5, "CREATE TABLE test (id INT)", 1) - if err != nil { - t.Fatalf("Failed to set schema version for db1: %v", err) - } - err = schemaVersionMgr.SetSchemaVersion("db2", 3, "CREATE TABLE test (id INT)", 2) - if err != nil { - t.Fatalf("Failed to set schema version for db2: %v", err) - } + bumpSchemaVersionForTest(t, dbMgr, "db1", 5) + bumpSchemaVersionForTest(t, dbMgr, "db2", 3) // Add an ALIVE peer where one database has higher schema version registry.Add(&NodeState{ @@ -424,10 +397,7 @@ func TestCheckPromotionCriteria_PeerNilSchemaVersions(t *testing.T) { t.Fatalf("Failed to create test database: %v", err) } - err = schemaVersionMgr.SetSchemaVersion("test_db", 5, "CREATE TABLE test (id INT)", 1) - if err != nil { - t.Fatalf("Failed to set schema version: %v", err) - } + bumpSchemaVersionForTest(t, dbMgr, "test_db", 5) // Add an ALIVE peer with nil schema versions registry.Add(&NodeState{ @@ -613,10 +583,7 @@ func TestPromotionCriteria_CatchUpThenPromote(t *testing.T) { } // Now simulate catching up by setting our schema version to 2 - err = schemaVersionMgr.SetSchemaVersion("marmot", 2, "CREATE TABLE test (id INT)", 2) - if err != nil { - t.Fatalf("Failed to set schema version: %v", err) - } + bumpSchemaVersionForTest(t, dbMgr, "marmot", 2) // Second check: should SUCCEED because we've caught up result = server.checkPromotionCriteria() diff --git a/grpc/shutdown_integration_test.go b/grpc/shutdown_integration_test.go index 081b05a2..2498c8fc 100644 --- a/grpc/shutdown_integration_test.go +++ b/grpc/shutdown_integration_test.go @@ -256,10 +256,10 @@ func TestShutdown_ReplicationHandlerLifecycle(t *testing.T) { const testDB = "lifecycle_db" require.NoError(t, dbMgr.CreateDatabase(testDB)) - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) + _, err = dbMgr.GetDatabase(db.SystemDatabaseName) require.NoError(t, err) - handler := NewReplicationHandler(localNodeID, dbMgr, clock, db.NewSchemaVersionManager(systemDB.GetMetaStore())) + handler := NewReplicationHandler(localNodeID, dbMgr, clock, db.NewSchemaVersionManager(dbMgr)) registry := NewNodeRegistry(localNodeID, "localhost:9000") handler.SetRegistry(registry) diff --git a/grpc/snapshot_export.go b/grpc/snapshot_export.go index f8ba9a27..84c84f96 100644 --- a/grpc/snapshot_export.go +++ b/grpc/snapshot_export.go @@ -22,12 +22,11 @@ const snapshotExportPrefix = "snapshot-export-" // cache while the export is mapped. The last release removes the directory, // so no directory outlives its readers and none is removed while one reads. type snapshotExport struct { - dir string - snapshots []db.SnapshotInfo - maxTxnID uint64 - schemaVersions map[string]uint64 - expiresAt time.Time - refs atomic.Int32 + dir string + snapshots []db.SnapshotInfo + maxTxnID uint64 + expiresAt time.Time + refs atomic.Int32 } // newSnapshotExport creates an empty export directory under dataDir, with one diff --git a/grpc/snapshot_schema_version.go b/grpc/snapshot_schema_version.go deleted file mode 100644 index 700d192f..00000000 --- a/grpc/snapshot_schema_version.go +++ /dev/null @@ -1,177 +0,0 @@ -package grpc - -import ( - "fmt" - "path/filepath" - - "github.com/maxpert/marmot/db" - "github.com/maxpert/marmot/encoding" - "github.com/rs/zerolog/log" - "google.golang.org/grpc/metadata" -) - -// snapshotMetaStoreOptions sizes the short-lived MetaStore handle used to record -// restored schema versions. The store is opened, written and closed immediately, -// so it only needs minimal memory. MemTableCount is 2 because Pebble rejects a -// stop-writes threshold below that when creating a store, and a joining node has -// no system MetaStore yet. -func snapshotMetaStoreOptions() db.PebbleMetaStoreOptions { - return db.PebbleMetaStoreOptions{ - CacheSizeMB: 16, - MemTableSizeMB: 16, - MemTableCount: 2, - L0CompactionThreshold: 4, - L0StopWrites: 12, - } -} - -// snapshotSchemaVersions reads this node's schema version for every database so -// they can be advertised alongside a snapshot. Returns an empty map when the -// versions cannot be read - the snapshot itself is still usable, the receiver -// simply has nothing to restore. -func snapshotSchemaVersions(dbManager *db.DatabaseManager) map[string]uint64 { - systemDB, err := dbManager.GetDatabase(db.SystemDatabaseName) - if err != nil { - log.Warn().Err(err).Msg("Failed to reach system database for snapshot schema versions") - return nil - } - - stored, err := systemDB.GetMetaStore().GetAllSchemaVersions() - if err != nil { - log.Warn().Err(err).Msg("Failed to read schema versions for snapshot") - return nil - } - - versions := make(map[string]uint64, len(stored)) - for database, version := range stored { - if version > 0 { - versions[database] = uint64(version) - } - } - return versions -} - -// persistSnapshotSchemaVersions records the schema versions carried by a snapshot -// into the system database's MetaStore, opening it directly by path. -// -// A snapshot transfers SQLite files only, while schema versions live in the -// MetaStore. A node that catches up without this keeps reporting version 0 and -// then refuses every transaction stamped with a higher RequiredSchemaVersion - -// including plain DML - so it silently stops replicating and never recovers, -// because the only other way to advance the version is committing a DDL -// transaction it can no longer take part in. -// -// Used by the join path, where the DatabaseManager does not exist yet. Opening -// the MetaStore a second time by path while a DatabaseManager already holds it -// open would fail: Pebble takes an exclusive lock per process. Runtime restores -// (anti-entropy) must go through persistSnapshotSchemaVersionsViaManager instead. -func persistSnapshotSchemaVersions(dataDir string, versions map[string]uint64) error { - if len(versions) == 0 { - return nil - } - - metaPath := filepath.Join(dataDir, db.SystemDatabaseName+"_meta.pebble") - metaStore, err := db.NewPebbleMetaStore(metaPath, snapshotMetaStoreOptions()) - if err != nil { - return fmt.Errorf("failed to open system MetaStore to restore schema versions: %w", err) - } - defer metaStore.Close() - - return db.RestoreSchemaVersions(metaStore, versions) -} - -// persistSnapshotSchemaVersionsViaManager records the schema versions carried by -// a snapshot into the system database's MetaStore, writing through an -// already-open DatabaseManager instead of opening a second handle on the same -// Pebble directory (which would deadlock against Pebble's per-process -// exclusive lock). Used by the anti-entropy runtime path, where the -// DatabaseManager already exists. -func persistSnapshotSchemaVersionsViaManager(dbManager *db.DatabaseManager, versions map[string]uint64) error { - if len(versions) == 0 { - return nil - } - - systemDB, err := dbManager.GetDatabase(db.SystemDatabaseName) - if err != nil { - return fmt.Errorf("system database unavailable: %w", err) - } - - return db.RestoreSchemaVersions(systemDB.GetMetaStore(), versions) -} - -// SnapshotSchemaVersions extracts the schema versions advertised with a snapshot, -// keyed by database name. -func SnapshotSchemaVersions(snapshotMetadata []*DatabaseSnapshotMetadata) map[string]uint64 { - versions := make(map[string]uint64, len(snapshotMetadata)) - for _, meta := range snapshotMetadata { - if meta == nil || meta.DatabaseName == "" || meta.SchemaVersion == 0 { - continue - } - versions[meta.DatabaseName] = meta.SchemaVersion - } - return versions -} - -// snapshotSchemaVersionsTrailerKey carries schema versions as gRPC trailer -// metadata on the StreamSnapshot RPC. Binary values need the "-bin" key suffix; -// grpc-go base64-encodes/decodes them on the wire automatically. -const snapshotSchemaVersionsTrailerKey = "marmot-snapshot-schema-versions-bin" - -// snapshotSchemaVersionsTrailer builds gRPC trailer metadata carrying schema -// versions captured atomically with the snapshot files actually streamed, so -// the receiver does not have to trust the separate (and potentially stale) -// read taken earlier by GetSnapshotInfo. Returns nil when there is nothing to -// carry, so callers see no trailer rather than an empty one. -func snapshotSchemaVersionsTrailer(versions map[string]uint64) metadata.MD { - if len(versions) == 0 { - return nil - } - - payload, err := encoding.Marshal(versions) - if err != nil { - log.Warn().Err(err).Msg("Failed to encode schema versions for snapshot trailer") - return nil - } - - return metadata.Pairs(snapshotSchemaVersionsTrailerKey, string(payload)) -} - -// decodeSchemaVersionsTrailer decodes schema versions carried in a snapshot -// stream's trailer. Returns (nil, nil) when no trailer was set - by an older -// server, or a stream with nothing to restore - so callers can fall back to -// the versions advertised earlier by GetSnapshotInfo without treating it as an -// error. -func decodeSchemaVersionsTrailer(md metadata.MD) (map[string]uint64, error) { - values := md.Get(snapshotSchemaVersionsTrailerKey) - if len(values) == 0 { - return nil, nil - } - - var versions map[string]uint64 - if err := encoding.Unmarshal([]byte(values[0]), &versions); err != nil { - return nil, fmt.Errorf("failed to decode schema versions trailer: %w", err) - } - return versions, nil -} - -// SnapshotStreamTrailer is satisfied by both ends of the StreamSnapshot gRPC -// stream. It is used to read the authoritative schema versions the server -// attaches as trailer metadata once the snapshot's files have been produced. -type SnapshotStreamTrailer interface { - Trailer() metadata.MD -} - -// SnapshotVersionsForRestore chooses which schema versions to restore after a -// snapshot. It prefers the versions carried in the stream's trailer - captured -// atomically with the files that were actually streamed - and falls back to -// the versions advertised earlier by GetSnapshotInfo, which can go stale if a -// DDL commits between that call and the StreamSnapshot call that follows it. -func SnapshotVersionsForRestore(snapshotMetadata []*DatabaseSnapshotMetadata, stream SnapshotStreamTrailer) map[string]uint64 { - trailerVersions, err := decodeSchemaVersionsTrailer(stream.Trailer()) - if err != nil { - log.Warn().Err(err).Msg("Failed to decode schema versions from snapshot trailer, falling back to GetSnapshotInfo metadata") - } else if len(trailerVersions) > 0 { - return trailerVersions - } - return SnapshotSchemaVersions(snapshotMetadata) -} diff --git a/grpc/snapshot_schema_version_test.go b/grpc/snapshot_schema_version_test.go deleted file mode 100644 index 310e47e4..00000000 --- a/grpc/snapshot_schema_version_test.go +++ /dev/null @@ -1,193 +0,0 @@ -package grpc - -import ( - "path/filepath" - "testing" - - "github.com/maxpert/marmot/db" - "github.com/maxpert/marmot/hlc" - "github.com/stretchr/testify/require" - "google.golang.org/grpc/metadata" -) - -// A node that catches up by snapshot receives only SQLite files; schema versions -// live in the system database's MetaStore and are not part of that transfer. -// Without restoring them the node reports version 0 forever and every later -// transaction carrying a RequiredSchemaVersion is refused, which silently drops -// the node out of replication. -func TestPersistSnapshotSchemaVersions(t *testing.T) { - dataDir := t.TempDir() - - metadata := []*DatabaseSnapshotMetadata{ - {DatabaseName: "appdb", SchemaVersion: 17}, - {DatabaseName: "otherdb", SchemaVersion: 3}, - } - - require.NoError(t, persistSnapshotSchemaVersions(dataDir, SnapshotSchemaVersions(metadata))) - - versions := readSchemaVersions(t, dataDir) - require.Equal(t, int64(17), versions["appdb"]) - require.Equal(t, int64(3), versions["otherdb"]) -} - -// Restoring an older snapshot must not roll a node's schema version backwards: -// the node may have applied DDL after the snapshot was taken. -func TestPersistSnapshotSchemaVersionsNeverGoesBackwards(t *testing.T) { - dataDir := t.TempDir() - - require.NoError(t, persistSnapshotSchemaVersions(dataDir, - SnapshotSchemaVersions([]*DatabaseSnapshotMetadata{{DatabaseName: "appdb", SchemaVersion: 17}}))) - require.NoError(t, persistSnapshotSchemaVersions(dataDir, - SnapshotSchemaVersions([]*DatabaseSnapshotMetadata{{DatabaseName: "appdb", SchemaVersion: 5}}))) - - require.Equal(t, int64(17), readSchemaVersions(t, dataDir)["appdb"]) -} - -func TestPersistSnapshotSchemaVersionsIgnoresEmptyMetadata(t *testing.T) { - dataDir := t.TempDir() - - require.NoError(t, persistSnapshotSchemaVersions(dataDir, SnapshotSchemaVersions(nil))) - require.NoError(t, persistSnapshotSchemaVersions(dataDir, SnapshotSchemaVersions([]*DatabaseSnapshotMetadata{ - {DatabaseName: "", SchemaVersion: 4}, // no database name - {DatabaseName: "appdb", SchemaVersion: 0}, // nothing to restore - }))) - - require.Empty(t, readSchemaVersions(t, dataDir)) -} - -// persistSnapshotSchemaVersions opens the system MetaStore by path. That is -// only safe on the startup join path, where no DatabaseManager exists yet. If -// a DatabaseManager is already running against the same dataDir - the runtime -// anti-entropy case - Pebble's per-process exclusive lock makes a second open -// of the same directory fail. This pins that failure mode so a future change -// cannot silently reintroduce the anti-entropy stall this fix removes. -func TestPersistSnapshotSchemaVersions_FailsWhenManagerAlreadyOpen(t *testing.T) { - dataDir := t.TempDir() - - dbMgr, err := db.NewDatabaseManager(dataDir, 1, hlc.NewClock(1)) - require.NoError(t, err) - defer dbMgr.Close() - - err = persistSnapshotSchemaVersions(dataDir, SnapshotSchemaVersions( - []*DatabaseSnapshotMetadata{{DatabaseName: "appdb", SchemaVersion: 1}})) - require.Error(t, err) -} - -// persistSnapshotSchemaVersionsViaManager is what the runtime path must use -// instead: it writes through the DatabaseManager's already-open system -// MetaStore, so it succeeds even while that same store is held open by the -// DatabaseManager - the exact situation that fails above. -func TestPersistSnapshotSchemaVersionsViaManager(t *testing.T) { - dataDir := t.TempDir() - - dbMgr, err := db.NewDatabaseManager(dataDir, 1, hlc.NewClock(1)) - require.NoError(t, err) - defer dbMgr.Close() - - versions := map[string]uint64{"appdb": 9} - require.NoError(t, persistSnapshotSchemaVersionsViaManager(dbMgr, versions)) - - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) - require.NoError(t, err) - got, err := systemDB.GetMetaStore().GetSchemaVersion("appdb") - require.NoError(t, err) - require.Equal(t, int64(9), got) -} - -func TestPersistSnapshotSchemaVersionsViaManager_IgnoresEmptyVersions(t *testing.T) { - dataDir := t.TempDir() - - dbMgr, err := db.NewDatabaseManager(dataDir, 1, hlc.NewClock(1)) - require.NoError(t, err) - defer dbMgr.Close() - - require.NoError(t, persistSnapshotSchemaVersionsViaManager(dbMgr, nil)) -} - -func readSchemaVersions(t *testing.T, dataDir string) map[string]int64 { - t.Helper() - - metaPath := filepath.Join(dataDir, db.SystemDatabaseName+"_meta.pebble") - metaStore, err := db.NewPebbleMetaStore(metaPath, snapshotMetaStoreOptions()) - require.NoError(t, err) - defer metaStore.Close() - - versions, err := metaStore.GetAllSchemaVersions() - require.NoError(t, err) - return versions -} - -// --- trailer encode/decode --- - -func TestSnapshotSchemaVersionsTrailer_RoundTrip(t *testing.T) { - versions := map[string]uint64{"appdb": 4, "otherdb": 1} - - trailer := snapshotSchemaVersionsTrailer(versions) - require.NotNil(t, trailer) - - got, err := decodeSchemaVersionsTrailer(trailer) - require.NoError(t, err) - require.Equal(t, versions, got) -} - -func TestSnapshotSchemaVersionsTrailer_EmptyVersionsProduceNoTrailer(t *testing.T) { - require.Nil(t, snapshotSchemaVersionsTrailer(nil)) - require.Nil(t, snapshotSchemaVersionsTrailer(map[string]uint64{})) -} - -// An absent trailer (older server, or nothing to restore) must decode to a -// nil map with no error, so callers fall back to GetSnapshotInfo metadata -// instead of treating "no trailer" as a decode failure. -func TestDecodeSchemaVersionsTrailer_AbsentIsNotAnError(t *testing.T) { - versions, err := decodeSchemaVersionsTrailer(metadata.MD{}) - require.NoError(t, err) - require.Nil(t, versions) -} - -func TestDecodeSchemaVersionsTrailer_CorruptPayloadErrors(t *testing.T) { - md := metadata.Pairs(snapshotSchemaVersionsTrailerKey, "not valid msgpack") - _, err := decodeSchemaVersionsTrailer(md) - require.Error(t, err) -} - -// --- SnapshotVersionsForRestore --- - -type fakeSnapshotStreamTrailer struct { - trailer metadata.MD -} - -func (f *fakeSnapshotStreamTrailer) Trailer() metadata.MD { return f.trailer } - -// The trailer - captured atomically with the files actually streamed - must -// win over GetSnapshotInfo's earlier, separately-read metadata whenever both -// are present. This is the fix for the race where a DDL commits between the -// GetSnapshotInfo and StreamSnapshot RPCs. -func TestSnapshotVersionsForRestore_PrefersTrailerOverMetadata(t *testing.T) { - staleMetadata := []*DatabaseSnapshotMetadata{{DatabaseName: "appdb", SchemaVersion: 3}} - stream := &fakeSnapshotStreamTrailer{trailer: snapshotSchemaVersionsTrailer(map[string]uint64{"appdb": 4})} - - got := SnapshotVersionsForRestore(staleMetadata, stream) - require.Equal(t, map[string]uint64{"appdb": 4}, got) -} - -// An older server never sets the trailer. Receivers must fall back to -// GetSnapshotInfo's metadata rather than restoring nothing. -func TestSnapshotVersionsForRestore_FallsBackWhenNoTrailer(t *testing.T) { - fallbackMetadata := []*DatabaseSnapshotMetadata{{DatabaseName: "appdb", SchemaVersion: 3}} - stream := &fakeSnapshotStreamTrailer{trailer: metadata.MD{}} - - got := SnapshotVersionsForRestore(fallbackMetadata, stream) - require.Equal(t, map[string]uint64{"appdb": 3}, got) -} - -// A corrupt trailer must not be trusted, but must also not make restore fail -// outright - fall back to the metadata instead. -func TestSnapshotVersionsForRestore_FallsBackOnCorruptTrailer(t *testing.T) { - fallbackMetadata := []*DatabaseSnapshotMetadata{{DatabaseName: "appdb", SchemaVersion: 3}} - stream := &fakeSnapshotStreamTrailer{ - trailer: metadata.Pairs(snapshotSchemaVersionsTrailerKey, "not valid msgpack"), - } - - got := SnapshotVersionsForRestore(fallbackMetadata, stream) - require.Equal(t, map[string]uint64{"appdb": 3}, got) -} diff --git a/grpc/test_helpers.go b/grpc/test_helpers.go index 7777566d..6134d7e7 100644 --- a/grpc/test_helpers.go +++ b/grpc/test_helpers.go @@ -1,12 +1,15 @@ package grpc import ( + "fmt" "os" "path/filepath" "testing" + "time" "github.com/maxpert/marmot/db" "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" ) // setupTestEnvironment creates a test database manager and schema version manager @@ -23,13 +26,56 @@ func setupTestEnvironment(t *testing.T, testName string) (string, *db.DatabaseMa t.Fatalf("Failed to create database manager: %v", err) } - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) - if err != nil { + if _, err := dbMgr.GetDatabase(db.SystemDatabaseName); err != nil { dbMgr.Close() t.Fatalf("Failed to get system database: %v", err) } - schemaVersionMgr := db.NewSchemaVersionManager(systemDB.GetMetaStore()) + schemaVersionMgr := db.NewSchemaVersionManager(dbMgr) return tmpDir, dbMgr, schemaVersionMgr } + +// bumpSchemaVersionForTest commits n trivial CREATE TABLE DDL transactions +// against database, through its real TransactionManager, so its +// __marmot_schema_version ends at exactly n. Schema versions are +// derived from real committed DDL now (SchemaVersionManager has no setter +// any more), so this stands in for the tests' former direct +// SchemaVersionManager.SetSchemaVersion(database, n, ...) calls. +func bumpSchemaVersionForTest(t *testing.T, dbMgr *db.DatabaseManager, database string, n int) { + t.Helper() + mdb, err := dbMgr.GetDatabase(database) + if err != nil { + t.Fatalf("bumpSchemaVersionForTest: database %s unavailable: %v", database, err) + } + txnMgr := mdb.GetTransactionManager() + + for i := 0; i < n; i++ { + txn, err := txnMgr.BeginTransaction(1) + if err != nil { + t.Fatalf("bumpSchemaVersionForTest: begin: %v", err) + } + table := fmt.Sprintf("__test_schema_bump_%d_%d", time.Now().UnixNano(), i) + stmt := protocol.Statement{ + Type: protocol.StatementDDL, + SQL: fmt.Sprintf("CREATE TABLE %s (id INTEGER PRIMARY KEY)", table), + TableName: table, + Database: database, + } + snapshot, err := db.SerializeData(db.DDLSnapshot{ + Type: int(stmt.Type), + Timestamp: time.Now().UnixNano(), + SQL: stmt.SQL, + TableName: stmt.TableName, + }) + if err != nil { + t.Fatalf("bumpSchemaVersionForTest: serialize DDL: %v", err) + } + if err := txnMgr.WriteIntent(txn, db.IntentTypeDDL, table, "ddl:"+table, stmt, snapshot); err != nil { + t.Fatalf("bumpSchemaVersionForTest: write intent: %v", err) + } + if err := txnMgr.CommitTransaction(txn); err != nil { + t.Fatalf("bumpSchemaVersionForTest: commit: %v", err) + } + } +} diff --git a/grpc/vector_statement.go b/grpc/vector_statement.go index 14e4a8ba..c5f86aa0 100644 --- a/grpc/vector_statement.go +++ b/grpc/vector_statement.go @@ -120,6 +120,7 @@ func protocolStatementFromProto(stmt *Statement) (protocol.Statement, error) { IntentKey: stmt.GetIntentKey(), AutoIDClaim: stmt.GetAutoIdClaim(), AutoIDClaimPayload: stmt.GetAutoIdClaimPayload(), + DatabaseGeneration: stmt.GetDatabaseGeneration(), } switch payload := stmt.Payload.(type) { diff --git a/marmot.go b/marmot.go index 0fb214a4..f1c7266d 100644 --- a/marmot.go +++ b/marmot.go @@ -230,7 +230,6 @@ func main() { case marmotgrpc.DELTA_SYNC: log.Info(). - Int("databases_behind", len(decision.DatabaseDeltas)). Str("peer", decision.PeerAddr). Msg("Delta sync required - will catch up after database initialization") // Delta sync will be performed after database manager is initialized @@ -326,14 +325,14 @@ func main() { } } - // Phase 5: Initialize schema version manager using system database's MetaStore + // Phase 5: Initialize schema version manager, reading each database's own + // __marmot_schema_version table. log.Info().Msg("Initializing schema version manager") - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) - if err != nil { + if _, err := dbMgr.GetDatabase(db.SystemDatabaseName); err != nil { log.Fatal().Err(err).Msg("Failed to get system database for schema versioning") return } - schemaVersionMgr := db.NewSchemaVersionManager(systemDB.GetMetaStore()) + schemaVersionMgr := db.NewSchemaVersionManager(dbMgr) // Wire up replication handlers log.Info().Msg("Wiring up replication handlers") @@ -353,26 +352,25 @@ func main() { // Phase 6: Setup anti-entropy service for catching up lagging nodes log.Info().Msg("Setting up anti-entropy service") - deltaSync := marmotgrpc.NewDeltaSyncClient(marmotgrpc.DeltaSyncConfig{ - NodeID: cfg.Config.NodeID, - Client: client, - DBManager: dbMgr, - Clock: clock, - ApplyTxnsFn: replicationHandler.HandleReplicateTransaction, - SchemaVersionMgr: schemaVersionMgr, + logPuller := marmotgrpc.NewLogPuller(marmotgrpc.LogPullerConfig{ + NodeID: cfg.Config.NodeID, + Client: client, + DBManager: dbMgr, }) - // If we determined we need delta sync, perform it now that database manager is initialized + // If we determined we need targeted catch-up, perform it now that the + // database manager is initialized: one bounded LogPuller pass per + // database behind, over the seed peer. if isJoiningCluster && catchUpDecision != nil && catchUpDecision.Strategy == marmotgrpc.DELTA_SYNC { - log.Info().Msg("Performing delta sync now that database manager is initialized") + log.Info().Msg("Performing startup log pull now that database manager is initialized") ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - err := catchUpClient.PerformDeltaSync(ctx, catchUpDecision, deltaSync) + err := catchUpClient.PerformLogPull(ctx, catchUpDecision, logPuller, client) cancel() if err != nil { - log.Fatal().Err(err).Msg("Failed to perform delta sync") + log.Fatal().Err(err).Msg("Failed to perform startup log pull") return } - log.Info().Msg("Delta sync completed successfully") + log.Info().Msg("Startup log pull completed successfully") } // Create snapshot function for anti-entropy. CatchUpFromPeer keeps the @@ -387,15 +385,31 @@ func main() { grpcServer.GetNodeRegistry(), client, dbMgr, - deltaSync, - clock, + logPuller, snapshotFunc, - schemaVersionMgr, ) - - // Wire anti-entropy refresh to DatabaseManager for GC watermark freshness - // This ensures GC queries fresh peer states before making deletion decisions - dbMgr.SetRefreshReplicationStatesFunc(antiEntropy.RefreshPeerReplicationStates) + adminHandlers.SetAntiEntropy(antiEntropy) + grpcServer.SetAntiEntropy(antiEntropy) + + // Wire GC's safe deletion position to this node's current cluster + // membership: every registry node + // whose status is not REMOVED, self included (LEAVING/DEAD/SUSPECT are + // still members, same as anti-entropy's own view - + // AntiEntropyService.currentMembers - unlike NodeRegistry.MemberIDs, + // which also excludes LEAVING for quorum purposes). wireGCCoordination + // (db/database_manager.go) turns this into each database's min consumed + // position across every other current member. + registry := grpcServer.GetNodeRegistry() + dbMgr.SetGCMembershipFunc(func() []uint64 { + nodes := registry.GetAll() + ids := make([]uint64, 0, len(nodes)) + for _, n := range nodes { + if n.Status != marmotgrpc.NodeStatus_REMOVED { + ids = append(ids, n.NodeId) + } + } + return ids + }) // Start anti-entropy service antiEntropy.Start() diff --git a/protocol/transaction.go b/protocol/transaction.go index 3d7fd3ed..7662f777 100644 --- a/protocol/transaction.go +++ b/protocol/transaction.go @@ -218,6 +218,18 @@ type Statement struct { // LoadDataPayload carries LOAD DATA LOCAL INFILE file bytes for replicated // non-DML bulk-load transactions. LoadDataPayload []byte `msgpack:"LoadDataPayload,omitempty"` + + // DatabaseGeneration fences a CREATE/DROP DATABASE statement against a + // stale coordinator. The coordinator stamps it + // from its own database registry view: CREATE stamps the database's next + // generation (or its current one, as a no-op, when the database is + // already live); DROP stamps the database's current generation. A + // participant refuses PREPARE when this is below its own local key for the + // database. Zero means "unstamped" - a coordinator that predates this + // field - and a participant computes the key itself instead of gating on + // it, for rolling-upgrade compatibility. Meaningless on every other + // statement type. + DatabaseGeneration uint64 `msgpack:"DatabaseGeneration,omitempty"` } // MergeExecParams produces the final positional argument list for executing diff --git a/replica/database_discovery_test.go b/replica/database_discovery_test.go index 91225917..6c1ae294 100644 --- a/replica/database_discovery_test.go +++ b/replica/database_discovery_test.go @@ -154,6 +154,30 @@ func (m *mockMarmotServiceClient) TransactionStream(ctx context.Context, opts .. return args.Get(0).(grpc.ClientStreamingClient[marmotgrpc.TransactionStreamMessage, marmotgrpc.TransactionResponse]), args.Error(1) } +func (m *mockMarmotServiceClient) ListCommittedLog(ctx context.Context, req *marmotgrpc.LogListRequest, opts ...grpc.CallOption) (*marmotgrpc.LogListResponse, error) { + args := m.Called(ctx, req) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*marmotgrpc.LogListResponse), args.Error(1) +} + +func (m *mockMarmotServiceClient) FetchTransactions(ctx context.Context, req *marmotgrpc.FetchTransactionsRequest, opts ...grpc.CallOption) (marmotgrpc.MarmotService_FetchTransactionsClient, error) { + args := m.Called(ctx, req) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(marmotgrpc.MarmotService_FetchTransactionsClient), args.Error(1) +} + +func (m *mockMarmotServiceClient) ListDatabaseRegistry(ctx context.Context, req *marmotgrpc.DatabaseRegistryRequest, opts ...grpc.CallOption) (*marmotgrpc.DatabaseRegistryResponse, error) { + args := m.Called(ctx, req) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*marmotgrpc.DatabaseRegistryResponse), args.Error(1) +} + // TestDiscoverNewDatabases_DetectsNewDatabase tests that discovery loop detects new databases func TestDiscoverNewDatabases_DetectsNewDatabase(t *testing.T) { dbMgr, clock, _, cleanup := setupStreamClientTest(t) diff --git a/replica/stream_client.go b/replica/stream_client.go index 626e793a..ab00e3ca 100644 --- a/replica/stream_client.go +++ b/replica/stream_client.go @@ -773,15 +773,8 @@ func (s *StreamClient) applySnapshot(ctx context.Context, snapshotInfo *marmotgr return fmt.Errorf("snapshot restore failed: %w", err) } - // Restore the schema versions the snapshot was taken at, otherwise this node - // reports version 0 and refuses every transaction requiring a newer schema. - // Prefer the versions captured atomically with these exact files (the - // stream trailer) over the earlier, potentially stale read from - // GetSnapshotInfo. - versions := marmotgrpc.SnapshotVersionsForRestore(snapshotInfo.DatabaseMetadata, stream) - if err := s.restoreSchemaVersions(versions); err != nil { - return fmt.Errorf("failed to restore schema versions from snapshot: %w", err) - } + // Each installed SQLite file carries its own __marmot_schema_version + // table, so nothing needs to be restored out of band here any more. log.Info(). Uint64("snapshot_txn_id", snapshotInfo.SnapshotTxnId). @@ -790,22 +783,6 @@ func (s *StreamClient) applySnapshot(ctx context.Context, snapshotInfo *marmotgr return nil } -// restoreSchemaVersions records snapshot schema versions through the live system -// MetaStore. The replica keeps its own system database, so the versions are -// written to the already-open store rather than by reopening it on disk. -func (s *StreamClient) restoreSchemaVersions(versions map[string]uint64) error { - if len(versions) == 0 { - return nil - } - - systemDB, err := s.dbManager.GetDatabase(db.SystemDatabaseName) - if err != nil { - return fmt.Errorf("system database unavailable: %w", err) - } - - return db.RestoreSchemaVersions(systemDB.GetMetaStore(), versions) -} - // replicaSnapshotStreamAdapter adapts MarmotService_StreamSnapshotClient to snapshot.ChunkReceiver type replicaSnapshotStreamAdapter struct { stream marmotgrpc.MarmotService_StreamSnapshotClient @@ -824,6 +801,8 @@ func (a *replicaSnapshotStreamAdapter) Recv() (*snapshot.Chunk, error) { Data: chunk.GetData(), MD5Checksum: chunk.GetChecksum(), IsLastForFile: chunk.GetIsLastForFile(), + FileSHA256: chunk.GetFileSha256(), + FileSizeBytes: chunk.GetFileSizeBytes(), }, nil } @@ -878,7 +857,7 @@ func (s *StreamClient) applyChangeEvent(ctx context.Context, event *marmotgrpc.C if err := s.applyVectorIndexChange(ctx, marmotgrpc.VectorChangeFromProto(change)); err != nil { return err } - _, err := marmotgrpc.StoreAppliedChangeEvent(mdb.GetMetaStore(), event.TxnId, event.Timestamp, database, event.Statements) + _, err := marmotgrpc.StoreAppliedChangeEvent(mdb, event.TxnId, event.Timestamp, event.Statements, event.OriginNodeId, event.RequiredSchemaVersion) return err } } @@ -914,7 +893,7 @@ func (s *StreamClient) applyChangeEvent(ctx context.Context, event *marmotgrpc.C log.Warn().Err(err).Str("database", database).Msg("Failed to reload schema after DDL") } } - seqNum, err := marmotgrpc.StoreAppliedChangeEvent(mdb.GetMetaStore(), event.TxnId, event.Timestamp, database, event.Statements) + seqNum, err := marmotgrpc.StoreAppliedChangeEvent(mdb, event.TxnId, event.Timestamp, event.Statements, event.OriginNodeId, event.RequiredSchemaVersion) if err != nil { return fmt.Errorf("failed to store streamed rows: %w", err) } diff --git a/telemetry/metrics.go b/telemetry/metrics.go index 91950305..31da3e80 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -65,9 +65,6 @@ var ( // ReplicaCommitSeconds measures replica-side commit phase latency ReplicaCommitSeconds Histogram = NoopStat{} - // ReplicaReplaySeconds measures replica-side replay phase latency (anti-entropy) - ReplicaReplaySeconds Histogram = NoopStat{} - // TwoPhaseQuorumAcks measures number of acks received per phase TwoPhaseQuorumAcks HistogramVec = noopHistogramVec{} @@ -121,9 +118,6 @@ var ( // ReplicationLagTxns tracks transaction lag per peer ReplicationLagTxns GaugeVec = noopGaugeVec{} - - // DeltaSyncTxnsTotal counts transactions applied via delta sync - DeltaSyncTxnsTotal Counter = NoopStat{} ) // RowLockStore metrics @@ -138,6 +132,23 @@ var ( RowLockTables Gauge = NoopStat{} ) +// LogPuller metrics: the per-peer log-pull anti-entropy round. +var ( + // LogPullStuckTxns tracks, per database, the number of transactions a + // LogPuller has given up retrying every round because they failed to + // replay MaxReplayAttempts times in a row. It is exported as marmot_v2_anti_entropy_stuck_txns, following + // this package's existing namespace/subsystem convention. + LogPullStuckTxns GaugeVec = noopGaugeVec{} + + // LogPullTxnsAppliedTotal counts transactions a LogPuller has applied + // (not merely covered by an existing marker) while pulling peers' logs. + LogPullTxnsAppliedTotal Counter = NoopStat{} + + // LogPullPairResultsTotal counts LogPuller.PullPair calls by outcome: + // caught_up, behind, needs_snapshot, unimplemented, error. + LogPullPairResultsTotal CounterVec = noopCounterVec{} +) + // InitMetrics initializes all Prometheus metrics. // Must be called after InitializeTelemetry(). func InitMetrics() { @@ -207,11 +218,6 @@ func InitMetrics() { "Replica-side commit phase latency in seconds", TwoPCBuckets, ) - ReplicaReplaySeconds = NewHistogramWithBuckets( - "replica_replay_seconds", - "Replica-side replay phase latency in seconds", - TwoPCBuckets, - ) TwoPhaseQuorumAcks = NewHistogramVec( "twophase_quorum_acks", "Number of quorum acknowledgments received", @@ -292,10 +298,6 @@ func InitMetrics() { "Transaction lag behind peer", []string{"peer"}, ) - DeltaSyncTxnsTotal = NewCounter( - "delta_sync_txns_total", - "Total transactions applied via delta sync", - ) // RowLockStore metrics RowLocksActive = NewGauge( @@ -310,6 +312,22 @@ func InitMetrics() { "row_lock_tables", "Number of tables with active locks", ) + + // LogPuller metrics + LogPullStuckTxns = NewGaugeVec( + "anti_entropy_stuck_txns", + "Number of transactions a log-pull round has stopped retrying every round after repeated replay failures", + []string{"database"}, + ) + LogPullTxnsAppliedTotal = NewCounter( + "log_pull_txns_applied_total", + "Total transactions applied by pulling peers' local commit logs", + ) + LogPullPairResultsTotal = NewCounterVec( + "log_pull_pair_results_total", + "LogPuller.PullPair calls by outcome", + []string{"outcome"}, + ) } // UpdateRowLockStats updates row lock gauges from stats diff --git a/test/autoinc_backstop_cluster_test.go b/test/autoinc_backstop_cluster_test.go index 64c3b447..68ad57c2 100644 --- a/test/autoinc_backstop_cluster_test.go +++ b/test/autoinc_backstop_cluster_test.go @@ -19,8 +19,8 @@ const ( localCommitFailedLog = "Local commit failed after remote quorum achieved" ) -// TestNarrowAutoInc_ClaimsCommitUnderConstantBaseSync is reviewer rr-s3cf2's -// H1 hunt, amplified without touching the backstop's interval: while every +// TestNarrowAutoInc_ClaimsCommitUnderConstantBaseSync amplifies the base +// sync without touching the backstop's interval: while every // node inserts into two SMALLINT tables (a claim every 31 ids), the test runs // the operator's sync command back to back on every node, so each node pulls // its peers' bases many times a second. A peer that committed a claim first @@ -43,8 +43,11 @@ func TestNarrowAutoInc_ClaimsCommitUnderConstantBaseSync(t *testing.T) { if err := harness.WaitForTableExists(tables[1], []int{1, 2, 3}, 30*time.Second); err != nil { t.Fatal(err) } + harness.Phase("start") - const load = 45 * time.Second + // 35s (not 45s): amplifies the backstop enough (100+ syncs) while keeping + // the whole test under a 55s budget. + const load = 35 * time.Second stop := time.Now().Add(load) var syncs, syncErrs atomic.Int64 var wg sync.WaitGroup @@ -94,6 +97,7 @@ func TestNarrowAutoInc_ClaimsCommitUnderConstantBaseSync(t *testing.T) { }(n) } wg.Wait() + harness.Phase("load-stopped") t.Logf("syncs=%d sync errors=%d inserted per node=%v ids=%d", syncs.Load(), syncErrs.Load(), inserted[1:], len(ledger)) if len(dups) > 0 { @@ -123,6 +127,6 @@ func TestNarrowAutoInc_ClaimsCommitUnderConstantBaseSync(t *testing.T) { } } if syncs.Load() < 100 { - t.Fatalf("only %d syncs ran in %s; the hunt did not amplify the backstop", syncs.Load(), load) + t.Fatalf("only %d syncs ran in %s; the test did not amplify the backstop", syncs.Load(), load) } } diff --git a/test/autoinc_claim_inprocess_test.go b/test/autoinc_claim_inprocess_test.go index 7dc17ed3..863b9488 100644 --- a/test/autoinc_claim_inprocess_test.go +++ b/test/autoinc_claim_inprocess_test.go @@ -1076,7 +1076,7 @@ func TestClaimRange_SurvivesFullClusterSnapshotRestore(t *testing.T) { // Snapshot every node BEFORE any of them stop. for _, id := range []uint64{1, 2, 3} { snapDir := t.TempDir() - snapshots, _, _, err := c.nodes[id].dm.TakeSnapshotToDir(snapDir) + snapshots, _, err := c.nodes[id].dm.TakeSnapshotToDir(snapDir) require.NoError(t, err) require.NotEmpty(t, snapshots) snapDirByNode[id] = snapDir diff --git a/test/autoinc_membership_cluster_test.go b/test/autoinc_membership_cluster_test.go index 125136bd..584b845f 100644 --- a/test/autoinc_membership_cluster_test.go +++ b/test/autoinc_membership_cluster_test.go @@ -44,6 +44,7 @@ func openTimedNodeDatabase(t *testing.T, harness *ClusterHarness, nodeID int, da func execTimed(conn *sql.DB, query string, args ...interface{}) (sql.Result, error) { ctx, cancel := context.WithTimeout(context.Background(), clusterQueryTimeout) defer cancel() + defer noteClientCall() return conn.ExecContext(ctx, query, args...) } @@ -108,6 +109,7 @@ func runAutoIncSync(harness *ClusterHarness, nodeID int) (autoIncSyncAnswer, err } req.Header.Set("X-Marmot-Secret", "test-secret") resp, err := (&http.Client{Timeout: 90 * time.Second}).Do(req) + noteClientCall() if err != nil { return autoIncSyncAnswer{}, err } @@ -213,12 +215,12 @@ func insertOnEvery(t *testing.T, harness *ClusterHarness, ledger idLedger, table } } -// TestNarrowAutoInc_HeldNodeReleasesAfterItsSeedReturns is reviewer B's C2 -// (b) and (d) - a wiped node whose only seed is down, so its join fails and -// its catch-up strategy cannot use the seed - and reviewer A's F-L1: the node -// is held while it cannot reach enough members, and releases once its seed -// is back. Before the R3c-14 fix it never connected to the returned seed, -// which it had learned of as ALIVE from another member, and stayed held. +// TestNarrowAutoInc_HeldNodeReleasesAfterItsSeedReturns: a wiped node whose +// only seed is down, so its join fails and its catch-up strategy cannot use +// the seed, is held while it cannot reach enough members, and releases once +// its seed is back. That needs it to connect to the returned seed, which it +// learned of as ALIVE from another member; a node that never connected +// stayed held. // // Mutations: never hold (both hold sites) - "was not held while its seed was // down" fires; drop the ALIVE callback on discovery (node_registry.go) - "did @@ -337,8 +339,7 @@ func waitForCompleteSync(t *testing.T, harness *ClusterHarness, nodeID int, rest } } -// TestNarrowAutoInc_WipedSeedAndItsJoinerHold is reviewer B's C2 (a) and -// (c): the seed node, which has no seeds of its own, loses its data +// TestNarrowAutoInc_WipedSeedAndItsJoinerHold: the seed node, which has no seeds of its own, loses its data // directory, and so does a node whose only seed it is, which therefore // catches up from an empty seed. Both restart held and release only by // merging claim bases from the member that kept them, and the sync command @@ -386,7 +387,7 @@ func TestNarrowAutoInc_WipedSeedAndItsJoinerHold(t *testing.T) { } // TestNarrowAutoInc_SyncAfterAddingAMember is the membership-change procedure -// (R3c-8b) on a growing cluster: a fourth node joins, becomes ALIVE and +// on a growing cluster: a fourth node joins, becomes ALIVE and // releases, the operator runs the sync command, and it reports every member // answered, unheld and reaching every member. Ids stay unique across all four. // diff --git a/test/autoinc_narrow_cluster_test.go b/test/autoinc_narrow_cluster_test.go index 017e0918..8fe4adf5 100644 --- a/test/autoinc_narrow_cluster_test.go +++ b/test/autoinc_narrow_cluster_test.go @@ -23,10 +23,12 @@ import ( const narrowInt32Max = 2147483647 -// startNarrowCluster starts a 3-node cluster and creates table on it. -func startNarrowCluster(t *testing.T, table, ddl string) *ClusterHarness { +// startNarrowCluster starts a 3-node cluster and creates table on it. opts +// configure the harness (e.g. GCIntervalSeconds) before any node config is +// written. +func startNarrowCluster(t *testing.T, table, ddl string, opts ...func(*ClusterHarness)) *ClusterHarness { t.Helper() - harness := NewClusterHarness(t) + harness := NewClusterHarness(t, opts...) if err := harness.StartCluster(); err != nil { harness.Cleanup() t.Fatalf("StartCluster: %v", err) diff --git a/test/cluster_integration_test.go b/test/cluster_integration_test.go index 442bd157..03e3af33 100644 --- a/test/cluster_integration_test.go +++ b/test/cluster_integration_test.go @@ -39,7 +39,6 @@ type testNode struct { antiEntropy *marmotgrpc.AntiEntropyService writeCoord *coordinator.WriteCoordinator readCoord *coordinator.ReadCoordinator - deltaSync *marmotgrpc.DeltaSyncClient catchUpClient *marmotgrpc.CatchUpClient } @@ -400,9 +399,9 @@ func startNode(t *testing.T, node *testNode, seedNodes []string) { node.dbMgr = dbMgr // Get system database for schema versioning - systemDB, err := dbMgr.GetDatabase(db.SystemDatabaseName) + _, err = dbMgr.GetDatabase(db.SystemDatabaseName) require.NoError(t, err, "Failed to get system database for node %d", node.nodeID) - schemaVersionMgr := db.NewSchemaVersionManager(systemDB.GetMetaStore()) + schemaVersionMgr := db.NewSchemaVersionManager(dbMgr) // Wire up replication handlers replicationHandler := marmotgrpc.NewReplicationHandler( @@ -424,29 +423,23 @@ func startNode(t *testing.T, node *testNode, seedNodes []string) { ) node.catchUpClient.SetDatabaseManager(dbMgr) - node.deltaSync = marmotgrpc.NewDeltaSyncClient(marmotgrpc.DeltaSyncConfig{ - NodeID: node.nodeID, - Client: client, - DBManager: dbMgr, - Clock: node.clock, - ApplyTxnsFn: replicationHandler.HandleReplicateTransaction, - SchemaVersionMgr: schemaVersionMgr, + logPuller := marmotgrpc.NewLogPuller(marmotgrpc.LogPullerConfig{ + NodeID: node.nodeID, + Client: client, + DBManager: dbMgr, }) snapshotFunc := func(ctx context.Context, peerNodeID uint64, peerAddr string, database string) error { return node.catchUpClient.CatchUpFromPeer(ctx, peerNodeID, peerAddr, database) } - schemaVersionMgr = db.NewSchemaVersionManager(systemDB.GetMetaStore()) node.antiEntropy = marmotgrpc.NewAntiEntropyServiceFromConfig( node.nodeID, grpcServer.GetNodeRegistry(), client, dbMgr, - node.deltaSync, - node.clock, + logPuller, snapshotFunc, - schemaVersionMgr, ) node.antiEntropy.Start() diff --git a/test/crash_recovery_test.go b/test/crash_recovery_test.go index 5e8b77bc..917f5289 100644 --- a/test/crash_recovery_test.go +++ b/test/crash_recovery_test.go @@ -14,16 +14,54 @@ import ( "strconv" "strings" "sync" + "sync/atomic" + "syscall" "testing" "time" _ "github.com/go-sql-driver/mysql" + + "github.com/maxpert/marmot/test/stallwatch" ) const ( baseGRPCPort = 8080 baseMySQLPort = 3306 numNodes = 3 + + // clusterStallTimeout bounds how long a cluster may go with no progress: + // no file written under any node's directory (logs, SQLite, meta store), + // no client call returned (noteClientCall) and no progress reported by + // the test (ClusterHarness.Progress). + clusterStallTimeout = 20 * time.Second + // clusterStallPoll is how often the stall watchdog looks. + clusterStallPoll = 250 * time.Millisecond + // clusterReadyDeadline bounds StartCluster's wait for each node to be + // ready for writes (WaitForReady). + clusterReadyDeadline = 20 * time.Second + // clusterConvergencePoll is how often WaitForClusterConvergence asks a + // node for its membership. + clusterConvergencePoll = 250 * time.Millisecond + // stallLogTailLines is how much of each node's log a stall report shows. + stallLogTailLines = 30 + // stallStackWait bounds the wait for a node to write its goroutine dump. + stallStackWait = 3 * time.Second + + // clusterTestBudget is the hard wall-clock budget for any test built on + // ClusterHarness (user rule: every cluster test finishes in <60s). It is + // enforced independently of the stall watchdog: a test can keep making + // progress (writes, restarts) and still blow well past a sane budget. + clusterTestBudget = 60 * time.Second + + // defaultGCIntervalSeconds is the harness's [replication] gc_interval_seconds. + defaultGCIntervalSeconds = 30 + // defaultHeartbeatTimeoutSeconds is the harness's [transaction] + // heartbeat_timeout_seconds. + defaultHeartbeatTimeoutSeconds = 10 + // unsetOverride marks a ClusterHarness int override field as "use the + // harness default" (0 is a real value for some of these fields, so it + // cannot double as the sentinel). + unsetOverride = -1 ) var ( @@ -101,10 +139,49 @@ type ClusterHarness struct { MarmotBin string t *testing.T cleanupOnce sync.Once + watchdog *stallwatch.Watchdog + + // NodeBin overrides MarmotBin for one node id, so a test can run a mix of + // binaries (e.g. a rolling-upgrade test). Set it before StartNode. + NodeBin map[int]string + + // GCIntervalSeconds overrides [replication] gc_interval_seconds on every + // node's config. unsetOverride (the default) keeps defaultGCIntervalSeconds. + GCIntervalSeconds int + // GCMinRetentionHours overrides [replication] gc_min_retention_hours. + // unsetOverride keeps the config's own default (2h). + GCMinRetentionHours int + // DeltaSyncThresholdSeconds overrides [replication] delta_sync_threshold_seconds. + // unsetOverride keeps the config's own default (3600s). A live-GC test + // needs this lowered together with GCMinRetentionHours: cfg.Validate + // requires gc_min_retention_hours >= delta_sync_threshold_seconds in hours. + DeltaSyncThresholdSeconds int + // HeartbeatTimeoutSeconds overrides [transaction] heartbeat_timeout_seconds, + // after which the stale-transaction GC aborts a PENDING transaction. + // unsetOverride keeps defaultHeartbeatTimeoutSeconds. + HeartbeatTimeoutSeconds int + + // budgetStart is when the harness (and so the test using it) began. + budgetStart time.Time + budgetDone chan struct{} + budgetOnce sync.Once + + phaseMu sync.Mutex + lastPhase time.Time + phases []phaseRecord } -// NewClusterHarness creates a new cluster harness for testing -func NewClusterHarness(t *testing.T) *ClusterHarness { +// phaseRecord is one named phase of a cluster test and how long it took +// since the previous phase (or since the harness was created). +type phaseRecord struct { + name string + elapsed time.Duration +} + +// NewClusterHarness creates a new cluster harness for testing. Options run +// before any node config is written, so they can set GCIntervalSeconds and +// friends. +func NewClusterHarness(t *testing.T, opts ...func(*ClusterHarness)) *ClusterHarness { cleanupStalePorts() baseDir := filepath.Join(testDataRoot, fmt.Sprintf("marmot_crash_test_%d", time.Now().UnixNano())) @@ -113,21 +190,157 @@ func NewClusterHarness(t *testing.T) *ClusterHarness { t.Fatalf("Failed to build Marmot: %v", err) } + now := time.Now() harness := &ClusterHarness{ - Nodes: make([]*ClusterNode, numNodes), - BaseDir: baseDir, - MarmotBin: marmotBin, - t: t, + Nodes: make([]*ClusterNode, numNodes), + BaseDir: baseDir, + MarmotBin: marmotBin, + t: t, + GCIntervalSeconds: unsetOverride, + HeartbeatTimeoutSeconds: unsetOverride, + GCMinRetentionHours: unsetOverride, + DeltaSyncThresholdSeconds: unsetOverride, + budgetStart: now, + budgetDone: make(chan struct{}), + lastPhase: now, + } + for _, opt := range opts { + opt(harness) } for i := 0; i < numNodes; i++ { node := harness.createNode(i + 1) harness.Nodes[i] = node } + harness.watchdog = stallwatch.Start(clusterStallTimeout, clusterStallPoll, harness.activity, harness.reportStall) + go harness.watchBudget() return harness } +// Progress tells the stall watchdog the test saw the cluster make progress, +// such as an ACKed write. +func (h *ClusterHarness) Progress() { + h.watchdog.Progress() +} + +// Phase logs how long this phase of the test took (since the previous phase, +// or since the harness was created) and records it for the budget report. +// Call it at each phase of a cluster test: start, load, kill/restart, +// converge, verify, ... +func (h *ClusterHarness) Phase(name string) { + h.phaseMu.Lock() + now := time.Now() + elapsed := now.Sub(h.lastPhase) + h.lastPhase = now + h.phases = append(h.phases, phaseRecord{name: name, elapsed: elapsed}) + h.phaseMu.Unlock() + h.t.Logf("phase=%s elapsed=%s", name, elapsed.Round(time.Millisecond)) +} + +// watchBudget fails the test once the harness has lived past clusterTestBudget, +// independently of the stall watchdog: a test can keep making progress and +// still blow well past the budget the user rule sets. +func (h *ClusterHarness) watchBudget() { + timer := time.NewTimer(clusterTestBudget) + defer timer.Stop() + select { + case <-timer.C: + h.reportBudgetExceeded() + case <-h.budgetDone: + } +} + +// reportBudgetExceeded fails the test with every recorded phase's elapsed +// time and every node's log tail, then stops the nodes so the test's own +// calls fail at once. A goroutine cannot call t.Fatalf, so this mirrors +// reportStall's pattern (t.Errorf + StopCluster). +func (h *ClusterHarness) reportBudgetExceeded() { + h.t.Errorf("cluster test exceeded its %s budget", clusterTestBudget) + h.phaseMu.Lock() + for _, p := range h.phases { + h.t.Logf("phase=%s elapsed=%s", p.name, p.elapsed.Round(time.Millisecond)) + } + h.phaseMu.Unlock() + for i := range h.Nodes { + nodeID := i + 1 + h.t.Logf("=== node %d: last %d log lines ===\n%s", nodeID, stallLogTailLines, h.getNodeLogTail(nodeID, stallLogTailLines)) + } + h.StopCluster() +} + +// clientCalls counts client calls the tests' shared helpers completed +// (noteClientCall); the stall watchdog counts each as progress. +var clientCalls atomic.Int64 + +// noteClientCall records that one client call to a node returned, with or +// without an error: a cluster answering its clients is not stalled. +func noteClientCall() { + clientCalls.Add(1) +} + +// activity digests everything the stall watchdog counts as progress: every +// file write under the harness's directory (size and modification time, so +// a write into a preallocated file counts too) and every returned client +// call. +func (h *ClusterHarness) activity() int64 { + total := clientCalls.Load() + _ = filepath.WalkDir(h.BaseDir, func(_ string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + if info, err := d.Info(); err == nil { + total += info.Size() + info.ModTime().UnixNano() + } + return nil + }) + return total +} + +// reportStall fails the test for a cluster that made no progress for +// clusterStallTimeout: it shows each node's log tail, has every running node +// dump its goroutines (SIGQUIT, into its log) and shows them, and stops the +// nodes so the test's own calls fail at once. +func (h *ClusterHarness) reportStall(idle time.Duration) { + h.t.Errorf("cluster stalled: no node wrote anything, no client call returned and the test reported no progress for %s", idle.Round(time.Second)) + for i, node := range h.Nodes { + nodeID := i + 1 + h.t.Logf("=== node %d: last %d log lines ===\n%s", nodeID, stallLogTailLines, h.getNodeLogTail(nodeID, stallLogTailLines)) + node.mu.Lock() + running, cmd := node.isRunning, node.Cmd + node.mu.Unlock() + if !running || cmd == nil || cmd.Process == nil { + continue + } + before, err := os.Stat(node.LogFile) + if err != nil { + continue + } + if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil { + h.t.Logf("node %d: SIGQUIT: %v", nodeID, err) + continue + } + deadline := time.Now().Add(stallStackWait) + for time.Now().Before(deadline) { + if after, err := os.Stat(node.LogFile); err == nil && after.Size() > before.Size() { + break + } + time.Sleep(100 * time.Millisecond) + } + time.Sleep(100 * time.Millisecond) + data, err := os.ReadFile(node.LogFile) + if err != nil || int64(len(data)) <= before.Size() { + h.t.Logf("node %d wrote no goroutine dump", nodeID) + continue + } + stackPath := filepath.Join(node.DataDir, "stall-stacks.txt") + if err := os.WriteFile(stackPath, data[before.Size():], 0o644); err == nil { + h.t.Logf("node %d goroutine dump: %s", nodeID, stackPath) + } + } + h.StopCluster() +} + // testDataRoot holds the integration tests' node data and binaries: Marmot's // runtime files live under /tmp/marmot. const testDataRoot = "/tmp/marmot/test" @@ -200,14 +413,28 @@ func (h *ClusterHarness) createNodeConfig(node *ClusterNode) { seedNodes = append(seedNodes, fmt.Sprintf("\"localhost:%d\"", baseGRPCPort+1)) } + gcInterval := defaultGCIntervalSeconds + if h.GCIntervalSeconds != unsetOverride { + gcInterval = h.GCIntervalSeconds + } + heartbeatTimeout := defaultHeartbeatTimeoutSeconds + if h.HeartbeatTimeoutSeconds != unsetOverride { + heartbeatTimeout = h.HeartbeatTimeoutSeconds + } + gcMinRetentionLine, deltaSyncThresholdLine := "", "" + if h.GCMinRetentionHours != unsetOverride { + gcMinRetentionLine = fmt.Sprintf("gc_min_retention_hours = %d\n", h.GCMinRetentionHours) + } + if h.DeltaSyncThresholdSeconds != unsetOverride { + deltaSyncThresholdLine = fmt.Sprintf("delta_sync_threshold_seconds = %d\n", h.DeltaSyncThresholdSeconds) + } + config := fmt.Sprintf(`# Marmot v2.9.16-beta Test Node %d node_id = %d data_dir = "%s" -[mvcc] -gc_interval_seconds = 30 -gc_retention_hours = 1 -heartbeat_timeout_seconds = 10 +[transaction] +heartbeat_timeout_seconds = %d conflict_window_seconds = 10 [connection_pool] @@ -248,7 +475,8 @@ write_timeout_ms = 10000 read_timeout_ms = 5000 enable_anti_entropy = true anti_entropy_interval_seconds = 5 - +gc_interval_seconds = %d +%s%s [mysql] enabled = true bind_address = "0.0.0.0" @@ -265,9 +493,13 @@ enabled = false node.NodeID, node.NodeID, node.DataDir, + heartbeatTimeout, node.GRPCPort, node.GRPCPort, strings.Join(seedNodes, ", "), + gcInterval, + gcMinRetentionLine, + deltaSyncThresholdLine, node.MySQLPort, ) @@ -289,12 +521,19 @@ func (h *ClusterHarness) StartNode(nodeID int) error { h.t.Logf("Starting node %d...", nodeID) - logFile, err := os.Create(node.LogFile) + // Append across restarts: a rolling-restart or kill/restart test needs + // every incarnation's log, not just the last one's. + logFile, err := os.OpenFile(node.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) if err != nil { return fmt.Errorf("failed to create log file: %v", err) } + fmt.Fprintf(logFile, "=== node %d start %s ===\n", nodeID, time.Now().Format(time.RFC3339Nano)) - cmd := exec.Command(h.MarmotBin, "--config", node.ConfigPath) + bin := h.MarmotBin + if b, ok := h.NodeBin[nodeID]; ok { + bin = b + } + cmd := exec.Command(bin, "--config", node.ConfigPath) cmd.Stdout = logFile cmd.Stderr = logFile @@ -339,9 +578,6 @@ func (h *ClusterHarness) StopNode(nodeID int) error { os.Remove(node.PIDFile) - // Small wait to ensure clean shutdown - time.Sleep(1 * time.Second) - h.t.Logf("Node %d stopped", nodeID) return nil } @@ -451,6 +687,7 @@ func (h *ClusterHarness) QueryNode(nodeID int, query string, args ...interface{} if err != nil { return nil, err } + defer noteClientCall() return db.Query(query, args...) } @@ -459,6 +696,7 @@ func (h *ClusterHarness) ExecNode(nodeID int, query string, args ...interface{}) if err != nil { return nil, err } + defer noteClientCall() return db.Exec(query, args...) } @@ -554,6 +792,88 @@ type AdminAPIResponse struct { Data []ClusterMember `json:"data"` } +// votesHeld asks node's admin API whether its AUTO_INCREMENT claim votes are +// held (GET /admin/cluster/autoinc/votes). +func (h *ClusterHarness) votesHeld(nodeID int) (bool, error) { + url := fmt.Sprintf("http://localhost:%d/admin/cluster/autoinc/votes", h.Nodes[nodeID-1].GRPCPort) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return false, err + } + req.Header.Set("X-Marmot-Secret", "test-secret") + resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req) + noteClientCall() + if err != nil { + return false, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return false, fmt.Errorf("status %d: %s", resp.StatusCode, string(body)) + } + var apiResp struct { + Data struct { + VotesHeld bool `json:"votes_held"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil { + return false, err + } + return apiResp.Data.VotesHeld, nil +} + +// WaitForVotesReleased polls until node's AUTO_INCREMENT claim votes are not +// held: a node whose system database is new declines every claim at PREPARE +// until it merged claim bases from enough members, so a narrow +// AUTO_INCREMENT insert right after a cluster's first start can return 1205 +// until then. +func (h *ClusterHarness) WaitForVotesReleased(nodeID int, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + h.Progress() + held, err := h.votesHeld(nodeID) + if err == nil && !held { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("node %d claim votes still held after %v (err=%v); log tail:\n%s", nodeID, timeout, err, h.getNodeLogTail(nodeID, 40)) + } + time.Sleep(clusterConvergencePoll) + } +} + +// WaitForReady polls until node is ready for any write: it reports itself +// ALIVE (a JOINING node is still catching up) and its claim votes are not +// held (WaitForVotesReleased), both within one named deadline. +func (h *ClusterHarness) WaitForReady(nodeID int, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + h.Progress() + if status, err := h.selfStatus(nodeID); err == nil && status == "ALIVE" { + break + } + if time.Now().After(deadline) { + return fmt.Errorf("node %d not ALIVE within %v; log tail:\n%s", nodeID, timeout, h.getNodeLogTail(nodeID, 40)) + } + time.Sleep(clusterConvergencePoll) + } + return h.WaitForVotesReleased(nodeID, time.Until(deadline)) +} + +// selfStatus is node's own membership status as its admin API reports it. +func (h *ClusterHarness) selfStatus(nodeID int) (string, error) { + members, err := h.getClusterStatus(nodeID) + if err != nil { + return "", err + } + for _, m := range members { + if m.NodeID == uint64(nodeID) { + return m.Status, nil + } + } + return "", fmt.Errorf("node %d absent from its own membership", nodeID) +} + // getClusterStatus fetches cluster status from a node's admin API func (h *ClusterHarness) getClusterStatus(nodeID int) ([]ClusterMember, error) { node := h.Nodes[nodeID-1] @@ -567,6 +887,7 @@ func (h *ClusterHarness) getClusterStatus(nodeID int) ([]ClusterMember, error) { client := &http.Client{Timeout: 5 * time.Second} resp, err := client.Do(req) + noteClientCall() if err != nil { return nil, err } @@ -585,43 +906,42 @@ func (h *ClusterHarness) getClusterStatus(nodeID int) ([]ClusterMember, error) { return apiResp.Data, nil } -// WaitForClusterConvergence waits for all nodes to be ALIVE according to cluster membership +// WaitForClusterConvergence waits until every node sees every node ALIVE +// in its cluster membership, polling each node's admin API until timeout. func (h *ClusterHarness) WaitForClusterConvergence(timeout time.Duration) error { - h.t.Logf("Waiting for cluster to converge (all nodes ALIVE)...") + h.t.Logf("Waiting for cluster to converge (all nodes ALIVE on every node)...") deadline := time.Now().Add(timeout) - errorLogged := false - for time.Now().Before(deadline) { - // Check from node 1's perspective - members, err := h.getClusterStatus(1) - if err != nil { - if !errorLogged { - h.t.Logf("Error getting cluster status (will retry): %v", err) - errorLogged = true + for node := 1; node <= numNodes; node++ { + for { + alive, err := h.aliveCount(node) + if err == nil && alive >= numNodes { + break } - time.Sleep(500 * time.Millisecond) - continue - } - errorLogged = false - - // Count ALIVE nodes - aliveCount := 0 - for _, m := range members { - if m.Status == "ALIVE" { - aliveCount++ + if time.Now().After(deadline) { + return fmt.Errorf("cluster did not converge on node %d within %v (alive=%d, err=%v)", node, timeout, alive, err) } + time.Sleep(clusterConvergencePoll) } + } + h.t.Logf("Cluster converged: every node sees %d nodes ALIVE", numNodes) + return nil +} - if aliveCount >= numNodes { - h.t.Logf("Cluster converged: %d nodes ALIVE", aliveCount) - return nil +// aliveCount returns how many members node's cluster membership reports +// ALIVE. +func (h *ClusterHarness) aliveCount(node int) (int, error) { + members, err := h.getClusterStatus(node) + if err != nil { + return 0, err + } + alive := 0 + for _, m := range members { + if m.Status == "ALIVE" { + alive++ } - - h.t.Logf("Cluster not yet converged: %d/%d nodes ALIVE", aliveCount, numNodes) - time.Sleep(1 * time.Second) } - - return fmt.Errorf("cluster did not converge within %v", timeout) + return alive, nil } // dumpNodeLogs saves node logs to /tmp for debugging @@ -663,22 +983,31 @@ func (h *ClusterHarness) StartCluster() error { } h.t.Logf("Waiting for MySQL servers to be ready...") - time.Sleep(3 * time.Second) - for i := 1; i <= numNodes; i++ { if err := h.WaitForAlive(i, 30*time.Second); err != nil { return err } } - // Wait for gossip cluster to fully converge (all nodes ALIVE) + // Wait for gossip cluster to fully converge (all nodes ALIVE, as seen + // by every node). if err := h.WaitForClusterConvergence(30 * time.Second); err != nil { h.dumpNodeLogs("cluster_start_fail") return err } - // Additional wait for gRPC connections to stabilize - time.Sleep(2 * time.Second) + // Every node on the harness's own binary must be ready for writes. A + // node on an overridden binary (NodeBin) may predate the readiness + // endpoint; a test mixing binaries waits for readiness itself. + for i := 1; i <= numNodes; i++ { + if _, overridden := h.NodeBin[i]; overridden { + continue + } + if err := h.WaitForReady(i, clusterReadyDeadline); err != nil { + h.dumpNodeLogs("cluster_start_not_ready") + return err + } + } h.t.Logf("Cluster started successfully") return nil @@ -693,6 +1022,8 @@ func (h *ClusterHarness) StopCluster() { func (h *ClusterHarness) Cleanup() { h.cleanupOnce.Do(func() { + h.watchdog.Stop() + h.budgetOnce.Do(func() { close(h.budgetDone) }) h.t.Logf("Cleaning up cluster harness...") h.StopCluster() time.Sleep(500 * time.Millisecond) @@ -733,6 +1064,7 @@ func TestDDLReplication(t *testing.T) { harness.dumpNodeLogs("ddl_test") t.Fatalf("DDL replication failed: %v", err) } + harness.Phase("converged") t.Logf("SUCCESS: Table %s replicated to all nodes", tableName) } @@ -745,6 +1077,7 @@ func TestBasicReplication(t *testing.T) { if err := harness.StartCluster(); err != nil { t.Fatalf("Failed to start cluster: %v", err) } + harness.Phase("start") tableName := "basic_test" @@ -772,16 +1105,14 @@ func TestBasicReplication(t *testing.T) { } // Wait for replication - time.Sleep(3 * time.Second) + if err := harness.WaitForRowCount(tableName, allNodes, 10, 15*time.Second); err != nil { + harness.dumpNodeLogs("basic_replication_fail") + t.Fatalf("rows did not replicate: %v", err) + } + harness.Phase("converged") - // Verify all nodes have the data for _, nodeID := range allNodes { - count := harness.getRowCount(nodeID, tableName) - if count != 10 { - harness.dumpNodeLogs("basic_replication_fail") - t.Fatalf("Node %d has %d rows, expected 10", nodeID, count) - } - t.Logf("Node %d has %d rows", nodeID, count) + t.Logf("Node %d has %d rows", nodeID, harness.getRowCount(nodeID, tableName)) } t.Logf("SUCCESS: Basic replication verified") @@ -795,6 +1126,7 @@ func TestNodeRestartRecovery(t *testing.T) { if err := harness.StartCluster(); err != nil { t.Fatalf("Failed to start cluster: %v", err) } + harness.Phase("start") tableName := "recovery_test" @@ -820,22 +1152,18 @@ func TestNodeRestartRecovery(t *testing.T) { t.Fatalf("Insert failed: %v", err) } } - time.Sleep(2 * time.Second) - - // Verify all nodes have initial data - for _, nodeID := range allNodes { - count := harness.getRowCount(nodeID, tableName) - if count != 5 { - t.Fatalf("Node %d has %d rows before crash, expected 5", nodeID, count) - } + if err := harness.WaitForRowCount(tableName, allNodes, 5, 15*time.Second); err != nil { + t.Fatalf("initial rows did not replicate: %v", err) } t.Logf("All nodes have 5 rows before crash") + harness.Phase("initial-load") // Kill node 3 t.Logf("Killing node 3...") if err := harness.KillNode(3); err != nil { t.Fatalf("Failed to kill node 3: %v", err) } + harness.Phase("node3-killed") // Write more data while node 3 is down t.Logf("Writing 5 more rows while node 3 is down...") @@ -846,16 +1174,11 @@ func TestNodeRestartRecovery(t *testing.T) { t.Fatalf("Insert failed while node 3 down: %v", err) } } - time.Sleep(2 * time.Second) - - // Verify nodes 1 & 2 have all data - for _, nodeID := range []int{1, 2} { - count := harness.getRowCount(nodeID, tableName) - if count != 10 { - t.Fatalf("Node %d has %d rows, expected 10", nodeID, count) - } + if err := harness.WaitForRowCount(tableName, []int{1, 2}, 10, 15*time.Second); err != nil { + t.Fatalf("rows did not replicate to nodes 1&2: %v", err) } t.Logf("Nodes 1 & 2 have 10 rows") + harness.Phase("missed-load") // Restart node 3 t.Logf("Restarting node 3...") @@ -865,23 +1188,15 @@ func TestNodeRestartRecovery(t *testing.T) { if err := harness.WaitForAlive(3, 20*time.Second); err != nil { t.Fatalf("Node 3 did not become ALIVE: %v", err) } + harness.Phase("node3-alive") // Wait for anti-entropy to catch up t.Logf("Waiting for node 3 to catch up via anti-entropy...") - var count3 int - for attempt := 0; attempt < 12; attempt++ { - time.Sleep(5 * time.Second) - count3 = harness.getRowCount(3, tableName) - t.Logf("Attempt %d: Node 3 has %d rows", attempt+1, count3) - if count3 == 10 { - break - } - } - - if count3 != 10 { + if err := harness.WaitForRowCount(tableName, []int{3}, 10, 20*time.Second); err != nil { harness.dumpNodeLogs("recovery_test_fail") - t.Fatalf("Node 3 did not catch up: has %d rows, expected 10", count3) + t.Fatalf("Node 3 did not catch up: %v", err) } + harness.Phase("caught-up") t.Logf("SUCCESS: Node 3 caught up after restart") } @@ -890,6 +1205,10 @@ func TestNodeRestartRecovery(t *testing.T) { // This test verifies that: // 1. Writes continue to succeed (on remaining nodes) during rolling restarts // 2. After all restarts complete, anti-entropy brings all nodes to eventual consistency +// +// Every wait here is a poll on a named deadline (WaitForRowCount / WaitForAlive), +// not a flat sleep, so the test finishes as soon as the cluster actually +// converges, keeping it well under the 60 s cluster test budget. func TestRollingRestart(t *testing.T) { harness := NewClusterHarness(t) defer harness.Cleanup() @@ -897,6 +1216,7 @@ func TestRollingRestart(t *testing.T) { if err := harness.StartCluster(); err != nil { t.Fatalf("Failed to start cluster: %v", err) } + harness.Phase("start") tableName := "rolling_test" @@ -914,26 +1234,23 @@ func TestRollingRestart(t *testing.T) { } // Insert baseline data before rolling restart - t.Logf("Inserting baseline 20 rows before rolling restart...") - for i := 1; i <= 20; i++ { + const baseline = 20 + t.Logf("Inserting baseline %d rows before rolling restart...", baseline) + for i := 1; i <= baseline; i++ { _, err := harness.ExecNode(1, fmt.Sprintf( "INSERT INTO %s (id, value) VALUES (%d, 'baseline_%d')", tableName, i, i)) if err != nil { t.Fatalf("Baseline insert failed: %v", err) } } - time.Sleep(3 * time.Second) - - // Verify all nodes have baseline data - for _, nodeID := range allNodes { - count := harness.getRowCount(nodeID, tableName) - if count != 20 { - t.Fatalf("Node %d has %d baseline rows, expected 20", nodeID, count) - } + if err := harness.WaitForRowCount(tableName, allNodes, baseline, 15*time.Second); err != nil { + t.Fatalf("baseline rows did not replicate: %v", err) } - t.Logf("All nodes have 20 baseline rows") + t.Logf("All nodes have %d baseline rows", baseline) + harness.Phase("baseline-load") - // Rolling restart each node - no writes during restart to test pure recovery + // Rolling restart each node - writes continue against the nodes still up. + expected := baseline for nodeID := 1; nodeID <= 3; nodeID++ { t.Logf("Rolling restart: stopping node %d...", nodeID) if err := harness.StopNode(nodeID); err != nil { @@ -947,14 +1264,25 @@ func TestRollingRestart(t *testing.T) { } t.Logf("Writing 5 rows to node %d while node %d is down...", targetNode, nodeID) for i := 0; i < 5; i++ { - rowID := 20 + (nodeID-1)*5 + i + 1 // Unique IDs: 21-25, 26-30, 31-35 + rowID := baseline + (nodeID-1)*5 + i + 1 // Unique IDs: 21-25, 26-30, 31-35 _, err := harness.ExecNode(targetNode, fmt.Sprintf( "INSERT INTO %s (id, value) VALUES (%d, 'rolling_%d')", tableName, rowID, rowID)) if err != nil { t.Logf("Warning: write during rolling restart failed: %v", err) + continue } + expected++ + } + + var stillUp []int + for _, n := range allNodes { + if n != nodeID { + stillUp = append(stillUp, n) + } + } + if err := harness.WaitForRowCount(tableName, stillUp, expected, 10*time.Second); err != nil { + t.Fatalf("writes during node %d's downtime did not land on the up nodes: %v", nodeID, err) } - time.Sleep(2 * time.Second) if err := harness.StartNode(nodeID); err != nil { t.Fatalf("Failed to restart node %d: %v", nodeID, err) @@ -964,42 +1292,22 @@ func TestRollingRestart(t *testing.T) { } t.Logf("Node %d restarted", nodeID) - // Wait for anti-entropy to sync the restarted node - time.Sleep(10 * time.Second) - } - - // Wait for eventual consistency across all nodes - t.Logf("Waiting for eventual consistency...") - var converged bool - var counts [3]int - - for attempt := 0; attempt < 12; attempt++ { - time.Sleep(5 * time.Second) - - for i := 1; i <= 3; i++ { - counts[i-1] = harness.getRowCount(i, tableName) - } - t.Logf("Attempt %d: node1=%d, node2=%d, node3=%d", attempt+1, counts[0], counts[1], counts[2]) - - // Check if all nodes have the same count - if counts[0] == counts[1] && counts[1] == counts[2] && counts[0] > 0 { - converged = true - break + if err := harness.WaitForRowCount(tableName, []int{nodeID}, expected, 20*time.Second); err != nil { + harness.dumpNodeLogs("rolling_test_catchup") + t.Fatalf("node %d did not catch up via anti-entropy: %v", nodeID, err) } + harness.Phase(fmt.Sprintf("node%d-rolled", nodeID)) } - if !converged { + // Final verification - all nodes converge on the same count. + if err := harness.WaitForRowCount(tableName, allNodes, expected, 15*time.Second); err != nil { harness.dumpNodeLogs("rolling_test_fail") - t.Fatalf("Cluster did not converge: node1=%d, node2=%d, node3=%d", counts[0], counts[1], counts[2]) - } - - // Verify we have baseline + rolling restart writes (20 baseline + up to 15 rolling) - expectedMin := 20 // At least baseline data - if counts[0] < expectedMin { - t.Fatalf("Expected at least %d rows, got %d", expectedMin, counts[0]) + t.Fatalf("cluster did not converge: %v", err) } + harness.Phase("converged") - t.Logf("SUCCESS: Rolling restart maintained availability and consistency (final count: %d)", counts[0]) + finalCount := harness.getRowCount(1, tableName) + t.Logf("SUCCESS: Rolling restart maintained availability and consistency (final count: %d)", finalCount) } // TestCrashMidTransaction tests that uncommitted transactions are rolled back @@ -1010,6 +1318,7 @@ func TestCrashMidTransaction(t *testing.T) { if err := harness.StartCluster(); err != nil { t.Fatalf("Failed to start cluster: %v", err) } + harness.Phase("start") tableName := "crash_txn_test" @@ -1071,6 +1380,7 @@ func TestCrashMidTransaction(t *testing.T) { if err := harness.WaitForAlive(1, 15*time.Second); err != nil { t.Fatalf("Node 1 did not become ALIVE: %v", err) } + harness.Phase("node1-alive") time.Sleep(3 * time.Second) @@ -1091,6 +1401,7 @@ func TestHighLoadRecovery(t *testing.T) { if err := harness.StartCluster(); err != nil { t.Fatalf("Failed to start cluster: %v", err) } + harness.Phase("start") tableName := "highload_test" allNodes := []int{1, 2, 3} @@ -1123,10 +1434,12 @@ func TestHighLoadRecovery(t *testing.T) { t.Fatalf("Initial replication failed: %v", err) } t.Logf("Phase 1 complete: All nodes have %d rows", initialRows) + harness.Phase("initial-load") // Phase 2: Kill node 3 t.Logf("Phase 2: Killing node 3...") harness.KillNode(3) + harness.Phase("node3-killed") // Phase 3: Insert more rows while node 3 is down additionalRows := 30 @@ -1146,6 +1459,7 @@ func TestHighLoadRecovery(t *testing.T) { t.Fatalf("Replication to nodes 1&2 failed: %v", err) } t.Logf("Phase 3 complete: Nodes 1 & 2 have %d rows", expectedTotal) + harness.Phase("missed-load") // Phase 4: Restart node 3 t.Logf("Phase 4: Restarting node 3...") @@ -1156,13 +1470,15 @@ func TestHighLoadRecovery(t *testing.T) { harness.dumpNodeLogs("highload_restart_alive") t.Fatalf("Node 3 did not become ALIVE: %v", err) } + harness.Phase("node3-alive") // Phase 5: Wait for node 3 to catch up t.Logf("Phase 5: Waiting for node 3 to catch up to %d rows...", expectedTotal) - if err := harness.WaitForRowCount(tableName, []int{3}, expectedTotal, 60*time.Second); err != nil { + if err := harness.WaitForRowCount(tableName, []int{3}, expectedTotal, 25*time.Second); err != nil { harness.dumpNodeLogs("highload_catchup") t.Fatalf("Node 3 failed to catch up: %v", err) } + harness.Phase("caught-up") // Final verification - all nodes have exact same count for _, nodeID := range allNodes { diff --git a/test/ddl_replication_test.go b/test/ddl_replication_test.go index a9e13dcc..ba0beeb9 100644 --- a/test/ddl_replication_test.go +++ b/test/ddl_replication_test.go @@ -110,6 +110,16 @@ func (m *DDLMockDatabaseManager) GetVectorIndexManager() coordinator.VectorIndex return nil } +// RegistryKeyGeneration reports generation 1 for any database this mock is +// tracking (live) and 0 for any it is not (never seen); the DDL mock has no +// tombstone/generation history of its own to model. +func (m *DDLMockDatabaseManager) RegistryKeyGeneration(name string) (uint64, bool, error) { + if m.DatabaseExists(name) { + return 1, true, nil + } + return 0, false, nil +} + // DDLMockReader implements coordinator.Reader for testing type DDLMockReader struct{} @@ -130,6 +140,11 @@ func (m *MockNodeRegistry) GetAll() []any { re func (m *MockNodeRegistry) IsLeaving(nodeID uint64) bool { return false } func (m *MockNodeRegistry) GetLocalNodeID() uint64 { return 0 } +// LegacyLogProtocolMembers reports no legacy members: this mock has no real +// membership, so every member implicitly serves the commit-log pull protocol +// (coordinator.NodeRegistry). +func (m *MockNodeRegistry) LegacyLogProtocolMembers() []uint64 { return nil } + // TestDDLStatementDetection validates that DDL statements are correctly identified func TestDDLStatementDetection(t *testing.T) { tests := []struct { @@ -311,19 +326,45 @@ func TestDDLIdempotencyRewriter(t *testing.T) { } } -// TestSchemaVersionManager validates schema version tracking per database +// commitDDLForTest commits one DDL statement against database through its +// TransactionManager directly (the same WriteIntent+CommitTransaction shape +// db/replication_engine.go's createDDLIntent/Commit use), so its +// __marmot_schema_version bumps for real. Schema versions are no +// longer settable directly - they are always derived from a committed DDL's +// own transaction. +func commitDDLForTest(t *testing.T, dbMgr *db.DatabaseManager, database, ddlSQL, table string) { + t.Helper() + mdb, err := dbMgr.GetDatabase(database) + require.NoError(t, err) + txnMgr := mdb.GetTransactionManager() + + txn, err := txnMgr.BeginTransaction(1) + require.NoError(t, err) + stmt := protocol.Statement{Type: protocol.StatementDDL, SQL: ddlSQL, TableName: table, Database: database} + snapshot, err := db.SerializeData(db.DDLSnapshot{Type: int(stmt.Type), SQL: ddlSQL, TableName: table}) + require.NoError(t, err) + require.NoError(t, txnMgr.WriteIntent(txn, db.IntentTypeDDL, table, "ddl:"+table, stmt, snapshot)) + require.NoError(t, txnMgr.CommitTransaction(txn)) +} + +// TestSchemaVersionManager validates schema version tracking per database: +// each database's own __marmot_schema_version SQLite table, read +// through SchemaVersionManager and bumped atomically by each committed DDL +// transaction - not by a direct setter, which no longer exists. func TestSchemaVersionManager(t *testing.T) { - // Create temp directory for PebbleDB tmpDir, err := testDataDir("schema-version-test-*") require.NoError(t, err) defer os.RemoveAll(tmpDir) - // Create PebbleMetaStore - metaStore, err := db.NewPebbleMetaStore(tmpDir, db.DefaultPebbleOptions()) + clock := hlc.NewClock(1) + dbMgr, err := db.NewDatabaseManager(tmpDir, 1, clock) require.NoError(t, err) - defer metaStore.Close() + defer dbMgr.Close() + + require.NoError(t, dbMgr.CreateDatabase("testdb")) + require.NoError(t, dbMgr.CreateDatabase("otherdb")) - svm := db.NewSchemaVersionManager(metaStore) + svm := db.NewSchemaVersionManager(dbMgr) // Test: Initial version is 0 t.Run("Initial version is 0", func(t *testing.T) { @@ -332,27 +373,23 @@ func TestSchemaVersionManager(t *testing.T) { require.Equal(t, uint64(0), version, "initial version should be 0") }) - // Test: Increment version - t.Run("Increment version", func(t *testing.T) { - newVersion, err := svm.IncrementSchemaVersion("testdb", "CREATE TABLE users (id INT)", 1001) - require.NoError(t, err) - require.Equal(t, uint64(1), newVersion, "version should increment to 1") + // Test: A committed DDL bumps the version + t.Run("Commit bumps version", func(t *testing.T) { + commitDDLForTest(t, dbMgr, "testdb", "CREATE TABLE users (id INT)", "users") - // Verify version persisted version, err := svm.GetSchemaVersion("testdb") require.NoError(t, err) - require.Equal(t, uint64(1), version) + require.Equal(t, uint64(1), version, "version should bump to 1") }) - // Test: Multiple increments - t.Run("Multiple increments", func(t *testing.T) { - newVersion, err := svm.IncrementSchemaVersion("testdb", "ALTER TABLE users ADD COLUMN name TEXT", 1002) - require.NoError(t, err) - require.Equal(t, uint64(2), newVersion) + // Test: Multiple DDL commits + t.Run("Multiple commits", func(t *testing.T) { + commitDDLForTest(t, dbMgr, "testdb", "ALTER TABLE users ADD COLUMN name TEXT", "users") + commitDDLForTest(t, dbMgr, "testdb", "CREATE INDEX idx ON users(name)", "users") - newVersion, err = svm.IncrementSchemaVersion("testdb", "CREATE INDEX idx ON users(name)", 1003) + version, err := svm.GetSchemaVersion("testdb") require.NoError(t, err) - require.Equal(t, uint64(3), newVersion) + require.Equal(t, uint64(3), version) }) // Test: Independent version counters per database @@ -367,10 +404,11 @@ func TestSchemaVersionManager(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(0), version2) - // Increment otherdb - newVersion, err := svm.IncrementSchemaVersion("otherdb", "CREATE TABLE products (id INT)", 2001) + // Bump otherdb + commitDDLForTest(t, dbMgr, "otherdb", "CREATE TABLE products (id INT)", "products") + version2, err = svm.GetSchemaVersion("otherdb") require.NoError(t, err) - require.Equal(t, uint64(1), newVersion) + require.Equal(t, uint64(1), version2) // testdb still at 3 version1, err = svm.GetSchemaVersion("testdb") @@ -472,36 +510,38 @@ func TestDDLLockManager(t *testing.T) { // TestDDLReplicationBasic validates end-to-end DDL execution and replication func TestDDLReplicationBasic(t *testing.T) { // Setup test infrastructure - dbMgr := NewDDLMockDatabaseManager() - - // Create temp directory for PebbleDB + // Create temp directory for the real DatabaseManager. Schema versions are + // now read from each database's own __marmot_schema_version SQLite table, + // so DDL must commit for real through a real DatabaseManager - + // the DDLMockDatabaseManager has no such table. tmpDir, err := testDataDir("ddl-replication-test-*") require.NoError(t, err) defer os.RemoveAll(tmpDir) - // Create PebbleMetaStore - metaStore, err := db.NewPebbleMetaStore(tmpDir, db.DefaultPebbleOptions()) + clock := hlc.NewClock(1) + dbMgr, err := db.NewDatabaseManager(tmpDir, 1, clock) require.NoError(t, err) - defer metaStore.Close() + defer dbMgr.Close() - svm := db.NewSchemaVersionManager(metaStore) + svm := db.NewSchemaVersionManager(dbMgr) lockMgr := coordinator.NewDDLLockManager(5 * time.Second) // Create test database err = dbMgr.CreateDatabase("testdb") require.NoError(t, err) - // Setup coordinators + // Setup coordinators. Remote nodes 2 and 3 stay mocked (MockReplicator); + // only node 1's own commit needs to be real for these assertions, which + // read node 1's own schema version. nodeProvider := &MockNodeProvider{nodes: []uint64{1, 2, 3}} replicator := &MockReplicator{} - clock := hlc.NewClock(1) reader := &DDLMockReader{} writeCoord := coordinator.NewWriteCoordinator( 1, nodeProvider, replicator, - replicator, + db.NewLocalReplicator(1, dbMgr, clock), 1*time.Second, clock, ) @@ -568,20 +608,19 @@ func TestDDLReplicationBasic(t *testing.T) { // TestDDLWithConcurrentDML validates DDL and DML interleaved execution func TestDDLWithConcurrentDML(t *testing.T) { - // Setup test infrastructure - dbMgr := NewDDLMockDatabaseManager() - - // Create temp directory for PebbleDB + // Setup test infrastructure. Schema versions are read from each + // database's own __marmot_schema_version SQLite table, so this + // needs a real DatabaseManager, not DDLMockDatabaseManager. tmpDir, err := testDataDir("ddl-concurrent-dml-test-*") require.NoError(t, err) defer os.RemoveAll(tmpDir) - // Create PebbleMetaStore - metaStore, err := db.NewPebbleMetaStore(tmpDir, db.DefaultPebbleOptions()) + clock := hlc.NewClock(1) + dbMgr, err := db.NewDatabaseManager(tmpDir, 1, clock) require.NoError(t, err) - defer metaStore.Close() + defer dbMgr.Close() - svm := db.NewSchemaVersionManager(metaStore) + svm := db.NewSchemaVersionManager(dbMgr) lockMgr := coordinator.NewDDLLockManager(5 * time.Second) err = dbMgr.CreateDatabase("testdb") @@ -589,14 +628,13 @@ func TestDDLWithConcurrentDML(t *testing.T) { nodeProvider := &MockNodeProvider{nodes: []uint64{1}} replicator := &MockReplicator{} - clock := hlc.NewClock(1) reader := &DDLMockReader{} writeCoord := coordinator.NewWriteCoordinator( 1, nodeProvider, replicator, - replicator, + db.NewLocalReplicator(1, dbMgr, clock), 1*time.Second, clock, ) diff --git a/test/gap_detection_test.go b/test/gap_detection_test.go index 37c46d12..58f3a092 100644 --- a/test/gap_detection_test.go +++ b/test/gap_detection_test.go @@ -26,7 +26,11 @@ import ( // - Node 3 successfully catches up after restart // - Final data consistency across all nodes -// TestLaggingNodeCatchup validates that a lagging node catches up via anti-entropy. +// TestLaggingNodeCatchup validates that a lagging node catches up via +// anti-entropy. Row counts and waits are sized to stay well under the 60 s +// cluster test budget: every wait is a poll on a named deadline (WaitForRowCount +// / WaitForAlive), never a flat sleep, so the test finishes as soon as the +// cluster actually converges. func TestLaggingNodeCatchup(t *testing.T) { harness := NewClusterHarness(t) defer harness.Cleanup() @@ -36,6 +40,7 @@ func TestLaggingNodeCatchup(t *testing.T) { if err := harness.StartCluster(); err != nil { t.Fatalf("Failed to start cluster: %v", err) } + harness.Phase("start") // Create test table t.Logf("Creating test table...") @@ -50,100 +55,58 @@ func TestLaggingNodeCatchup(t *testing.T) { t.Fatalf("Table did not replicate: %v", err) } - // Insert initial rows - t.Logf("Inserting 100 initial rows...") - for i := 1; i <= 100; i++ { + const initialRows = 50 + t.Logf("Inserting %d initial rows...", initialRows) + for i := 1; i <= initialRows; i++ { _, err := harness.ExecNode(1, "INSERT INTO test_lagging (id, value) VALUES (?, ?)", i, fmt.Sprintf("value_%d", i)) if err != nil { t.Fatalf("Failed to insert row %d: %v", i, err) } } - - // Wait for replication - time.Sleep(3 * time.Second) - - // Verify initial data on all nodes - t.Logf("Verifying initial data on all nodes...") - for nodeID := 1; nodeID <= 3; nodeID++ { - count := harness.getRowCount(nodeID, "test_lagging") - if count != 100 { - t.Fatalf("Node %d has %d rows, expected 100", nodeID, count) - } - t.Logf("Node %d: %d rows (OK)", nodeID, count) + if err := harness.WaitForRowCount("test_lagging", allNodes, initialRows, 15*time.Second); err != nil { + t.Fatalf("initial rows did not replicate: %v", err) } + harness.Phase("initial-load") // Kill node 3 t.Logf("Killing node 3...") if err := harness.KillNode(3); err != nil { t.Fatalf("Failed to kill node 3: %v", err) } + harness.Phase("node3-killed") - // Wait for cluster to detect node 3 is down - time.Sleep(3 * time.Second) - - // Insert many rows while node 3 is down - t.Logf("Inserting 5000 rows while node 3 is down...") - for i := 101; i <= 5100; i++ { + // Insert enough rows while node 3 is down that it is meaningfully behind. + const laggingRows = 500 + total := initialRows + laggingRows + t.Logf("Inserting %d rows while node 3 is down...", laggingRows) + for i := initialRows + 1; i <= total; i++ { _, err := harness.ExecNode(1, "INSERT INTO test_lagging (id, value) VALUES (?, ?)", i, fmt.Sprintf("value_%d", i)) if err != nil { t.Fatalf("Failed to insert row %d: %v", i, err) } - if i%1000 == 0 { - t.Logf("Inserted %d rows...", i) - } } - - t.Logf("Completed inserting 5000 rows") - - // Verify node 1 has all data - count1 := harness.getRowCount(1, "test_lagging") - t.Logf("Node 1 has %d rows", count1) - if count1 < 5100 { - t.Fatalf("Node 1 should have at least 5100 rows, got %d", count1) + if err := harness.WaitForRowCount("test_lagging", []int{1, 2}, total, 20*time.Second); err != nil { + t.Fatalf("rows did not replicate to nodes 1&2: %v", err) } + harness.Phase("lagging-load") // Restart node 3 t.Logf("Restarting node 3...") if err := harness.StartNode(3); err != nil { t.Fatalf("Failed to restart node 3: %v", err) } - - if err := harness.WaitForAlive(3, 30*time.Second); err != nil { + if err := harness.WaitForAlive(3, 20*time.Second); err != nil { t.Fatalf("Node 3 did not become alive: %v", err) } + harness.Phase("node3-alive") - // Wait for anti-entropy to sync - t.Logf("Waiting up to 120 seconds for anti-entropy to sync node 3...") - maxWait := 120 * time.Second - startTime := time.Now() - lastCount := 0 - - for time.Since(startTime) < maxWait { - time.Sleep(5 * time.Second) - - count3 := harness.getRowCount(3, "test_lagging") - count1 := harness.getRowCount(1, "test_lagging") - - t.Logf("Progress: Node 3 has %d rows, Node 1 has %d rows (elapsed: %v)", - count3, count1, time.Since(startTime).Round(time.Second)) - - if count3 > lastCount { - lastCount = count3 - } - - // Success condition: Node 3 is within 100 rows of Node 1 - if count3 >= count1-100 { - t.Logf("SUCCESS: Node 3 caught up! Node 3: %d rows, Node 1: %d rows", count3, count1) - return - } + if err := harness.WaitForRowCount("test_lagging", []int{3}, total, 20*time.Second); err != nil { + harness.dumpNodeLogs("lagging_node_fail") + t.Fatalf("node 3 did not catch up: %v", err) } + harness.Phase("caught-up") - // Timeout - check final state count3 := harness.getRowCount(3, "test_lagging") - count1Final := harness.getRowCount(1, "test_lagging") - - harness.dumpNodeLogs("lagging_node_fail") - t.Errorf("TIMEOUT: Node 3 did not catch up after 120 seconds") - t.Errorf("Final state: Node 3 has %d rows, Node 1 has %d rows", count3, count1Final) - t.Fatal("Lagging node catchup failed") + count1 := harness.getRowCount(1, "test_lagging") + t.Logf("SUCCESS: Node 3 caught up! Node 3: %d rows, Node 1: %d rows", count3, count1) } diff --git a/test/information_schema_cluster_test.go b/test/information_schema_cluster_test.go index 3a565690..23e6c81e 100644 --- a/test/information_schema_cluster_test.go +++ b/test/information_schema_cluster_test.go @@ -27,6 +27,7 @@ type isAnswer struct { func queryAnswer(conn *sql.DB, query string, args ...interface{}) (isAnswer, error) { ctx, cancel := context.WithTimeout(context.Background(), clusterQueryTimeout) defer cancel() + defer noteClientCall() rows, err := conn.QueryContext(ctx, query, args...) if err != nil { return isAnswer{}, err @@ -64,8 +65,8 @@ func queryAnswer(conn *sql.DB, query string, args ...interface{}) (isAnswer, err return answer, rows.Err() } -// TestInformationSchemaAnswersOnEveryPath is R3c-9 (reviewer A's F-P1 and -// F-P2) end to end: every INFORMATION_SCHEMA table Marmot serves answers, +// TestInformationSchemaAnswersOnEveryPath checks end to end that every +// INFORMATION_SCHEMA table Marmot serves answers, // with and without filters, the same rows and columns to a prepared // statement as to the text query, on every node; and DDL coordinated from // the node afterwards completes. Before, a prepared COLUMNS or TABLES query diff --git a/test/node_outage_convergence_cluster_test.go b/test/node_outage_convergence_cluster_test.go new file mode 100644 index 00000000..c9a1a022 --- /dev/null +++ b/test/node_outage_convergence_cluster_test.go @@ -0,0 +1,883 @@ +package test + +// Cluster tests for convergence across node outages: every ACKed write +// across two databases, an explicit transaction held open across a peer's +// outage, database create/drop while a node is down, a node killed mid +// catch-up, convergence with GC actually running, and transactions a killed +// node had prepared but never committed. +// +// These check row presence (missing=0) and duplicate ids only, never row +// values. A value mismatch across nodes - an older row image overwriting a +// newer one - is a separate, known limitation of replaying a peer's log over +// rows a node already changed; comparing values here would conflate it with +// the presence guarantees these tests exist to check. +// +// A transaction a node holds locally PENDING (its own coordinator killed +// before its local commit, or a participant COMMIT that failed against a +// pinned session) must not block that database's log pull until the +// stale-transaction GC ends it, or an ACKed row stays missing well past +// convergenceDeadline: the puller resolves such a record in the same round +// (grpc.LogPuller's resolveLocalPending). grpc/log_puller_test.go pins each +// resolution deterministically, these tests the end-to-end outcome. + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/maxpert/marmot/protocol/mysqlcode" +) + +// keyLedger records every ACKed row key (database.table/id) across possibly +// several databases, and each writer's own keys so a writer can UPDATE one of +// its own rows. Unlike rowLedger (f2_convergence_cluster_test.go, one +// database) it does not compare values across nodes - see the file comment. +type keyLedger struct { + harness *ClusterHarness + mu sync.Mutex + keys map[string]bool + own map[string][]string + dups []string +} + +func newKeyLedger(h *ClusterHarness) *keyLedger { + return &keyLedger{harness: h, keys: map[string]bool{}, own: map[string][]string{}} +} + +func ledgerKey(database, table string, id int64) string { + return fmt.Sprintf("%s.%s/%d", database, table, id) +} + +// acked records key as ACKed by writer. A duplicate key (the same id +// generated twice) is recorded, never silently overwritten. +func (l *keyLedger) acked(writer, key string) { + l.harness.Progress() + l.mu.Lock() + defer l.mu.Unlock() + if l.keys[key] { + l.dups = append(l.dups, key) + } + l.keys[key] = true + l.own[writer] = append(l.own[writer], key) +} + +// pick returns one of writer's own ACKed keys, deterministically spread by i. +func (l *keyLedger) pick(writer string, i int) (string, bool) { + l.mu.Lock() + defer l.mu.Unlock() + keys := l.own[writer] + if len(keys) == 0 { + return "", false + } + return keys[(i*7919)%len(keys)], true +} + +func (l *keyLedger) size() int { + l.mu.Lock() + defer l.mu.Unlock() + return len(l.keys) +} + +// rowIDsOn reads every row id (not value: see file comment) of every +// database.table on node. +func rowIDsOn(t *testing.T, h *ClusterHarness, node int, tables map[string][]string) (map[string]bool, error) { + got := map[string]bool{} + for database, ts := range tables { + conn := openTimedNodeDatabase(t, h, node, database) + for _, table := range ts { + a, err := queryAnswer(conn, "SELECT id FROM "+table) + if err != nil { + return nil, fmt.Errorf("node %d %s.%s: %v", node, database, table, err) + } + for _, r := range a.rows { + got[database+"."+table+"/"+r[0]] = true + } + } + } + return got, nil +} + +// waitKeysConverged polls until every node holds every ACKed row's id, within +// deadline. +func waitKeysConverged(t *testing.T, h *ClusterHarness, l *keyLedger, tables map[string][]string, deadline time.Duration) { + t.Helper() + start := time.Now() + for { + h.Progress() + var missing []string + for node := 1; node <= numNodes; node++ { + got, err := rowIDsOn(t, h, node, tables) + if err != nil { + missing = append(missing, err.Error()) + continue + } + l.mu.Lock() + for k := range l.keys { + if !got[k] { + missing = append(missing, fmt.Sprintf("node %d %s absent", node, k)) + } + } + l.mu.Unlock() + } + if len(missing) == 0 { + t.Logf("converged: %d ACKed rows on every node, %s after load stopped", l.size(), time.Since(start).Round(time.Millisecond)) + return + } + if time.Since(start) > deadline { + sort.Strings(missing) + if len(missing) > 20 { + missing = append(missing[:20], fmt.Sprintf("... %d more", len(missing)-20)) + } + h.dumpNodeLogs("cluster_" + strings.ReplaceAll(t.Name(), "/", "_")) + t.Fatalf("ACKed rows still missing %s after load stopped:\n%s", deadline, strings.Join(missing, "\n")) + } + time.Sleep(convergencePoll) + } +} + +// runKeyWriters runs one writer per node in nodes: it inserts round-robin into +// every database.table and every third statement updates one of its own +// ACKed rows (the id, and so the ledger key, does not change). A statement +// that fails is simply retried on the next iteration. +func runKeyWriters(t *testing.T, h *ClusterHarness, l *keyLedger, nodes []int, targets [][2]string, stop <-chan struct{}) *sync.WaitGroup { + var wg sync.WaitGroup + for _, n := range nodes { + wg.Add(1) + go func(n int) { + defer wg.Done() + conns := map[string]*sql.DB{} + for _, tg := range targets { + if conns[tg[0]] == nil { + conns[tg[0]] = openTimedNodeDatabase(t, h, n, tg[0]) + } + } + writer := fmt.Sprintf("n%d", n) + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + tg := targets[i%len(targets)] + if i%3 == 2 { + if key, ok := l.pick(writer, i); ok { + parts := strings.SplitN(key, "/", 2) + dt := strings.SplitN(parts[0], ".", 2) + v := fmt.Sprintf("n%d#%d-u", n, i) + if _, err := execTimed(conns[dt[0]], "UPDATE "+dt[1]+" SET v = ? WHERE id = "+parts[1], v); err != nil { + time.Sleep(convergencePoll) + } + continue + } + } + conn := conns[tg[0]] + v := fmt.Sprintf("n%d#%d", n, i) + res, err := execTimed(conn, "INSERT INTO "+tg[1]+" (v) VALUES (?)", v) + if err != nil { + time.Sleep(convergencePoll) + continue + } + if id, err := res.LastInsertId(); err == nil { + l.acked(writer, ledgerKey(tg[0], tg[1], id)) + } + } + }(n) + } + return &wg +} + +// sleepWithProgress sleeps d while reporting progress to the stall watchdog, so an +// intentionally idle phase (e.g. "node stopped, about to restart") is never +// mistaken for a stall. +func sleepWithProgress(h *ClusterHarness, d time.Duration) { + end := time.Now().Add(d) + for time.Now().Before(end) { + h.Progress() + time.Sleep(convergencePoll) + } +} + +// startWithTables starts a cluster and creates each database (unless "marmot") and +// each database.table (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT). +func startWithTables(t *testing.T, h *ClusterHarness, tables map[string][]string) { + t.Helper() + if err := h.StartCluster(); err != nil { + h.Cleanup() + t.Fatalf("StartCluster: %v", err) + } + createTables(t, h, []int{1, 2, 3}, tables) +} + +func createTables(t *testing.T, h *ClusterHarness, nodes []int, tables map[string][]string) { + t.Helper() + for database, ts := range tables { + if database != "marmot" { + if _, err := h.ExecNode(1, "CREATE DATABASE "+database); err != nil { + t.Fatalf("CREATE DATABASE %s: %v", database, err) + } + waitDatabasePresence(t, h, nodes, database, true, 30*time.Second) + } + for _, table := range ts { + if _, err := execTimed(openTimedNodeDatabase(t, h, nodes[0], database), "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)"); err != nil { + t.Fatalf("CREATE TABLE %s.%s: %v", database, table, err) + } + for _, n := range nodes { + waitTableOn(t, h, n, database, table, 30*time.Second) + } + } + } +} + +func hasDatabase(t *testing.T, h *ClusterHarness, node int, database string) (bool, error) { + a, err := queryAnswer(openTimedNodeDatabase(t, h, node, "marmot"), "SHOW DATABASES") + if err != nil { + return false, err + } + for _, r := range a.rows { + if strings.EqualFold(r[0], database) { + return true, nil + } + } + return false, nil +} + +func waitDatabasePresence(t *testing.T, h *ClusterHarness, nodes []int, database string, present bool, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for _, n := range nodes { + for { + h.Progress() + ok, err := hasDatabase(t, h, n, database) + if err == nil && ok == present { + break + } + if time.Now().After(deadline) { + h.dumpNodeLogs("cluster_db_" + database) + t.Fatalf("database %s present=%v not reached on node %d within %s (err=%v)", database, present, n, timeout, err) + } + time.Sleep(convergencePoll) + } + } +} + +func waitTableOn(t *testing.T, h *ClusterHarness, node int, database, table string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + conn := openTimedNodeDatabase(t, h, node, database) + for { + h.Progress() + if _, err := queryAnswer(conn, "SELECT COUNT(*) FROM "+table); err == nil { + return + } + if time.Now().After(deadline) { + h.dumpNodeLogs("cluster_table_" + table) + t.Fatalf("%s.%s not on node %d within %s", database, table, node, timeout) + } + time.Sleep(convergencePoll) + } +} + +// schemaVersionOf reads database's schema version through node's admin API. +func schemaVersionOf(h *ClusterHarness, node int, database string) (string, error) { + url := fmt.Sprintf("http://localhost:%d/admin/%s/metadata/schema/version", h.Nodes[node-1].GRPCPort, database) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return "", err + } + req.Header.Set("X-Marmot-Secret", "test-secret") + resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req) + noteClientCall() + if err != nil { + return "", err + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + var v interface{} + if err := json.Unmarshal(body, &v); err != nil { + return "", fmt.Errorf("status %d body %s", resp.StatusCode, body) + } + return fmt.Sprintf("%d:%s", resp.StatusCode, strings.TrimSpace(string(body))), nil +} + +// waitSchemaVersionsEqual waits until every node reports the same schema +// version for every database. +func waitSchemaVersionsEqual(t *testing.T, h *ClusterHarness, databases []string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for { + h.Progress() + var diff []string + for _, database := range databases { + var vs []string + for n := 1; n <= numNodes; n++ { + v, err := schemaVersionOf(h, n, database) + if err != nil { + v = "err:" + err.Error() + } + vs = append(vs, v) + } + if vs[0] != vs[1] || vs[0] != vs[2] { + diff = append(diff, fmt.Sprintf("%s: %v", database, vs)) + } + } + if len(diff) == 0 { + return + } + if time.Now().After(deadline) { + t.Fatalf("schema versions differ after %s: %v", timeout, diff) + } + time.Sleep(time.Second) + } +} + +// waitLogLine waits for the n-th occurrence of line in node's log. +func waitLogLine(h *ClusterHarness, node int, line string, n int, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + data, _ := os.ReadFile(h.Nodes[node-1].LogFile) + if strings.Count(string(data), line) >= n { + return true + } + time.Sleep(20 * time.Millisecond) + } + return false +} + +// TestNodeDownTwoDatabasesConverges: a node stopped for 15s under +// insert+update load across two databases, then returned: every ACKed row on +// every node within convergenceDeadline. +func TestNodeDownTwoDatabasesConverges(t *testing.T) { + h := NewClusterHarness(t) + defer h.Cleanup() + tables := map[string][]string{"outagea": {"t"}, "outageb": {"t"}} + startWithTables(t, h, tables) + h.Phase("start") + + l := newKeyLedger(h) + stop := make(chan struct{}) + wg := runKeyWriters(t, h, l, []int{1, 2, 3}, [][2]string{{"outagea", "t"}, {"outageb", "t"}}, stop) + sleepWithProgress(h, 3*time.Second) + h.Phase("load") + + if err := h.StopNode(3); err != nil { + t.Fatal(err) + } + h.Phase("node3-stopped") + sleepWithProgress(h, 15*time.Second) + + if err := h.StartNode(3); err != nil { + t.Fatal(err) + } + if err := h.WaitForAlive(3, restartDeadline); err != nil { + t.Fatal(err) + } + h.Phase("node3-alive") + sleepWithProgress(h, 3*time.Second) + close(stop) + wg.Wait() + h.Phase("load-stopped") + + if len(l.dups) > 0 { + t.Fatalf("duplicate ids: %v", l.dups) + } + waitKeysConverged(t, h, l, tables, convergenceDeadline) + h.Phase("converged") +} + +// TestExplicitTxnAcrossNodeDownConverges: an explicit transaction (BEGIN; +// several DML; COMMIT) held open on node 1, with node 3 stopped during it: +// delivered everywhere. +func TestExplicitTxnAcrossNodeDownConverges(t *testing.T) { + h := NewClusterHarness(t) + defer h.Cleanup() + tables := map[string][]string{"marmot": {"xa", "xb"}} + startWithTables(t, h, tables) + h.Phase("start") + + l := newKeyLedger(h) + stop := make(chan struct{}) + wg := runKeyWriters(t, h, l, []int{2, 3}, [][2]string{{"marmot", "xa"}, {"marmot", "xb"}}, stop) + sleepWithProgress(h, 3*time.Second) + h.Phase("load") + + db1 := openTimedNodeDatabase(t, h, 1, "marmot") + var txnKeys []string + for attempt := 1; ; attempt++ { + keys, err := runExplicitTxnAcrossOutage(t, h, l, db1, attempt == 1) + if err == nil { + txnKeys = keys + break + } + if !isDeadlockRetry(err) || attempt == explicitTxnAttempts { + t.Fatalf("explicit txn attempt %d: %v", attempt, err) + } + // 1213 is documented retryable: nothing of the transaction was + // committed, and its ids were already returned, so the product cannot + // re-claim them at COMMIT - the client retries the whole transaction. + t.Logf("explicit txn attempt %d refused with retryable 1213, retrying: %v", attempt, err) + sleepWithProgress(h, explicitTxnBackoff) + } + h.Phase("commit") + for _, k := range txnKeys { + l.acked("txn", k) + } + t.Logf("explicit txn committed %d rows, ACKed=%d", len(txnKeys), l.size()) + sleepWithProgress(h, 3*time.Second) + + if err := h.StartNode(3); err != nil { + t.Fatal(err) + } + if err := h.WaitForAlive(3, restartDeadline); err != nil { + t.Fatal(err) + } + h.Phase("node3-alive") + sleepWithProgress(h, 2*time.Second) + close(stop) + wg.Wait() + h.Phase("load-stopped") + + if len(l.dups) > 0 { + t.Fatalf("duplicate ids: %v", l.dups) + } + waitKeysConverged(t, h, l, tables, convergenceDeadline) + h.Phase("converged") +} + +// explicitTxnAttempts caps how often TestExplicitTxnAcrossNodeDownConverges +// runs its explicit transaction on the documented-retryable 1213, and +// explicitTxnBackoff is the pause before each retry. +const ( + explicitTxnAttempts = 5 + explicitTxnBackoff = time.Second +) + +// isDeadlockRetry reports whether err is MySQL 1213, the retryable refusal +// of a whole transaction. +func isDeadlockRetry(err error) bool { + var mysqlErr *mysql.MySQLError + return errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlcode.ErrCodeDeadlock +} + +// runExplicitTxnAcrossOutage runs the explicit transaction of +// TestExplicitTxnAcrossNodeDownConverges on node 1 - three +// inserts, node 3 stopped mid-transaction, three more inserts and an UPDATE +// of a row node 2 committed, held open a while before COMMIT - and returns +// the keys it inserted once COMMIT succeeds. Only the first attempt stops +// node 3 and holds the transaction open; a retry runs straight through. Any +// failure rolls it back. +func runExplicitTxnAcrossOutage(t *testing.T, h *ClusterHarness, l *keyLedger, db1 *sql.DB, firstAttempt bool) ([]string, error) { + conn, err := db1.Conn(context.Background()) + if err != nil { + return nil, err + } + defer conn.Close() + committed := false + defer func() { + if !committed { + _, _ = connExecTimed(conn, "ROLLBACK") + } + }() + var keys []string + insert := func(table, v string) error { + res, err := connExecTimed(conn, "INSERT INTO "+table+" (v) VALUES (?)", v) + if err != nil { + return err + } + id, _ := res.LastInsertId() + keys = append(keys, ledgerKey("marmot", table, id)) + return nil + } + if _, err := connExecTimed(conn, "BEGIN"); err != nil { + return nil, err + } + h.Phase("begin") + for i, table := range []string{"xa", "xb", "xa"} { + if err := insert(table, fmt.Sprintf("txn-pre-%d", i)); err != nil { + return nil, err + } + } + if firstAttempt { + sleepWithProgress(h, 2*time.Second) + if err := h.StopNode(3); err != nil { + t.Fatal(err) + } + } + for i := 0; i < 3; i++ { + // xa: node 1 already holds an id range for it, so no claim (which + // needs a quorum while node 3 is not yet suspected) is needed. + if err := insert("xa", fmt.Sprintf("txn-post-%d", i)); err != nil { + return nil, err + } + } + // Update a row node 2 committed before this transaction. + if key, ok := l.pick("n2", 1); ok && strings.HasPrefix(key, "marmot.xa/") { + if _, err := connExecTimed(conn, "UPDATE xa SET v = ? WHERE id = "+strings.TrimPrefix(key, "marmot.xa/"), "txn-upd"); err != nil { + return nil, err + } + } + if firstAttempt { + sleepWithProgress(h, 3*time.Second) + } + if _, err := connExecTimed(conn, "COMMIT"); err != nil { + return nil, err + } + committed = true + return keys, nil +} + +// connExecTimed runs one statement on an already-open *sql.Conn under +// clusterQueryTimeout (execTimed takes a *sql.DB, which an explicit +// transaction's pinned connection is not). +func connExecTimed(conn *sql.Conn, q string, args ...interface{}) (sql.Result, error) { + ctx, cancel := context.WithTimeout(context.Background(), clusterQueryTimeout) + defer cancel() + defer noteClientCall() + return conn.ExecContext(ctx, q, args...) +} + +// TestDatabaseOpsWhileNodeDownConverge: CREATE DATABASE and DROP DATABASE +// (and a drop+recreate) while a node is down: the returning node converges +// and no dropped database is resurrected. +func TestDatabaseOpsWhileNodeDownConverge(t *testing.T) { + h := NewClusterHarness(t) + defer h.Cleanup() + startWithTables(t, h, map[string][]string{"holddb": {"t"}, "recreatedb": {"t"}}) + h.Phase("start") + + l := newKeyLedger(h) + for i := 0; i < 5; i++ { + for _, database := range []string{"holddb", "recreatedb"} { + if _, err := execTimed(openTimedNodeDatabase(t, h, 1, database), "INSERT INTO t (v) VALUES (?)", fmt.Sprintf("old-%d", i)); err != nil { + t.Fatal(err) + } + } + } + for n := 1; n <= numNodes; n++ { + for _, database := range []string{"holddb", "recreatedb"} { + for { + a, err := queryAnswer(openTimedNodeDatabase(t, h, n, database), "SELECT COUNT(*) FROM t") + if err == nil && a.rows[0][0] == "5" { + break + } + sleepWithProgress(h, convergencePoll) + } + } + } + h.Phase("baseline-load") + + if err := h.StopNode(3); err != nil { + t.Fatal(err) + } + h.Phase("node3-stopped") + up := []int{1, 2} + if _, err := h.ExecNode(2, "DROP DATABASE holddb"); err != nil { + t.Fatalf("DROP holddb: %v", err) + } + if _, err := h.ExecNode(1, "DROP DATABASE recreatedb"); err != nil { + t.Fatalf("DROP recreatedb: %v", err) + } + waitDatabasePresence(t, h, up, "recreatedb", false, 30*time.Second) + createTables(t, h, up, map[string][]string{"newdb": {"t"}, "recreatedb": {"t"}}) + for i := 0; i < 20; i++ { + for _, database := range []string{"newdb", "recreatedb"} { + v := fmt.Sprintf("new-%d", i) + res, err := execTimed(openTimedNodeDatabase(t, h, up[i%2], database), "INSERT INTO t (v) VALUES (?)", v) + if err != nil { + t.Fatalf("insert %s: %v", database, err) + } + id, _ := res.LastInsertId() + l.acked("w", ledgerKey(database, "t", id)) + } + } + h.Phase("ops-done-node3-down") + + if err := h.StartNode(3); err != nil { + t.Fatal(err) + } + if err := h.WaitForAlive(3, restartDeadline); err != nil { + t.Fatal(err) + } + h.Phase("node3-alive") + waitDatabasePresence(t, h, []int{3}, "holddb", false, convergenceDeadline) + waitDatabasePresence(t, h, []int{3}, "newdb", true, convergenceDeadline) + waitKeysConverged(t, h, l, map[string][]string{"newdb": {"t"}, "recreatedb": {"t"}}, convergenceDeadline) + h.Phase("converged") + + // One more anti-entropy round: an old peer must not bring holddb back. + sleepWithProgress(h, 6*time.Second) + for n := 1; n <= numNodes; n++ { + ok, err := hasDatabase(t, h, n, "holddb") + if err != nil || ok { + t.Fatalf("node %d: holddb present=%v err=%v after the drop", n, ok, err) + } + } + h.Phase("no-resurrection-checked") +} + +// TestKillMidCatchUpConverges: kill -9 of a node mid catch-up (DDL and DML +// missed while down), then restart: converges, no duplicate apply, equal +// schema versions. The kill lands 150ms after the startup pull began, inside +// the 20ms-300ms range over which this was seen to hold. +func TestKillMidCatchUpConverges(t *testing.T) { + h := NewClusterHarness(t) + defer h.Cleanup() + tables := map[string][]string{"marmot": {"catchupa"}, "catchup": {"catchupb"}} + startWithTables(t, h, tables) + h.Phase("start") + + l := newKeyLedger(h) + stop := make(chan struct{}) + targets := [][2]string{{"marmot", "catchupa"}, {"catchup", "catchupb"}} + wg := runKeyWriters(t, h, l, []int{1, 2}, targets, stop) + sleepWithProgress(h, 3*time.Second) + h.Phase("load") + + if err := h.StopNode(3); err != nil { + t.Fatal(err) + } + h.Phase("node3-stopped") + sleepWithProgress(h, 4*time.Second) + + c1 := openTimedNodeDatabase(t, h, 1, "marmot") + for _, q := range []string{"ALTER TABLE catchupa ADD COLUMN c1 INT", "CREATE TABLE catchupd (id INT PRIMARY KEY)"} { + if _, err := execTimed(c1, q); err != nil { + t.Fatalf("%s: %v", q, err) + } + } + if _, err := execTimed(openTimedNodeDatabase(t, h, 2, "catchup"), "ALTER TABLE catchupb ADD COLUMN c2 INT"); err != nil { + t.Fatalf("alter catchupb: %v", err) + } + sleepWithProgress(h, 5*time.Second) + h.Phase("missed-ddl") + + const pullLine = "Starting startup log pull" + before, _ := os.ReadFile(h.Nodes[2].LogFile) + if err := h.StartNode(3); err != nil { + t.Fatal(err) + } + if !waitLogLine(h, 3, pullLine, strings.Count(string(before), pullLine)+1, 20*time.Second) { + t.Logf("startup pull line not seen; killing anyway") + } + time.Sleep(150 * time.Millisecond) + if err := h.KillNode(3); err != nil { + t.Fatal(err) + } + h.Phase("killed-mid-catchup") + sleepWithProgress(h, 2*time.Second) + + if err := h.StartNode(3); err != nil { + t.Fatal(err) + } + if err := h.WaitForAlive(3, restartDeadline); err != nil { + t.Fatal(err) + } + h.Phase("node3-alive") + sleepWithProgress(h, 2*time.Second) + close(stop) + wg.Wait() + h.Phase("load-stopped") + + if len(l.dups) > 0 { + t.Fatalf("duplicate ids: %v", l.dups) + } + waitKeysConverged(t, h, l, tables, convergenceDeadline) + h.Phase("converged") + waitSchemaVersionsEqual(t, h, []string{"marmot", "catchup"}, 15*time.Second) + h.Phase("schema-versions-equal") + + for n := 1; n <= numNodes; n++ { + for _, q := range []struct{ db, q string }{ + {"marmot", "SELECT COUNT(*) FROM catchupd"}, + {"marmot", "SELECT c1 FROM catchupa LIMIT 1"}, + {"catchup", "SELECT c2 FROM catchupb LIMIT 1"}, + } { + if _, err := queryAnswer(openTimedNodeDatabase(t, h, n, q.db), q.q); err != nil { + t.Errorf("node %d %s: %v", n, q.q, err) + } + } + } +} + +// TestConvergesWithLiveGC runs the cluster with GC actually deleting log +// entries (gc_interval_seconds lives in [replication]; a harness that wrote +// it elsewhere would silently run the 60s default and never see GC delete +// anything). With GC on and running every 5s, writes settle +// enough to be GC'd (seen via the log line GC logs when it deletes committed +// records, db/transaction.go cleanupOldTransactionRecords), and a node killed +// and restarted afterward still converges to every ACKed row - GC must not +// remove anything a still-behind node needs to catch up on. +func TestConvergesWithLiveGC(t *testing.T) { + const table = "livegc" + harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)", func(h *ClusterHarness) { + // gc_interval_seconds must stay >= anti_entropy_interval_seconds (the + // harness's 5s) and gc_min_retention_hours must stay >= + // delta_sync_threshold_seconds in hours (cfg.Validate enforces + // both), so both retention knobs are dropped to 0: a committed + // record becomes GC-eligible as soon as every peer's watermark has + // passed it, instead of after 2h. + h.GCIntervalSeconds = 5 + h.GCMinRetentionHours = 0 + h.DeltaSyncThresholdSeconds = 0 + }) + defer harness.Cleanup() + harness.Phase("start") + + tables := []string{table} + ledger := newRowLedger(harness) + stop := make(chan struct{}) + wg := runLedgerWriters(t, harness, ledger, []int{1, 2, 3}, tables, stop) + ledger.waitForMore(t, 150, "before the GC pass") + harness.Phase("load") + + const gcLine = "GC: Cleaned up old transaction records" + deadline := time.Now().Add(20 * time.Second) + for n := 1; n <= numNodes; n++ { + for { + data, _ := os.ReadFile(harness.Nodes[n-1].LogFile) + if strings.Contains(string(data), gcLine) { + break + } + if time.Now().After(deadline) { + close(stop) + wg.Wait() + harness.dumpNodeLogs("live_gc_no_pass") + t.Fatalf("node %d: no %q within %s of live GC running", n, gcLine, 20*time.Second) + } + harness.Progress() + time.Sleep(convergencePoll) + } + } + harness.Phase("gc-pass-seen-on-every-node") + + ledger.waitForMore(t, 50, "after the GC pass") + if err := harness.KillNode(3); err != nil { + t.Fatalf("kill node 3: %v", err) + } + harness.Phase("node3-killed") + ledger.waitForMore(t, 50, "while node 3 is down") + if err := harness.StartNode(3); err != nil { + t.Fatalf("restart node 3: %v", err) + } + if err := harness.WaitForAlive(3, restartDeadline); err != nil { + t.Fatalf("node 3 did not come back: %v", err) + } + harness.Phase("node3-alive") + ledger.waitForMore(t, 50, "after node 3 returned") + close(stop) + wg.Wait() + harness.Phase("load-stopped") + + if len(ledger.dups) > 0 { + t.Fatalf("duplicate ids: %v", ledger.dups) + } + waitLedgerConverged(t, harness, ledger, tables) + harness.Phase("converged") +} + +// missingKeysOn lists every ACKed row of tables that node lacks. +func missingKeysOn(t *testing.T, h *ClusterHarness, l *keyLedger, node int, tables map[string][]string) ([]string, error) { + got, err := rowIDsOn(t, h, node, tables) + if err != nil { + return nil, err + } + l.mu.Lock() + defer l.mu.Unlock() + var missing []string + for k := range l.keys { + if !got[k] { + missing = append(missing, k) + } + } + sort.Strings(missing) + return missing, nil +} + +// TestPreparedOnKilledNodeCommittedByPeersConverges: transactions node 3 +// durably prepared but never committed, which its +// peers did commit: an explicit transaction's pinned session holds node 3's +// SQLite writer while nodes 1 and 2 commit writes, so every COMMIT to node 3 +// waits on that writer; node 3 is then killed (kill -9) and restarted with +// those transactions recovered PENDING. The stale-transaction GC is kept out +// of the test's window (heartbeat_timeout_seconds far above it), so only the +// log pull resolving those records - committing them through the local +// commit path - can deliver the rows. Node 3 must hold every ACKed row the +// moment it reports itself ALIVE (JOINING -> ALIVE only once caught up), and +// every node must converge within convergenceDeadline. +func TestPreparedOnKilledNodeCommittedByPeersConverges(t *testing.T) { + h := NewClusterHarness(t, func(h *ClusterHarness) { h.HeartbeatTimeoutSeconds = 600 }) + defer h.Cleanup() + tables := map[string][]string{"marmot": {"pa", "pb"}} + startWithTables(t, h, tables) + h.Phase("start") + + l := newKeyLedger(h) + stop := make(chan struct{}) + wg := runKeyWriters(t, h, l, []int{1, 2}, [][2]string{{"marmot", "pa"}}, stop) + sleepWithProgress(h, 2*time.Second) + h.Phase("load") + + conn, err := openTimedNodeDatabase(t, h, 3, "marmot").Conn(context.Background()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + for _, q := range []string{"BEGIN", "INSERT INTO pb (v) VALUES ('pinned')"} { + if _, err := connExecTimed(conn, q); err != nil { + t.Fatalf("%s: %v", q, err) + } + } + h.Phase("writer-pinned") + before := l.size() + sleepWithProgress(h, 4*time.Second) + if l.size() == before { + t.Fatal("no write committed while node 3's writer was pinned") + } + if err := h.KillNode(3); err != nil { + t.Fatal(err) + } + close(stop) + wg.Wait() + h.Phase("node3-killed") + + if err := h.StartNode(3); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(convergenceDeadline) + for { + h.Progress() + status, err := h.selfStatus(3) + if err == nil && status == "ALIVE" { + // Read right after observing ALIVE; a read error only means + // node 3's MySQL listener is not up yet. + missing, readErr := missingKeysOn(t, h, l, 3, tables) + if readErr == nil && len(missing) > 0 { + t.Fatalf("node 3 promoted ALIVE while missing %d ACKed rows: %v", len(missing), missing) + } + if readErr == nil { + break + } + } + if time.Now().After(deadline) { + missing, readErr := missingKeysOn(t, h, l, 3, tables) + h.dumpNodeLogs("cluster_prepared_killed") + t.Fatalf("node 3 not ALIVE within %s (status=%q err=%v, read err=%v); missing %d ACKed rows: %v", + convergenceDeadline, status, err, readErr, len(missing), missing) + } + time.Sleep(convergencePoll) + } + h.Phase("node3-alive") + + waitKeysConverged(t, h, l, tables, convergenceDeadline) + h.Phase("converged") +} diff --git a/test/rolling_upgrade_cluster_test.go b/test/rolling_upgrade_cluster_test.go new file mode 100644 index 00000000..7e9f5557 --- /dev/null +++ b/test/rolling_upgrade_cluster_test.go @@ -0,0 +1,216 @@ +package test + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/maxpert/marmot/protocol/mysqlcode" +) + +// preLogPullCommit is the last commit before nodes pulled each other's commit +// logs: the release a rolling upgrade to the log-pull protocol starts from. +const preLogPullCommit = "7a66596" + +// upgradeGossipDeadline bounds each wait for gossip to carry a node's new log +// protocol version to its peers after that node restarts. +const upgradeGossipDeadline = 20 * time.Second + +var ( + preLogPullBinOnce sync.Once + preLogPullBinPath string + preLogPullBinErr error +) + +// preLogPullBinary builds the preLogPullCommit binary once per test process, from a +// read-only `git archive` of that commit, and reuses a binary a previous run +// left under testDataRoot. Call it before NewClusterHarness: the build is not +// part of the cluster's time budget. +func preLogPullBinary(t *testing.T) string { + t.Helper() + preLogPullBinOnce.Do(func() { + dir := filepath.Join(testDataRoot, "marmot_pref2_"+preLogPullCommit) + bin := filepath.Join(dir, "marmot") + if _, err := os.Stat(bin); err == nil { + preLogPullBinPath = bin + return + } + src := filepath.Join(dir, "src") + if err := os.MkdirAll(src, 0o755); err != nil { + preLogPullBinErr = err + return + } + archive := exec.Command("sh", "-c", fmt.Sprintf("git archive %s | tar -x -C %s", preLogPullCommit, src)) + archive.Dir = repoRoot() + if out, err := archive.CombinedOutput(); err != nil { + preLogPullBinErr = fmt.Errorf("git archive %s: %v\n%s", preLogPullCommit, err, out) + return + } + build := exec.Command("go", "build", "-tags", "sqlite_preupdate_hook sqlite_fts5 sqlite_json sqlite_math_functions sqlite_foreign_keys sqlite_stat4 sqlite_vacuum_incr", "-o", bin, ".") + build.Dir = src + if out, err := build.CombinedOutput(); err != nil { + preLogPullBinErr = fmt.Errorf("build %s: %v\n%s", preLogPullCommit, err, out) + return + } + preLogPullBinPath = bin + }) + if preLogPullBinErr != nil { + t.Fatalf("pre-log-pull binary: %v", preLogPullBinErr) + } + return preLogPullBinPath +} + +// restartOnBinary stops node, switches it to bin ("" for the harness's own +// binary) and starts it again, waiting until it answers. +func restartOnBinary(t *testing.T, h *ClusterHarness, node int, bin string) { + t.Helper() + if err := h.StopNode(node); err != nil { + t.Fatal(err) + } + if bin == "" { + delete(h.NodeBin, node) + } else { + h.NodeBin[node] = bin + } + if err := h.StartNode(node); err != nil { + t.Fatal(err) + } + if err := h.WaitForAlive(node, restartDeadline); err != nil { + t.Fatal(err) + } +} + +// ddlRefusedNaming reports whether err is the rolling-upgrade refusal (the +// retryable 1213) naming exactly the members in legacy. +func ddlRefusedNaming(err error, legacy string) bool { + var mysqlErr *mysql.MySQLError + return errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlcode.ErrCodeDeadlock && + strings.Contains(mysqlErr.Message, "node(s) "+legacy+" ") +} + +// waitDDLRefusedNaming retries ddl on node until it is refused naming exactly +// legacy (gossip carries each restarted node's protocol version to its peers +// asynchronously), failing after upgradeGossipDeadline. +func waitDDLRefusedNaming(t *testing.T, h *ClusterHarness, node int, ddl, legacy string) { + t.Helper() + deadline := time.Now().Add(upgradeGossipDeadline) + for { + h.Progress() + _, err := execTimed(openTimedNodeDatabase(t, h, node, "marmot"), ddl) + if ddlRefusedNaming(err, legacy) { + return + } + if err == nil { + t.Fatalf("node %d accepted %q while node(s) %s still run the older release", node, ddl, legacy) + } + if time.Now().After(deadline) { + t.Fatalf("node %d never refused %q naming %s within %s; last error: %v", node, ddl, legacy, upgradeGossipDeadline, err) + } + time.Sleep(convergencePoll) + } +} + +// waitDDLAccepted retries ddl on node until it succeeds, failing after +// upgradeGossipDeadline. +func waitDDLAccepted(t *testing.T, h *ClusterHarness, node int, ddl string) { + t.Helper() + deadline := time.Now().Add(upgradeGossipDeadline) + for { + h.Progress() + _, err := execTimed(openTimedNodeDatabase(t, h, node, "marmot"), ddl) + if err == nil { + return + } + if time.Now().After(deadline) { + t.Fatalf("node %d still refuses %q %s after the upgrade finished: %v", node, ddl, upgradeGossipDeadline, err) + } + time.Sleep(convergencePoll) + } +} + +// insertFromEveryNode inserts one row into marmot.t through each node and +// records it in l. +func insertFromEveryNode(t *testing.T, h *ClusterHarness, l *keyLedger, tag string) { + t.Helper() + for n := 1; n <= numNodes; n++ { + res, err := execTimed(openTimedNodeDatabase(t, h, n, "marmot"), "INSERT INTO t (v) VALUES (?)", fmt.Sprintf("%s-n%d", tag, n)) + if err != nil { + t.Fatalf("%s: INSERT through node %d: %v", tag, n, err) + } + id, err := res.LastInsertId() + if err != nil { + t.Fatal(err) + } + l.acked(fmt.Sprintf("n%d", n), ledgerKey("marmot", "t", id)) + } +} + +// TestRollingUpgradeRefusesDDLUntilEveryNodeServesLogPull: a rolling upgrade +// from the release before the log-pull protocol. A cluster started on +// preLogPullCommit gets nodes 1 and 2 upgraded; while node 3 still runs the old +// binary, DDL is refused cluster-wide with the retryable error naming node 3 +// (through an upgraded coordinator) and does not commit through the old +// coordinator either, while DML keeps replicating. Once node 3 is upgraded +// too, every node coordinates both writes and DDL, and the schema versions +// agree. +func TestRollingUpgradeRefusesDDLUntilEveryNodeServesLogPull(t *testing.T) { + oldBin := preLogPullBinary(t) + h := NewClusterHarness(t, func(h *ClusterHarness) { + h.NodeBin = map[int]string{1: oldBin, 2: oldBin, 3: oldBin} + }) + defer h.Cleanup() + tables := map[string][]string{"marmot": {"t"}} + startWithTables(t, h, tables) + h.Phase("start-old-release") + + restartOnBinary(t, h, 1, "") + restartOnBinary(t, h, 2, "") + // The old release started the cluster without the readiness endpoint + // StartCluster waits on; the upgraded nodes answer it. + for _, n := range []int{1, 2} { + if err := h.WaitForVotesReleased(n, upgradeGossipDeadline); err != nil { + t.Fatal(err) + } + } + h.Phase("nodes-1-2-upgraded") + + waitDDLRefusedNaming(t, h, 1, "CREATE TABLE mixed1 (id INT PRIMARY KEY)", "[3]") + waitDDLRefusedNaming(t, h, 2, "CREATE TABLE mixed2 (id INT PRIMARY KEY)", "[3]") + if _, err := execTimed(openTimedNodeDatabase(t, h, 3, "marmot"), "CREATE TABLE mixed3 (id INT PRIMARY KEY)"); err == nil { + t.Fatal("the old coordinator's DDL committed although its upgraded participants must refuse it") + } + l := newKeyLedger(h) + insertFromEveryNode(t, h, l, "mixed") + waitKeysConverged(t, h, l, tables, convergenceDeadline) + h.Phase("mixed-checked") + + restartOnBinary(t, h, 3, "") + h.Phase("node-3-upgraded") + + for n := 1; n <= numNodes; n++ { + table := fmt.Sprintf("after%d", n) + waitDDLAccepted(t, h, n, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + tables["marmot"] = append(tables["marmot"], table) + for m := 1; m <= numNodes; m++ { + waitTableOn(t, h, m, "marmot", table, upgradeGossipDeadline) + } + } + for n := 1; n <= numNodes; n++ { + for m := 1; m <= numNodes; m++ { + if h.tableExistsOnNode(m, fmt.Sprintf("mixed%d", n)) { + t.Fatalf("DDL refused while mixed left table mixed%d on node %d", n, m) + } + } + } + insertFromEveryNode(t, h, l, "upgraded") + waitKeysConverged(t, h, l, map[string][]string{"marmot": {"t"}}, convergenceDeadline) + waitSchemaVersionsEqual(t, h, []string{"marmot"}, upgradeGossipDeadline) + h.Phase("upgraded-checked") +} diff --git a/test/stallwatch/watchdog.go b/test/stallwatch/watchdog.go new file mode 100644 index 00000000..ddfd85cd --- /dev/null +++ b/test/stallwatch/watchdog.go @@ -0,0 +1,65 @@ +// Package stallwatch detects a test cluster that has stopped making progress, +// so a hung cluster test fails within a bounded time instead of running into +// its package timeout. +package stallwatch + +import ( + "sync" + "sync/atomic" + "time" +) + +// Watchdog calls onStall once when, for longer than timeout, neither the +// observed value changed nor Progress was called. The observed value is any +// cheap digest of the cluster's activity (for a process cluster: the bytes +// its nodes have written). +type Watchdog struct { + lastProgress atomic.Int64 // unix nanoseconds + stop chan struct{} + stopOnce sync.Once + done chan struct{} +} + +// Start begins watching: observe is polled every poll interval. +func Start(timeout, poll time.Duration, observe func() int64, onStall func(idle time.Duration)) *Watchdog { + w := &Watchdog{stop: make(chan struct{}), done: make(chan struct{})} + w.Progress() + go w.run(timeout, poll, observe, onStall) + return w +} + +// Progress records that the caller saw the cluster make progress. +func (w *Watchdog) Progress() { + w.lastProgress.Store(time.Now().UnixNano()) +} + +// Stop ends watching and waits for the watcher to exit. It is safe to call +// more than once, including after onStall has run. +func (w *Watchdog) Stop() { + w.stopOnce.Do(func() { close(w.stop) }) + <-w.done +} + +func (w *Watchdog) run(timeout, poll time.Duration, observe func() int64, onStall func(idle time.Duration)) { + defer close(w.done) + ticker := time.NewTicker(poll) + defer ticker.Stop() + last := observe() + for { + select { + case <-w.stop: + return + case <-ticker.C: + } + if v := observe(); v != last { + last = v + w.Progress() + continue + } + idle := time.Since(time.Unix(0, w.lastProgress.Load())) + if idle > timeout { + onStall(idle) + return + } + } +} diff --git a/test/stallwatch/watchdog_test.go b/test/stallwatch/watchdog_test.go new file mode 100644 index 00000000..0a309bd4 --- /dev/null +++ b/test/stallwatch/watchdog_test.go @@ -0,0 +1,66 @@ +package stallwatch + +import ( + "sync/atomic" + "testing" + "time" +) + +const ( + testTimeout = 300 * time.Millisecond + testPoll = 20 * time.Millisecond +) + +// TestWatchdogFiresWhenNothingChanges: a cluster whose observed activity and +// ledger both stand still is reported stalled, once, soon after the timeout. +// +// Mutation: never call onStall. "a silent cluster was not reported" fires. +func TestWatchdogFiresWhenNothingChanges(t *testing.T) { + fired := make(chan time.Duration, 2) + w := Start(testTimeout, testPoll, func() int64 { return 7 }, func(idle time.Duration) { fired <- idle }) + defer w.Stop() + select { + case idle := <-fired: + if idle < testTimeout { + t.Fatalf("reported stalled after %s, before the %s timeout", idle, testTimeout) + } + case <-time.After(10 * testTimeout): + t.Fatal("a silent cluster was not reported") + } + select { + case <-fired: + t.Fatal("a stall was reported twice") + case <-time.After(3 * testTimeout): + } +} + +// TestWatchdogStaysQuietWhileTheClusterMakesProgress: activity the watchdog +// observes, or progress the test reports, keeps it from firing. +// +// Mutation: ignore observed changes. "observed activity was reported as a +// stall" fires. +func TestWatchdogStaysQuietWhileTheClusterMakesProgress(t *testing.T) { + var counter atomic.Int64 + fired := make(chan time.Duration, 1) + w := Start(testTimeout, testPoll, func() int64 { return counter.Add(1) }, func(idle time.Duration) { fired <- idle }) + time.Sleep(4 * testTimeout) + w.Stop() + select { + case <-fired: + t.Fatal("observed activity was reported as a stall") + default: + } + + w = Start(testTimeout, testPoll, func() int64 { return 1 }, func(idle time.Duration) { fired <- idle }) + deadline := time.Now().Add(4 * testTimeout) + for time.Now().Before(deadline) { + w.Progress() + time.Sleep(testPoll) + } + w.Stop() + select { + case <-fired: + t.Fatal("reported progress was reported as a stall") + default: + } +} diff --git a/test/verification_test.go b/test/verification_test.go index 4eb3376d..96d80512 100644 --- a/test/verification_test.go +++ b/test/verification_test.go @@ -119,6 +119,13 @@ func (tdm *TestDatabaseManager) GetVectorIndexManager() coordinator.VectorIndexM return nil } +// RegistryKeyGeneration reports generation 1, live, for every name: this +// fixture serves a single always-present database regardless of name (see +// GetDatabase/DatabaseExists above). +func (tdm *TestDatabaseManager) RegistryKeyGeneration(name string) (uint64, bool, error) { + return 1, true, nil +} + func TestMySQLServerIntegration(t *testing.T) { // Setup temporary DB with MetaStore tmpDir := t.TempDir() diff --git a/test/write_ledger_convergence_cluster_test.go b/test/write_ledger_convergence_cluster_test.go new file mode 100644 index 00000000..a05ff8ab --- /dev/null +++ b/test/write_ledger_convergence_cluster_test.go @@ -0,0 +1,244 @@ +package test + +import ( + "fmt" + "sort" + "strings" + "sync" + "testing" + "time" +) + +// convergenceDeadline bounds how long after the write load stops every node +// must hold every ACKed row: the harness's 5 s anti-entropy interval, a +// second round for a pair held back by a transaction still settling, and +// margin for the per-pair pulls themselves. +const convergenceDeadline = 30 * time.Second + +const ( + // loadWindow is how long the ClaimsUnderDDL load runs. + loadWindow = 20 * time.Second + // rowLedgerStep is how many further ACKed rows each phase of the + // kill-and-restart test waits for, within phaseDeadline. + rowLedgerStep = 200 + phaseDeadline = 20 * time.Second + // restartDeadline bounds the restarted node's return. + restartDeadline = 30 * time.Second + // convergencePoll is the polling interval of every wait here. + convergencePoll = 250 * time.Millisecond +) + +// rowLedger records every ACKed row as table/id -> value, and reports each +// one to the harness's stall watchdog as progress. +type rowLedger struct { + harness *ClusterHarness + mu sync.Mutex + rows map[string]string + dups []string +} + +func newRowLedger(harness *ClusterHarness) *rowLedger { + return &rowLedger{harness: harness, rows: map[string]string{}} +} + +// size returns the number of ACKed rows recorded. +func (l *rowLedger) size() int { + l.mu.Lock() + defer l.mu.Unlock() + return len(l.rows) +} + +// waitForMore waits until n more rows than now are ACKed, or fails. +func (l *rowLedger) waitForMore(t *testing.T, n int, phase string) { + t.Helper() + want := l.size() + n + deadline := time.Now().Add(phaseDeadline) + for l.size() < want { + if time.Now().After(deadline) { + t.Fatalf("%s: only %d of %d ACKed rows within %s", phase, l.size(), want, phaseDeadline) + } + time.Sleep(convergencePoll) + } +} + +func (l *rowLedger) record(table string, id int64, v string) { + l.harness.Progress() + l.mu.Lock() + defer l.mu.Unlock() + k := fmt.Sprintf("%s/%d", table, id) + if prev, ok := l.rows[k]; ok { + l.dups = append(l.dups, fmt.Sprintf("%s: %s and %s", k, prev, v)) + } + l.rows[k] = v +} + +// missingOn returns the ledger rows node does not hold with their ACKed value. +func (l *rowLedger) missingOn(t *testing.T, harness *ClusterHarness, node int, tables []string) []string { + t.Helper() + conn := openTimedNodeDatabase(t, harness, node, "marmot") + got := map[string]string{} + for _, table := range tables { + a, err := queryAnswer(conn, "SELECT id, v FROM "+table) + if err != nil { + return []string{fmt.Sprintf("node %d %s unreadable: %v", node, table, err)} + } + for _, r := range a.rows { + got[table+"/"+r[0]] = r[1] + } + } + l.mu.Lock() + defer l.mu.Unlock() + var missing []string + for k, v := range l.rows { + if got[k] != v { + missing = append(missing, fmt.Sprintf("node %d %s want %q got %q", node, k, v, got[k])) + } + } + sort.Strings(missing) + return missing +} + +// waitLedgerConverged polls until every node holds every ACKed row or +// convergenceDeadline passes, and fails with the rows still missing. +func waitLedgerConverged(t *testing.T, harness *ClusterHarness, ledger *rowLedger, tables []string) { + t.Helper() + start := time.Now() + for { + var missing []string + for node := 1; node <= numNodes; node++ { + missing = append(missing, ledger.missingOn(t, harness, node, tables)...) + } + if len(missing) == 0 { + t.Logf("converged: %d ACKed rows on every node %s after load stopped", len(ledger.rows), time.Since(start).Round(time.Millisecond)) + return + } + if time.Since(start) > convergenceDeadline { + if len(missing) > 20 { + missing = append(missing[:20], fmt.Sprintf("... %d more", len(missing)-20)) + } + t.Fatalf("ACKed rows still missing %s after load stopped:\n%s", convergenceDeadline, strings.Join(missing, "\n")) + } + time.Sleep(convergencePoll) + } +} + +// runLedgerWriters inserts into tables round-robin from every node in nodes until +// stop closes, recording each ACKed row. A node may be down part of the time; +// its failed statements are simply not ACKed. +func runLedgerWriters(t *testing.T, harness *ClusterHarness, ledger *rowLedger, nodes []int, tables []string, stop <-chan struct{}) *sync.WaitGroup { + var wg sync.WaitGroup + for _, n := range nodes { + wg.Add(1) + go func(n int) { + defer wg.Done() + conn := openTimedNodeDatabase(t, harness, n, "marmot") + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + table := tables[i%len(tables)] + v := fmt.Sprintf("n%d#%d", n, i) + res, err := execTimed(conn, "INSERT INTO "+table+" (v) VALUES (?)", v) + if err != nil { + time.Sleep(convergencePoll) + continue + } + if id, err := res.LastInsertId(); err == nil { + ledger.record(table, id, v) + } + } + }(n) + } + return &wg +} + +// TestClaimsUnderDDLConvergesOnEveryNode: narrow inserts on all three nodes +// while DDL commits from every node. A node that declines a PREPARE (its +// schema version is briefly behind) misses those transactions; anti-entropy +// must deliver every ACKed row to every node within convergenceDeadline of +// the load stopping. On 7a66596 30-50 rows per table stayed missing on every +// node forever. +func TestClaimsUnderDDLConvergesOnEveryNode(t *testing.T) { + harness := startNarrowCluster(t, "s1", "CREATE TABLE s1 (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + defer harness.Cleanup() + c1 := openTimedNodeDatabase(t, harness, 1, "marmot") + if _, err := execTimed(c1, "CREATE TABLE s2 (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)"); err != nil { + t.Fatal(err) + } + waitForTableIn(t, harness, "marmot", "s2", 30*time.Second) + harness.Phase("start") + + tables := []string{"s1", "s2"} + ledger := newRowLedger(harness) + stop := make(chan struct{}) + wg := runLedgerWriters(t, harness, ledger, []int{1, 2, 3}, tables, stop) + + deadline := time.Now().Add(loadWindow) + for i := 0; time.Now().Before(deadline); i++ { + creator, alterer := i%3+1, (i+1)%3+1 + cc := openTimedNodeDatabase(t, harness, creator, "marmot") + if _, err := execTimed(cc, fmt.Sprintf("CREATE TABLE d%d (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)", i)); err != nil { + t.Logf("create d%d on node %d: %v", i, creator, err) + continue + } + if i > 0 { + ca := openTimedNodeDatabase(t, harness, alterer, "marmot") + if _, err := execTimed(ca, fmt.Sprintf("ALTER TABLE d%d ADD COLUMN c%d INT", i-1, i)); err != nil { + t.Logf("alter d%d on node %d: %v", i-1, alterer, err) + } + } + } + close(stop) + wg.Wait() + harness.Phase("load-stopped") + + if len(ledger.dups) > 0 { + t.Fatalf("duplicate ids: %v", ledger.dups) + } + if len(ledger.rows) == 0 { + t.Fatal("no row was ACKed") + } + waitLedgerConverged(t, harness, ledger, tables) + harness.Phase("converged") +} + +// TestKilledNodeConvergesAfterRestart: a node SIGKILLed under write load +// and restarted converges to every row the cluster ACKed, whether it +// coordinated the row, missed it while down, or held it only in its own log. +func TestKilledNodeConvergesAfterRestart(t *testing.T) { + harness := startNarrowCluster(t, "kr", "CREATE TABLE kr (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + defer harness.Cleanup() + + tables := []string{"kr"} + ledger := newRowLedger(harness) + stop := make(chan struct{}) + wg := runLedgerWriters(t, harness, ledger, []int{1, 2, 3}, tables, stop) + harness.Phase("start") + + ledger.waitForMore(t, rowLedgerStep, "before the kill") + harness.Phase("load") + if err := harness.KillNode(3); err != nil { + t.Fatalf("kill node 3: %v", err) + } + harness.Phase("node3-killed") + ledger.waitForMore(t, rowLedgerStep, "while node 3 is down") + if err := harness.StartNode(3); err != nil { + t.Fatalf("restart node 3: %v", err) + } + if err := harness.WaitForAlive(3, restartDeadline); err != nil { + t.Fatalf("node 3 did not come back: %v", err) + } + harness.Phase("node3-alive") + ledger.waitForMore(t, rowLedgerStep, "after node 3 returned") + close(stop) + wg.Wait() + harness.Phase("load-stopped") + + if len(ledger.dups) > 0 { + t.Fatalf("duplicate ids: %v", ledger.dups) + } + waitLedgerConverged(t, harness, ledger, tables) + harness.Phase("converged") +} From 8914a41c988b3aa0869a6c1eff864b6085bf3415 Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Tue, 29 Sep 2026 10:01:12 -0500 Subject: [PATCH 07/17] feat(replication): per-row last-writer-wins on the coordinator's commit timestamp A node that caught up from several peers applied each transaction once but not in commit order, so an older row image could overwrite a newer one, an older INSERT could resurrect a deleted row, and nodes diverged for good. No commit timestamp was shared: every node computed its own. The coordinator now merges every participant's PREPARE-time clock into its own, decides one commit HLC, and sends it on COMMIT (unary and streamed) and in the committed log listing. Every node commits with it and merges it into its clock, including on replay, and the clock is seeded at open from the highest commit recorded, so a later conflicting commit always gets a larger version. Each user database keeps __marmot_row_version (table, intent key, wall, logical, origin node, deleted). Every commit path stamps each row in the same SQLite transaction as the change and applies the change only if its version is not older than the stored one. A DELETE leaves a tombstone, an UPDATE of an absent row inserts its after-image, and an UPDATE that moves the primary key tombstones the old key independently of the new one. LOAD DATA rows are stamped per row the same way, in batches under one savepoint. DROP TABLE removes a table's versions, RENAME re-keys them, and a primary key change clears them. Tombstones older than every known member's log retention are purged in bounded chunks through a partial index. The stamp costs one extra indexed upsert per row, with no lock and no round trip. Its statement is prepared once per transaction. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- coordinator/write_coordinator.go | 24 + ...ite_coordinator_commit_ddl_timeout_test.go | 5 +- .../write_coordinator_commit_ts_test.go | 25 + .../write_coordinator_commithelpers_test.go | 40 +- db/cdc_applier.go | 38 +- db/database_manager.go | 44 +- db/db_integration.go | 8 + db/fk_cascade_cdc_test.go | 5 +- db/gc_log_test.go | 8 +- db/load_data_bench_test.go | 46 ++ db/local_replicator.go | 1 + db/log_position.go | 36 + db/meta_store.go | 7 +- db/meta_store_memory.go | 7 +- db/meta_store_pebble.go | 41 +- db/replay_apply.go | 57 +- db/replication_engine.go | 16 +- db/row_version.go | 458 ++++++++++++ db/row_version_load.go | 234 +++++++ db/row_version_test.go | 650 ++++++++++++++++++ db/rowid_sentinel_cdc_test.go | 2 +- db/schema_change.go | 17 +- db/sqlite_batch_committer.go | 17 +- db/stale_gc_commit_race_test.go | 2 +- db/test_helpers.go | 4 + db/transaction.go | 71 +- docs/src/pages/replication.mdx | 15 + grpc/client.go | 10 +- grpc/client_test.go | 5 + grpc/log_puller.go | 11 +- grpc/log_puller_row_version_test.go | 148 ++++ grpc/log_puller_test.go | 1 + grpc/log_server.go | 34 +- grpc/marmot.pb.go | 225 +++--- grpc/marmot.proto | 5 + grpc/node_registry.go | 15 + grpc/node_registry_test.go | 33 + grpc/replication_handler.go | 1 + grpc/replicator_adapter.go | 15 +- grpc/replicator_adapter_test.go | 44 ++ grpc/server.go | 2 +- grpc/transaction_stream_handler_test.go | 8 + hlc/clock.go | 5 + marmot.go | 5 + protocol/load_data_local.go | 27 + replica/stream_client.go | 4 +- test/node_outage_convergence_cluster_test.go | 8 +- test/row_version_convergence_cluster_test.go | 146 ++++ 48 files changed, 2408 insertions(+), 222 deletions(-) create mode 100644 coordinator/write_coordinator_commit_ts_test.go create mode 100644 db/load_data_bench_test.go create mode 100644 db/row_version.go create mode 100644 db/row_version_load.go create mode 100644 db/row_version_test.go create mode 100644 grpc/log_puller_row_version_test.go create mode 100644 test/row_version_convergence_cluster_test.go diff --git a/coordinator/write_coordinator.go b/coordinator/write_coordinator.go index d3b0522a..0fe88046 100644 --- a/coordinator/write_coordinator.go +++ b/coordinator/write_coordinator.go @@ -97,6 +97,9 @@ type ReplicationRequest struct { Database string // Minimum schema version required to execute this transaction RequiredSchemaVersion uint64 + // CommitTS is, on COMMIT, the commit timestamp every participant commits + // with (decideCommitTS). + CommitTS hlc.Timestamp } // ReplicationPhase indicates which phase of 2PC @@ -112,6 +115,10 @@ const ( type ReplicationResponse struct { Success bool Error string + // AppliedAt is the participant's clock when it answered; zero when it + // sent none. The coordinator merges every PREPARE's into its clock + // before deciding the commit timestamp. + AppliedAt hlc.Timestamp // ConflictDetected indicates write-write conflict ConflictDetected bool ConflictDetails string @@ -344,6 +351,7 @@ func (wc *WriteCoordinator) buildCommitRequest(txn *Transaction) *ReplicationReq TxnID: txn.ID, Phase: PhaseCommit, StartTS: txn.StartTS, + CommitTS: txn.CommitTS, NodeID: wc.nodeID, Database: txn.Database, Statements: commitStmts, @@ -632,6 +640,21 @@ func (wc *WriteCoordinator) commitLocalAfterRemoteQuorum(ctx context.Context, re return localResp, nil } +// decideCommitTS returns the transaction's commit timestamp, identical on +// every node that commits it. Every participant's PREPARE-time clock is +// merged in first: a participant grants a row's intent only after it has +// committed that row's previous writer, merging that writer's commit +// timestamp into its clock, so the result is later than the commit +// timestamp of every earlier conflicting transaction. +func (wc *WriteCoordinator) decideCommitTS(prepResponses map[uint64]*ReplicationResponse) hlc.Timestamp { + for _, resp := range prepResponses { + if resp != nil && !resp.AppliedAt.IsZero() { + wc.clock.Update(resp.AppliedAt) + } + } + return wc.clock.Now() +} + // runCommitPhase orchestrates the commit phase of 2PC. // Quorum of write intents created successfully with no conflicts. // @@ -644,6 +667,7 @@ func (wc *WriteCoordinator) runCommitPhase(ctx context.Context, txn *Transaction Int("prepared_nodes", len(prepResponses)). Msg("PREPARE phase complete, starting COMMIT") + txn.CommitTS = wc.decideCommitTS(prepResponses) commitReq := wc.buildCommitRequest(txn) // Count other prepared nodes (excluding self) diff --git a/coordinator/write_coordinator_commit_ddl_timeout_test.go b/coordinator/write_coordinator_commit_ddl_timeout_test.go index 2b1ce18b..ce43eb16 100644 --- a/coordinator/write_coordinator_commit_ddl_timeout_test.go +++ b/coordinator/write_coordinator_commit_ddl_timeout_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/maxpert/marmot/cfg" + "github.com/maxpert/marmot/hlc" ) // ============================================================================= @@ -101,7 +102,7 @@ func TestRunCommitPhase_DDLTransactionSurvivesCommitDelayBeyondWriteTimeout(t *t remoteReplicator.SetDelay(txn.ID, 500*time.Millisecond) // wc.timeout stands in for the regular write timeout (write_timeout_ms). - wc := NewWriteCoordinator(1, nodeProvider, remoteReplicator, localReplicator, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nodeProvider, remoteReplicator, localReplicator, 100*time.Millisecond, hlc.NewClock(1)) prepResponses := map[uint64]*ReplicationResponse{ 1: CreateSuccessResponse(), @@ -147,7 +148,7 @@ func TestRunCommitPhase_DMLTransactionCommitStillTimesOutAtWriteTimeout(t *testi remoteReplicator.SetDelay(txn.ID, 500*time.Millisecond) - wc := NewWriteCoordinator(1, nodeProvider, remoteReplicator, localReplicator, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nodeProvider, remoteReplicator, localReplicator, 100*time.Millisecond, hlc.NewClock(1)) prepResponses := map[uint64]*ReplicationResponse{ 1: CreateSuccessResponse(), diff --git a/coordinator/write_coordinator_commit_ts_test.go b/coordinator/write_coordinator_commit_ts_test.go new file mode 100644 index 00000000..c0f939fb --- /dev/null +++ b/coordinator/write_coordinator_commit_ts_test.go @@ -0,0 +1,25 @@ +package coordinator + +import ( + "testing" + "time" + + "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" +) + +// The commit timestamp must follow every participant's PREPARE-time clock: +// a participant that granted a row's intent had already committed that +// row's previous writer, so its clock is past that writer's commit. +func TestDecideCommitTSFollowsEveryPreparedParticipantsClock(t *testing.T) { + wc := NewWriteCoordinator(1, nil, nil, nil, time.Second, hlc.NewClock(1)) + ahead := hlc.Timestamp{WallTime: time.Now().Add(time.Hour).UnixNano(), Logical: 7, NodeID: 3} + + got := wc.decideCommitTS(map[uint64]*ReplicationResponse{ + 2: {Success: true}, + 3: {Success: true, AppliedAt: ahead}, + }) + + require.True(t, hlc.After(got, ahead), "commit %v must follow participant clock %v", got, ahead) + require.Equal(t, uint64(1), got.NodeID) +} diff --git a/coordinator/write_coordinator_commithelpers_test.go b/coordinator/write_coordinator_commithelpers_test.go index b466d49a..c1d6bb89 100644 --- a/coordinator/write_coordinator_commithelpers_test.go +++ b/coordinator/write_coordinator_commithelpers_test.go @@ -4,6 +4,8 @@ import ( "context" "testing" "time" + + "github.com/maxpert/marmot/hlc" ) // ============================================================================= @@ -14,7 +16,7 @@ import ( func TestSendRemoteCommits_SpawnCorrectGoroutineCount(t *testing.T) { InitTestTelemetry() mock := newMockReplicator() - wc := NewWriteCoordinator(1, nil, mock, mock, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, mock, mock, 100*time.Millisecond, hlc.NewClock(1)) // Setup: 4 prepared nodes (3 remote + coordinator) preparedNodes := map[uint64]*ReplicationResponse{ @@ -56,7 +58,7 @@ func TestSendRemoteCommits_SpawnCorrectGoroutineCount(t *testing.T) { func TestSendRemoteCommits_ExcludeCoordinator(t *testing.T) { InitTestTelemetry() mock := newMockReplicator() - wc := NewWriteCoordinator(1, nil, mock, mock, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, mock, mock, 100*time.Millisecond, hlc.NewClock(1)) preparedNodes := map[uint64]*ReplicationResponse{ 1: CreateSuccessResponse(), // Coordinator (should be excluded) @@ -93,7 +95,7 @@ func TestSendRemoteCommits_ExcludeCoordinator(t *testing.T) { func TestSendRemoteCommits_ChannelCapacityMatch(t *testing.T) { InitTestTelemetry() mock := newMockReplicator() - wc := NewWriteCoordinator(1, nil, mock, mock, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, mock, mock, 100*time.Millisecond, hlc.NewClock(1)) testCases := []struct { name string @@ -156,7 +158,7 @@ func TestSendRemoteCommits_DetachedContext(t *testing.T) { mock.commitLatency = 100 * time.Millisecond mock.mu.Unlock() - wc := NewWriteCoordinator(1, nil, mock, mock, 200*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, mock, mock, 200*time.Millisecond, hlc.NewClock(1)) preparedNodes := map[uint64]*ReplicationResponse{ 1: CreateSuccessResponse(), // Coordinator @@ -214,7 +216,7 @@ collectLoop: func TestSendRemoteCommits_EmptyPreparedNodes(t *testing.T) { InitTestTelemetry() mock := newMockReplicator() - wc := NewWriteCoordinator(1, nil, mock, mock, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, mock, mock, 100*time.Millisecond, hlc.NewClock(1)) preparedNodes := map[uint64]*ReplicationResponse{} @@ -247,7 +249,7 @@ func TestSendRemoteCommits_EmptyPreparedNodes(t *testing.T) { func TestSendRemoteCommits_SingleRemoteNode(t *testing.T) { InitTestTelemetry() mock := newMockReplicator() - wc := NewWriteCoordinator(1, nil, mock, mock, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, mock, mock, 100*time.Millisecond, hlc.NewClock(1)) preparedNodes := map[uint64]*ReplicationResponse{ 1: CreateSuccessResponse(), // Coordinator @@ -295,7 +297,7 @@ func TestSendRemoteCommits_TimeoutPropagation(t *testing.T) { // Set coordinator timeout to 3 seconds coordinatorTimeout := 3 * time.Second - wc := NewWriteCoordinator(1, nil, mock, mock, coordinatorTimeout, nil) + wc := NewWriteCoordinator(1, nil, mock, mock, coordinatorTimeout, hlc.NewClock(1)) preparedNodes := map[uint64]*ReplicationResponse{ 1: CreateSuccessResponse(), // Coordinator @@ -327,7 +329,7 @@ func TestSendRemoteCommits_TimeoutPropagation(t *testing.T) { func TestSendRemoteCommits_CorrectTxnID(t *testing.T) { InitTestTelemetry() mock := newMockReplicator() - wc := NewWriteCoordinator(1, nil, mock, mock, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, mock, mock, 100*time.Millisecond, hlc.NewClock(1)) expectedTxnID := uint64(12345) @@ -370,7 +372,7 @@ func TestSendRemoteCommits_CorrectTxnID(t *testing.T) { // Test 7.1: Exact quorum achieved func TestWaitForRemoteQuorum_ExactQuorumAchieved(t *testing.T) { InitTestTelemetry() - wc := NewWriteCoordinator(1, nil, nil, nil, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, nil, nil, 100*time.Millisecond, hlc.NewClock(1)) mocker := NewChannelMocker() commitChan := make(chan response, 3) @@ -401,7 +403,7 @@ func TestWaitForRemoteQuorum_ExactQuorumAchieved(t *testing.T) { // Test 7.2: Over quorum achieved - all nodes respond successfully func TestWaitForRemoteQuorum_OverQuorumAchieved(t *testing.T) { InitTestTelemetry() - wc := NewWriteCoordinator(1, nil, nil, nil, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, nil, nil, 100*time.Millisecond, hlc.NewClock(1)) mocker := NewChannelMocker() commitChan := make(chan response, 4) @@ -431,7 +433,7 @@ func TestWaitForRemoteQuorum_UnderQuorumTimeout(t *testing.T) { InitTestTelemetry() // Set very short timeout to speed up test - wc := NewWriteCoordinator(1, nil, nil, nil, 50*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, nil, nil, 50*time.Millisecond, hlc.NewClock(1)) mocker := NewChannelMocker() commitChan := make(chan response, 3) @@ -468,7 +470,7 @@ func TestWaitForRemoteQuorum_UnderQuorumTimeout(t *testing.T) { // Test 7.4: Early exit on quorum - don't wait for all responses func TestWaitForRemoteQuorum_EarlyExitOnQuorum(t *testing.T) { InitTestTelemetry() - wc := NewWriteCoordinator(1, nil, nil, nil, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, nil, nil, 100*time.Millisecond, hlc.NewClock(1)) mocker := NewChannelMocker() commitChan := make(chan response, 4) @@ -505,7 +507,7 @@ func TestWaitForRemoteQuorum_EarlyExitOnQuorum(t *testing.T) { // Test 7.5: All nodes fail - no ACKs func TestWaitForRemoteQuorum_AllNodesFail(t *testing.T) { InitTestTelemetry() - wc := NewWriteCoordinator(1, nil, nil, nil, 50*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, nil, nil, 50*time.Millisecond, hlc.NewClock(1)) mocker := NewChannelMocker() commitChan := make(chan response, 3) @@ -532,7 +534,7 @@ func TestWaitForRemoteQuorum_AllNodesFail(t *testing.T) { // Test 7.6: Mixed success and error - count only successes func TestWaitForRemoteQuorum_MixedSuccessError(t *testing.T) { InitTestTelemetry() - wc := NewWriteCoordinator(1, nil, nil, nil, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, nil, nil, 100*time.Millisecond, hlc.NewClock(1)) mocker := NewChannelMocker() commitChan := make(chan response, 4) @@ -567,7 +569,7 @@ func TestWaitForRemoteQuorum_TimeoutPerResponse(t *testing.T) { InitTestTelemetry() // Coordinator timeout is 100ms per response - wc := NewWriteCoordinator(1, nil, nil, nil, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, nil, nil, 100*time.Millisecond, hlc.NewClock(1)) mocker := NewChannelMocker() commitChan := make(chan response, 3) @@ -600,7 +602,7 @@ func TestWaitForRemoteQuorum_TimeoutPerResponse(t *testing.T) { // Test 7.8: Zero quorum needed - early exit on first iteration func TestWaitForRemoteQuorum_ZeroQuorumNeeded(t *testing.T) { InitTestTelemetry() - wc := NewWriteCoordinator(1, nil, nil, nil, 50*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, nil, nil, 50*time.Millisecond, hlc.NewClock(1)) commitChan := make(chan response, 2) @@ -626,7 +628,7 @@ func TestWaitForRemoteQuorum_ZeroQuorumNeeded(t *testing.T) { // Test 7.9: Negative quorum needed - early exit on first iteration func TestWaitForRemoteQuorum_NegativeQuorumNeeded(t *testing.T) { InitTestTelemetry() - wc := NewWriteCoordinator(1, nil, nil, nil, 50*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, nil, nil, 50*time.Millisecond, hlc.NewClock(1)) commitChan := make(chan response, 2) @@ -651,7 +653,7 @@ func TestWaitForRemoteQuorum_NegativeQuorumNeeded(t *testing.T) { // Test 7.10: Channel closed early - handle gracefully func TestWaitForRemoteQuorum_ChannelClosed(t *testing.T) { InitTestTelemetry() - wc := NewWriteCoordinator(1, nil, nil, nil, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, nil, nil, 100*time.Millisecond, hlc.NewClock(1)) commitChan := make(chan response, 2) @@ -683,7 +685,7 @@ func TestWaitForRemoteQuorum_ChannelClosed(t *testing.T) { // Test 7.11: Nil responses - not counted as ACK func TestWaitForRemoteQuorum_NilResponses(t *testing.T) { InitTestTelemetry() - wc := NewWriteCoordinator(1, nil, nil, nil, 100*time.Millisecond, nil) + wc := NewWriteCoordinator(1, nil, nil, nil, 100*time.Millisecond, hlc.NewClock(1)) commitChan := make(chan response, 3) diff --git a/db/cdc_applier.go b/db/cdc_applier.go index 4deef619..4c1b6b2a 100644 --- a/db/cdc_applier.go +++ b/db/cdc_applier.go @@ -33,13 +33,6 @@ func quoteSQLiteIdentList(columns []string) []string { return quoted } -func ApplyCDCEntry(exec CDCExecutor, schema CDCSchemaProvider, entry *IntentEntry) error { - if entry == nil { - return nil - } - return ApplyCDCValues(exec, schema, OpType(entry.Operation), entry.Table, entry.OldValues, entry.NewValues) -} - func ApplyCDCValues(exec CDCExecutor, schema CDCSchemaProvider, opType OpType, tableName string, oldValues, newValues map[string][]byte) error { switch opType { case OpTypeInsert, OpTypeReplace: @@ -101,14 +94,27 @@ func ApplyCDCInsert(exec CDCExecutor, tableName string, newValues map[string][]b func ApplyCDCUpdate(exec CDCExecutor, schema CDCSchemaProvider, tableName string, oldValues, newValues map[string][]byte) error { + result, err := execCDCUpdate(exec, schema, "UPDATE", tableName, oldValues, newValues) + if err != nil { + return err + } + logZeroRowsAffected(result, "ApplyCDCUpdate", tableName) + return nil +} + +// execCDCUpdate runs ApplyCDCUpdate's UPDATE, as verb ("UPDATE" or +// "UPDATE OR REPLACE"), and returns its result. +func execCDCUpdate(exec CDCExecutor, schema CDCSchemaProvider, verb, tableName string, + oldValues, newValues map[string][]byte) (sql.Result, error) { + if len(newValues) == 0 { - return fmt.Errorf("ApplyCDCUpdate %s: no values to update", tableName) + return nil, fmt.Errorf("ApplyCDCUpdate %s: no values to update", tableName) } // Get primary key columns from schema primaryKeys, err := schema.GetPrimaryKeys(tableName) if err != nil { - return fmt.Errorf("ApplyCDCUpdate %s: failed to get primary keys: %w", tableName, err) + return nil, fmt.Errorf("ApplyCDCUpdate %s: failed to get primary keys: %w", tableName, err) } // Build SET clause from newValues (sorted for determinism) @@ -125,7 +131,7 @@ func ApplyCDCUpdate(exec CDCExecutor, schema CDCSchemaProvider, tableName string setClauses[i] = fmt.Sprintf("%s = ?", quoteSQLiteIdent(col)) value, err := unmarshalCDCValue(newValues[col]) if err != nil { - return fmt.Errorf("ApplyCDCUpdate %s: failed to deserialize column %s: %w", tableName, col, err) + return nil, fmt.Errorf("ApplyCDCUpdate %s: failed to deserialize column %s: %w", tableName, col, err) } values = append(values, value) } @@ -140,29 +146,29 @@ func ApplyCDCUpdate(exec CDCExecutor, schema CDCSchemaProvider, tableName string // Fallback to newValues if oldValues doesn't have PK pkBytes, ok = newValues[pkCol] if !ok { - return fmt.Errorf("ApplyCDCUpdate %s: primary key column %s not found in CDC data", tableName, pkCol) + return nil, fmt.Errorf("ApplyCDCUpdate %s: primary key column %s not found in CDC data", tableName, pkCol) } } clause, nextValues, err := cdcPrimaryKeyPredicate("ApplyCDCUpdate", tableName, pkCol, pkBytes, values) if err != nil { - return err + return nil, err } whereClauses[i] = clause values = nextValues } - sqlStmt := fmt.Sprintf("UPDATE %s SET %s WHERE %s", + sqlStmt := fmt.Sprintf("%s %s SET %s WHERE %s", + verb, quoteSQLiteIdent(tableName), strings.Join(setClauses, ", "), strings.Join(whereClauses, " AND ")) result, err := exec.Exec(sqlStmt, values...) if err != nil { - return fmt.Errorf("ApplyCDCUpdate %s: %w", tableName, err) + return nil, fmt.Errorf("ApplyCDCUpdate %s: %w", tableName, err) } - logZeroRowsAffected(result, "ApplyCDCUpdate", tableName) - return nil + return result, nil } // ApplyCDCDelete performs DELETE using CDC row data. diff --git a/db/database_manager.go b/db/database_manager.go index 4b84263d..b1614d50 100644 --- a/db/database_manager.go +++ b/db/database_manager.go @@ -45,19 +45,20 @@ type DatabaseProvider interface { // so one of them can close a database outside mu while the others wait; no GC // pass ever takes lifecycleMu. type DatabaseManager struct { - lifecycleMu sync.Mutex - mu sync.RWMutex - databases map[string]*ReplicatedDatabase - detached map[string]*detachedDatabase // user databases out of service for a snapshot restore - systemDB *ReplicatedDatabase - dataDir string - nodeID uint64 - clock *hlc.Clock - cdcHub CDCHub // CDC notification hub, can be nil - vecIndexMgr *VectorIndexManager // Optional vector index manager - autoIncClaimStore *AutoIncClaimStore // AUTO_INCREMENT claim store, backed by systemDB - membershipView atomic.Pointer[func() int] // this node's view of total cluster membership - gcMembershipFunc atomic.Pointer[func() []uint64] // current member node ids (self included, not REMOVED); source for each database's GC safe position + lifecycleMu sync.Mutex + mu sync.RWMutex + databases map[string]*ReplicatedDatabase + detached map[string]*detachedDatabase // user databases out of service for a snapshot restore + systemDB *ReplicatedDatabase + dataDir string + nodeID uint64 + clock *hlc.Clock + cdcHub CDCHub // CDC notification hub, can be nil + vecIndexMgr *VectorIndexManager // Optional vector index manager + autoIncClaimStore *AutoIncClaimStore // AUTO_INCREMENT claim store, backed by systemDB + membershipView atomic.Pointer[func() int] // this node's view of total cluster membership + gcMembershipFunc atomic.Pointer[func() []uint64] // current member node ids (self included, not REMOVED); source for each database's GC safe position + tombstoneMemberCount atomic.Pointer[func() int] // members for the tombstone horizon (SetTombstoneMemberCountFunc) } // SetClusterMembership installs view as the source of this node's view of @@ -87,6 +88,15 @@ func (dm *DatabaseManager) SetGCMembershipFunc(view func() []uint64) { dm.gcMembershipFunc.Store(&view) } +// SetTombstoneMemberCountFunc installs count as the member count every +// database's tombstone GC bounds its horizon with (purgeTombstones): every +// member whose log could still hold an entry, which is every member this +// node has known, REMOVED ones included. count returns 0 while that is not +// yet known, and no tombstone is purged. Until this is called, none is. +func (dm *DatabaseManager) SetTombstoneMemberCountFunc(count func() int) { + dm.tombstoneMemberCount.Store(&count) +} + // DatabaseMetadata represents database registry information type DatabaseMetadata struct { Name string @@ -403,6 +413,14 @@ func (dm *DatabaseManager) wireGCCoordination(mdb *ReplicatedDatabase, dbName st return safe, true }) + txnMgr.SetMemberCountFunc(func() int { + countPtr := dm.tombstoneMemberCount.Load() + if countPtr == nil { + return 0 + } + return (*countPtr)() + }) + // Wire CDC notifier if available if dm.cdcHub != nil { txnMgr.SetNotifier(dm.cdcHub) diff --git a/db/db_integration.go b/db/db_integration.go index ca7b987f..a223feb4 100644 --- a/db/db_integration.go +++ b/db/db_integration.go @@ -242,6 +242,10 @@ func NewReplicatedDatabase(dbPath string, nodeID uint64, clock *hlc.Clock, metaS closeAll() return nil, err } + if err := ensureRowVersionTable(writeDB); err != nil { + closeAll() + return nil, err + } if err := repairAppliedTxnMetadata(writeDB, metaStore, o.dbName); err != nil { closeAll() return nil, err @@ -261,6 +265,10 @@ func NewReplicatedDatabase(dbPath string, nodeID uint64, clock *hlc.Clock, metaS // Create transaction manager (uses write connection + MetaStore + schema cache) txnMgr := NewTransactionManager(writeDB, metaStore, clock, schemaCache) + if err := seedClockFromLog(clock, metaStore); err != nil { + closeAll() + return nil, err + } // Create batch committer for SQLite-level batching (opens its own optimized connection) var batchCommitter *SQLiteBatchCommitter diff --git a/db/fk_cascade_cdc_test.go b/db/fk_cascade_cdc_test.go index 1e936321..e78150aa 100644 --- a/db/fk_cascade_cdc_test.go +++ b/db/fk_cascade_cdc_test.go @@ -7,6 +7,7 @@ import ( "context" "testing" + "github.com/maxpert/marmot/hlc" "github.com/stretchr/testify/require" ) @@ -156,7 +157,7 @@ func TestFKCascadeCDC_ReplicaOwnCascadeThenRedundantDeleteNoOps(t *testing.T) { require.NotNil(t, parentEntry) require.NotNil(t, childEntry) - require.NoError(t, ApplyCDCEntry(replica.GetWriteDB(), &schemaCacheAdapter{cache: replica.schemaCache}, parentEntry)) + require.NoError(t, applyEntryInTx(t, replica.GetWriteDB(), &schemaCacheAdapter{cache: replica.schemaCache}, parentEntry, hlc.Timestamp{WallTime: 1, NodeID: 1})) var childCountAfterParentDelete int require.NoError(t, replica.GetWriteDB().QueryRow(`SELECT COUNT(*) FROM child`).Scan(&childCountAfterParentDelete)) @@ -164,7 +165,7 @@ func TestFKCascadeCDC_ReplicaOwnCascadeThenRedundantDeleteNoOps(t *testing.T) { // Now apply the source's redundant, explicit child-delete CDC entry: // must be a harmless no-op, not an error. - require.NoError(t, ApplyCDCEntry(replica.GetWriteDB(), &schemaCacheAdapter{cache: replica.schemaCache}, childEntry)) + require.NoError(t, applyEntryInTx(t, replica.GetWriteDB(), &schemaCacheAdapter{cache: replica.schemaCache}, childEntry, hlc.Timestamp{WallTime: 1, NodeID: 1})) var finalParentCount, finalChildCount int require.NoError(t, replica.GetWriteDB().QueryRow(`SELECT COUNT(*) FROM parent`).Scan(&finalParentCount)) diff --git a/db/gc_log_test.go b/db/gc_log_test.go index 3a35bfbd..793f7164 100644 --- a/db/gc_log_test.go +++ b/db/gc_log_test.go @@ -46,7 +46,7 @@ func TestListCommittedLogOrderingLimitMoreStable(t *testing.T) { if len(entries) != 2 { t.Fatalf("page 1 len = %d, want 2", len(entries)) } - if entries[0] != positions[0] || entries[1] != positions[1] { + if entries[0].LogPosition != positions[0] || entries[1].LogPosition != positions[1] { t.Fatalf("page 1 = %v, want %v", entries, positions[:2]) } if !more { @@ -57,7 +57,7 @@ func TestListCommittedLogOrderingLimitMoreStable(t *testing.T) { } // Second page: continue after the last entry of page 1. - entries2, _, more2, err := store.ListCommittedLog(entries[len(entries)-1], 10) + entries2, _, more2, err := store.ListCommittedLog(entries[len(entries)-1].LogPosition, 10) if err != nil { t.Fatalf("ListCommittedLog page 2: %v", err) } @@ -65,7 +65,7 @@ func TestListCommittedLogOrderingLimitMoreStable(t *testing.T) { t.Fatalf("page 2 len = %d, want 3", len(entries2)) } for i, want := range positions[2:] { - if entries2[i] != want { + if entries2[i].LogPosition != want { t.Fatalf("page 2[%d] = %v, want %v", i, entries2[i], want) } } @@ -99,7 +99,7 @@ func TestListCommittedLogExcludesUncommitted(t *testing.T) { if err != nil { t.Fatalf("ListCommittedLog: %v", err) } - if len(entries) != 1 || entries[0] != committed { + if len(entries) != 1 || entries[0].LogPosition != committed { t.Fatalf("entries = %v, want only %v", entries, committed) } } diff --git a/db/load_data_bench_test.go b/db/load_data_bench_test.go new file mode 100644 index 00000000..79b3c927 --- /dev/null +++ b/db/load_data_bench_test.go @@ -0,0 +1,46 @@ +package db + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/maxpert/marmot/hlc" + "github.com/stretchr/testify/require" +) + +// loadDataBenchRows is how many rows each benchmarked LOAD DATA carries. +const loadDataBenchRows = 10_000 + +// BenchmarkReplayLoadData replays one LOAD DATA of loadDataBenchRows new +// rows per iteration. +func BenchmarkReplayLoadData(b *testing.B) { + dbMgr, err := NewDatabaseManager(b.TempDir(), 1, hlc.NewClock(1)) + require.NoError(b, err) + defer dbMgr.Close() + require.NoError(b, dbMgr.CreateDatabase("app")) + mdb, err := dbMgr.GetDatabase("app") + require.NoError(b, err) + _, err = mdb.GetWriteDB().Exec(`CREATE TABLE docs (id INTEGER PRIMARY KEY, title TEXT)`) + require.NoError(b, err) + require.NoError(b, mdb.ReloadSchema()) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + var payload strings.Builder + for r := 0; r < loadDataBenchRows; r++ { + fmt.Fprintf(&payload, "%d,title-%d\n", i*loadDataBenchRows+r, r) + } + row := &EncodedCapturedRow{ + Table: "docs", Op: uint8(OpTypeLoadData), + LoadSQL: "LOAD DATA LOCAL INFILE 'x.csv' INTO TABLE docs FIELDS TERMINATED BY ',' LINES TERMINATED BY '\\n' (id, title)", + LoadData: []byte(payload.String()), + } + txn := &ReplayTxn{TxnID: uint64(i + 1), OriginNodeID: 2, CommitTS: hlc.Timestamp{WallTime: int64(i + 1), NodeID: 2}, Rows: []*EncodedCapturedRow{row}} + b.StartTimer() + _, err := mdb.ApplyReplayedTxn(context.Background(), txn, false) + require.NoError(b, err) + } +} diff --git a/db/local_replicator.go b/db/local_replicator.go index 02ebb8ee..d50191cf 100644 --- a/db/local_replicator.go +++ b/db/local_replicator.go @@ -68,6 +68,7 @@ func (lr *LocalReplicator) handleCommit(ctx context.Context, req *coordinator.Re TxnID: req.TxnID, Database: req.Database, Statements: req.Statements, + CommitTS: req.CommitTS, } result := lr.engine.Commit(ctx, engineReq) diff --git a/db/log_position.go b/db/log_position.go index 52d9d6f7..d03eb61d 100644 --- a/db/log_position.go +++ b/db/log_position.go @@ -8,6 +8,8 @@ import ( "sync/atomic" "github.com/cockroachdb/pebble" + "github.com/maxpert/marmot/encoding" + "github.com/maxpert/marmot/hlc" "github.com/rs/zerolog/log" ) @@ -353,3 +355,37 @@ func computeInitialLogSeq(db *pebble.DB) (uint64, error) { return max + 1, nil } + +// CommittedLogEntry is one committed log entry: its position, and the commit +// timestamp its transaction committed at (NodeID is its origin). The +// timestamp is stored with the position itself, so a listed entry carries +// it however its transaction's other records fare. It is zero for an entry +// written before positions carried one. +type CommittedLogEntry struct { + LogPosition + CommitTS hlc.Timestamp +} + +// logEntryStamp is the seq-index value: the entry's commit timestamp. +type logEntryStamp struct { + Wall int64 `msgpack:"w"` + Logical int32 `msgpack:"l"` + Node uint64 `msgpack:"n"` +} + +func encodeLogEntryStamp(commitTS hlc.Timestamp) ([]byte, error) { + return encoding.Marshal(logEntryStamp{Wall: commitTS.WallTime, Logical: commitTS.Logical, Node: commitTS.NodeID}) +} + +// decodeLogEntryStamp decodes a seq-index value; an empty one, from before +// positions carried a timestamp, is the zero timestamp. +func decodeLogEntryStamp(val []byte) (hlc.Timestamp, error) { + if len(val) == 0 { + return hlc.Timestamp{}, nil + } + var stamp logEntryStamp + if err := encoding.Unmarshal(val, &stamp); err != nil { + return hlc.Timestamp{}, fmt.Errorf("decode log entry commit timestamp: %w", err) + } + return hlc.Timestamp{WallTime: stamp.Wall, Logical: stamp.Logical, NodeID: stamp.Node}, nil +} diff --git a/db/meta_store.go b/db/meta_store.go index 7d57afd6..9639c904 100644 --- a/db/meta_store.go +++ b/db/meta_store.go @@ -125,8 +125,8 @@ type MetaStore interface { // after `after`, in position order, restricted to COMMITTED // transactions with Seq <= the stable point read at the start of the // call, up to limit. more is true when further stable entries exist - // beyond the returned page. - ListCommittedLog(after LogPosition, limit int) (entries []LogPosition, stable uint64, more bool, err error) + // beyond the returned page. Each entry carries its commit timestamp. + ListCommittedLog(after LogPosition, limit int) (entries []CommittedLogEntry, stable uint64, more bool, err error) // GetPullCursor and SetPullCursor persist C[self,peer,d]: this node's // pull position in peerNodeID's log for this database. @@ -207,6 +207,9 @@ type MetaStore interface { // Aggregation queries for anti-entropy GetMaxCommittedTxnID() (uint64, error) + // MaxCommitWall returns the largest commit wall time this store has + // logged, local or replayed, surviving restarts; 0 for an empty log. + MaxCommitWall() (int64, error) GetCommittedTxnCount() (int64, error) // Streaming for anti-entropy delta sync diff --git a/db/meta_store_memory.go b/db/meta_store_memory.go index eee168ef..aecc275e 100644 --- a/db/meta_store_memory.go +++ b/db/meta_store_memory.go @@ -261,7 +261,7 @@ func (m *MemoryMetaStore) StableSeq() uint64 { } // ListCommittedLog delegates to Pebble. -func (m *MemoryMetaStore) ListCommittedLog(after LogPosition, limit int) ([]LogPosition, uint64, bool, error) { +func (m *MemoryMetaStore) ListCommittedLog(after LogPosition, limit int) ([]CommittedLogEntry, uint64, bool, error) { return m.pebble.ListCommittedLog(after, limit) } @@ -459,6 +459,11 @@ func (m *MemoryMetaStore) CleanupOldTransactionRecords(minRetention, maxRetentio return m.pebble.CleanupOldTransactionRecords(minRetention, maxRetention, safe) } +// MaxCommitWall delegates to Pebble. +func (m *MemoryMetaStore) MaxCommitWall() (int64, error) { + return m.pebble.MaxCommitWall() +} + // GetMaxCommittedTxnID delegates to Pebble. func (m *MemoryMetaStore) GetMaxCommittedTxnID() (uint64, error) { return m.pebble.GetMaxCommittedTxnID() diff --git a/db/meta_store_pebble.go b/db/meta_store_pebble.go index 5fc623d0..a20c634d 100644 --- a/db/meta_store_pebble.go +++ b/db/meta_store_pebble.go @@ -1119,8 +1119,12 @@ func (s *PebbleMetaStore) CommitTransaction(txnID uint64, commitTS hlc.Timestamp return err } - // Add to sequence index (kept for backward compatibility and GC) - if err := batch.Set(pebbleTxnSeqKey(seqNum, txnID), nil, nil); err != nil { + // Add to sequence index, carrying the commit timestamp with the position + stamp, err := encodeLogEntryStamp(commitTS) + if err != nil { + return err + } + if err := batch.Set(pebbleTxnSeqKey(seqNum, txnID), stamp, nil); err != nil { return err } @@ -1133,6 +1137,9 @@ func (s *PebbleMetaStore) CommitTransaction(txnID uint64, commitTS hlc.Timestamp if err := s.counters.UpdateMaxInBatch(batch, "max_committed_txn_id", int64(txnID)); err != nil { return err } + if err := s.counters.UpdateMaxInBatch(batch, maxCommitWallCounter, commitTS.WallTime); err != nil { + return err + } if err := s.counters.IncInBatch(batch, "committed_txn_count", 1); err != nil { return err } @@ -1223,8 +1230,12 @@ func (s *PebbleMetaStore) StoreReplayedTransaction(txnID, originNodeID uint64, c // Heartbeat not needed for replayed transactions (already committed) - // Add to sequence index (kept for backward compatibility and GC) - if err := batch.Set(pebbleTxnSeqKey(seqNum, txnID), nil, nil); err != nil { + // Add to sequence index, carrying the commit timestamp with the position + stamp, err := encodeLogEntryStamp(hlc.Timestamp{WallTime: commitTS.WallTime, Logical: commitTS.Logical, NodeID: originNodeID}) + if err != nil { + return err + } + if err := batch.Set(pebbleTxnSeqKey(seqNum, txnID), stamp, nil); err != nil { return err } @@ -1236,6 +1247,9 @@ func (s *PebbleMetaStore) StoreReplayedTransaction(txnID, originNodeID uint64, c if err := s.counters.UpdateMaxInBatch(batch, "max_committed_txn_id", int64(txnID)); err != nil { return err } + if err := s.counters.UpdateMaxInBatch(batch, maxCommitWallCounter, commitTS.WallTime); err != nil { + return err + } if err := s.counters.IncInBatch(batch, "committed_txn_count", 1); err != nil { return err } @@ -1737,7 +1751,7 @@ func (s *PebbleMetaStore) GetIntent(tableName, intentKey string) (*WriteIntentRe // Seq <= the stable point read at the start of the call (StableSeq, taken // before the iterator opens; see logSeqTracker's doc comment for why that // snapshot is safe), up to limit. -func (s *PebbleMetaStore) ListCommittedLog(after LogPosition, limit int) ([]LogPosition, uint64, bool, error) { +func (s *PebbleMetaStore) ListCommittedLog(after LogPosition, limit int) ([]CommittedLogEntry, uint64, bool, error) { stable := s.StableSeq() if limit <= 0 { return nil, stable, false, nil @@ -1751,7 +1765,7 @@ func (s *PebbleMetaStore) ListCommittedLog(after LogPosition, limit int) ([]LogP } defer iter.Close() - var entries []LogPosition + var entries []CommittedLogEntry more := false for iter.SeekGE(lower); iter.Valid(); iter.Next() { key := iter.Key() @@ -1781,7 +1795,11 @@ func (s *PebbleMetaStore) ListCommittedLog(after LogPosition, limit int) ([]LogP more = true break } - entries = append(entries, pos) + commitTS, err := decodeLogEntryStamp(iter.Value()) + if err != nil { + return nil, stable, false, err + } + entries = append(entries, CommittedLogEntry{LogPosition: pos, CommitTS: commitTS}) } if err := iter.Error(); err != nil { return nil, stable, false, err @@ -2514,6 +2532,15 @@ func (s *PebbleMetaStore) GetMaxSeqNum() (uint64, error) { return maxSeqNumFromIndex(s.db) } +// maxCommitWallCounter records the largest commit wall time this store has +// logged, local or replayed (MaxCommitWall). +const maxCommitWallCounter = "max_commit_wall" + +// MaxCommitWall implements MetaStore. +func (s *PebbleMetaStore) MaxCommitWall() (int64, error) { + return s.counters.Load(maxCommitWallCounter) +} + // GetMaxCommittedTxnID returns the maximum committed transaction ID func (s *PebbleMetaStore) GetMaxCommittedTxnID() (uint64, error) { return s.counters.LoadUint64("max_committed_txn_id") diff --git a/db/replay_apply.go b/db/replay_apply.go index 8a3a5696..6d82f369 100644 --- a/db/replay_apply.go +++ b/db/replay_apply.go @@ -41,6 +41,12 @@ var ErrReplayPending = errors.New("replay: local transaction is PENDING") // HolderTxnID. var ErrReplayIntentConflict = errors.New("replay: row held by a different local transaction") +// ErrReplayRowWithoutKey is returned by ApplyReplayedTxn, before anything +// is logged or applied, for a transaction carrying a DML row with no intent +// key: such a row cannot be versioned, so it is neither applied unversioned +// nor skipped. The puller counts it as the transaction's failed attempt. +var ErrReplayRowWithoutKey = errors.New("replay: DML row carries no intent key") + // ReplayIntentConflictError is ErrReplayIntentConflict's concrete form: it // additionally names the local transaction (HolderTxnID) whose intent or CDC // row lock blocked the replay - either the intent holder or the CDC row-lock @@ -187,11 +193,13 @@ func (mdb *ReplicatedDatabase) DiscardAbandonedBegin(txnID uint64) error { // 5. One SQLite tx: re-check the marker inside the tx (authoritative - the // read-only check above can race a concurrent apply); if present, roll // back and return applied=false. Otherwise apply every remaining row (DML -// via ApplyCDCValues, DDL via ApplyReplayedDDL with owner=t.OriginNodeID, -// LOAD DATA via ApplyLoadDataInTx), finish any schema change, bump the -// schema version iff the txn carries DDL, write the marker, commit. -// 6. After commit: reload the schema cache and refresh the read pool if DDL -// changed it (warn-only, matching handleReplay's existing contract), +// and LOAD DATA versioned per row at t.CommitTS, skipping a row a newer +// change already owns, DDL via ApplyReplayedDDL with owner=t.OriginNodeID), +// finish any schema change, bump the schema version iff the txn carries +// DDL, write the marker, commit. +// 6. After commit: merge t.CommitTS into this node's clock, reload the +// schema cache and refresh the read pool if DDL changed it (warn-only, +// matching handleReplay's existing contract), // update the cached schema version, then apply committed vector CDC for // the DML rows (idempotent by txn/seq; errors are logged, not returned - // the row data itself already committed). @@ -293,8 +301,9 @@ func (mdb *ReplicatedDatabase) commitReplayedTx(ctx context.Context, t *ReplayTx // anything if this node itself holds t.TxnID PENDING (ErrReplayPending), or // if a DML row t touches is locally held, by intent or CDC row lock, by a // different transaction (*ReplayIntentConflictError, matching -// errors.Is(err, ErrReplayIntentConflict)). Returns t's existing log -// record, if any, for the caller to reuse its SeqNum. +// errors.Is(err, ErrReplayIntentConflict)), or if a DML row carries no +// intent key (ErrReplayRowWithoutKey). Returns t's existing log record, if +// any, for the caller to reuse its SeqNum. func (mdb *ReplicatedDatabase) replayPrecheck(t *ReplayTxn) (*TransactionRecord, error) { rec, err := mdb.metaStore.GetTransaction(t.TxnID) if err != nil { @@ -305,9 +314,12 @@ func (mdb *ReplicatedDatabase) replayPrecheck(t *ReplayTxn) (*TransactionRecord, } for _, row := range t.Rows { - if !isDMLOp(OpType(row.Op)) || len(row.IntentKey) == 0 { + if !isDMLOp(OpType(row.Op)) { continue } + if len(row.IntentKey) == 0 { + return nil, fmt.Errorf("%w: txn %d, table %s", ErrReplayRowWithoutKey, t.TxnID, row.Table) + } key := string(row.IntentKey) if intent, ierr := mdb.metaStore.GetIntent(row.Table, key); ierr == nil && intent != nil && intent.TxnID != t.TxnID { return nil, &ReplayIntentConflictError{HolderTxnID: intent.TxnID} @@ -351,20 +363,30 @@ func (mdb *ReplicatedDatabase) applyReplayVectorControl(ctx context.Context, t * return nil } -// applyReplayedRowsInTx applies every row of t inside tx - DML via -// ApplyCDCValues, DDL via ApplyReplayedDDL (accumulating into change), LOAD -// DATA via ApplyLoadDataInTx - and returns the CDC entries to notify vector -// CDC with after commit, and whether any row was DDL. +// applyReplayedRowsInTx applies every row of t inside tx - DML and LOAD +// DATA versioned per row at t.CommitTS (versionedApplier), DDL via +// ApplyReplayedDDL (accumulating into change) - and returns +// the CDC entries of the DML rows actually applied, to notify vector CDC with +// after commit, and whether any row was DDL. func (mdb *ReplicatedDatabase) applyReplayedRowsInTx(ctx context.Context, tx *sql.Tx, t *ReplayTxn, seqNum uint64, schemaAdapter *schemaCacheAdapter, change *SchemaChange) ([]common.CDCEntry, bool, error) { hasDDL := false dmlEntries := make([]common.CDCEntry, 0, len(t.Rows)) + applier, err := newVersionedApplier(tx, schemaAdapter) + if err != nil { + return nil, false, err + } + defer applier.Close() for _, row := range t.Rows { switch OpType(row.Op) { case OpTypeInsert, OpTypeReplace, OpTypeUpdate, OpTypeDelete: - if err := ApplyCDCValues(tx, schemaAdapter, OpType(row.Op), row.Table, row.OldValues, row.NewValues); err != nil { + applied, err := applier.apply(OpType(row.Op), row.Table, row.IntentKey, row.OldValues, row.NewValues, t.CommitTS) + if err != nil { return nil, false, fmt.Errorf("apply replayed CDC row for %s: %w", row.Table, err) } + if !applied { + continue + } dmlEntries = append(dmlEntries, common.CDCEntry{ Table: row.Table, IntentKey: row.IntentKey, @@ -380,7 +402,7 @@ func (mdb *ReplicatedDatabase) applyReplayedRowsInTx(ctx context.Context, tx *sq } hasDDL = true case OpTypeLoadData: - if _, err := ApplyLoadDataInTx(tx, row.LoadSQL, row.LoadData); err != nil { + if err := applier.applyLoadData(row.LoadSQL, row.LoadData, t.CommitTS); err != nil { return nil, false, fmt.Errorf("apply replayed LOAD DATA: %w", err) } case OpTypeVectorIndex: @@ -395,11 +417,14 @@ func (mdb *ReplicatedDatabase) applyReplayedRowsInTx(ctx context.Context, tx *sq } // afterReplayCommit runs ApplyReplayedTxn's step 6 post-commit refresh: -// reload the schema cache and refresh the read pool when DDL changed it, +// merge t's commit timestamp into this node's clock, so a transaction this +// node prepares next on the same rows is ordered after it, reload the +// schema cache and refresh the read pool when DDL changed it, // advance the cached schema version, and apply committed vector CDC for the // DML rows. Every failure here is logged, not returned - the row data itself // already committed in SQLite. func (mdb *ReplicatedDatabase) afterReplayCommit(ctx context.Context, t *ReplayTxn, seqNum uint64, change *SchemaChange, hasDDL bool, newSchemaVersion uint64, dmlEntries []common.CDCEntry) { + mdb.txnMgr.clock.Update(t.CommitTS) if !change.Empty() { if err := mdb.ReloadSchema(); err != nil { log.Warn().Err(err).Uint64("txn_id", t.TxnID).Str("database", mdb.dbName).Msg("ApplyReplayedTxn: failed to reload schema after DDL") @@ -506,7 +531,7 @@ func (mdb *ReplicatedDatabase) ReapplyLocalLog(ctx context.Context) (int, error) } for _, pos := range entries { - after = pos + after = pos.LogPosition if markers[pos.TxnID] { continue } diff --git a/db/replication_engine.go b/db/replication_engine.go index 47f49320..f1757ee7 100644 --- a/db/replication_engine.go +++ b/db/replication_engine.go @@ -61,6 +61,9 @@ type CommitRequest struct { TxnID uint64 Database string Statements []protocol.Statement // decision metadata; DML row images are durable from PREPARE + // CommitTS is the commit timestamp the coordinator decided, identical on + // every node; zero from a coordinator too old to send one. + CommitTS hlc.Timestamp } // CommitResult contains the result of the commit phase @@ -841,17 +844,18 @@ func (re *ReplicationEngine) Commit(ctx context.Context, req *CommitRequest) *Co // The gate is the statement flag, not a store lookup: reading intents on // every commit would put a Pebble scan on the write path. The flag decides // only whether to look; every value written comes from the intent. - result, _ := commitPreparedTxn(re.dbMgr, replicatedDB, txn, req.Database, statementsCarryAutoIDClaim(req.Statements)) + result, _ := commitPreparedTxn(re.dbMgr, replicatedDB, txn, req.Database, req.CommitTS, statementsCarryAutoIDClaim(req.Statements)) return result } // CommitLocallyPrepared commits database's durably prepared transaction // txnID through the same local commit path a COMMIT RPC takes, for a // transaction some peer's committed log proves was decided COMMITTED while -// this node never received (or failed to apply) its COMMIT. With no +// this node never received (or failed to apply) its COMMIT, with the commit +// timestamp that log records for it (zero if the peer sent none). With no // coordinator statements to read the claim flag from, whether the // transaction carries an AUTO_INCREMENT claim is read from its intents. -func (dm *DatabaseManager) CommitLocallyPrepared(database string, txnID uint64) error { +func (dm *DatabaseManager) CommitLocallyPrepared(database string, txnID uint64, commitTS hlc.Timestamp) error { replicatedDB, err := dm.GetDatabase(database) if err != nil { return fmt.Errorf("database %s: %w", database, err) @@ -874,7 +878,7 @@ func (dm *DatabaseManager) CommitLocallyPrepared(database string, txnID uint64) break } } - _, err = commitPreparedTxn(dm, replicatedDB, txn, database, carriesClaim) + _, err = commitPreparedTxn(dm, replicatedDB, txn, database, commitTS, carriesClaim) return err } @@ -891,7 +895,7 @@ func committedLocally(replicatedDB *ReplicatedDatabase, txnID uint64) bool { // and refresh the read pool after DDL. It returns the result to answer a // COMMIT with and the underlying error (nil when that result is a success, // including for a transaction another caller already committed). -func commitPreparedTxn(dbMgr DatabaseProvider, replicatedDB *ReplicatedDatabase, txn *Transaction, database string, applyClaims bool) (*CommitResult, error) { +func commitPreparedTxn(dbMgr DatabaseProvider, replicatedDB *ReplicatedDatabase, txn *Transaction, database string, commitTS hlc.Timestamp, applyClaims bool) (*CommitResult, error) { // Apply any AUTO_INCREMENT range claim BEFORE committing, and refuse the // commit if it cannot be applied (the claim's first invariant: a // participant must never ACK COMMIT unless the claim row is durably in its @@ -919,7 +923,7 @@ func commitPreparedTxn(dbMgr DatabaseProvider, replicatedDB *ReplicatedDatabase, } return claimErr } - err := replicatedDB.GetTransactionManager().CommitTransactionAfter(txn, applyClaim) + err := replicatedDB.GetTransactionManager().CommitTransactionAfter(txn, commitTS, applyClaim) switch { case claimErr != nil: return &CommitResult{Success: false, ClaimNotApplicable: errors.Is(claimErr, ErrAutoIncClaimNotApplicable), diff --git a/db/row_version.go b/db/row_version.go new file mode 100644 index 00000000..3a2a781a --- /dev/null +++ b/db/row_version.go @@ -0,0 +1,458 @@ +package db + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "reflect" + "strings" + "time" + + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol/filter" +) + +// rowVersionTableDDL creates the per-row version table. Every replicated row +// change stamps its row's key with the change's commit version (the +// transaction's commit HLC, tie-broken by its origin node), in the same +// SQLite transaction as the change itself; a change older than the stored +// version is skipped. deleted marks a tombstone: the version of a DELETE, +// kept so an older INSERT or UPDATE cannot bring the row back. pk is the +// row's intent key exactly as the CDC hook captured it. +const rowVersionTableDDL = ` +CREATE TABLE IF NOT EXISTS __marmot_row_version ( + tbl TEXT NOT NULL, + pk BLOB NOT NULL, + wall INTEGER NOT NULL, + logical INTEGER NOT NULL, + node INTEGER NOT NULL, + deleted INTEGER NOT NULL, + PRIMARY KEY (tbl, pk) +) WITHOUT ROWID` + +// rowVersionTombstoneIndexDDL indexes tombstones by age, so the purge finds +// them without scanning every live row's version. +const rowVersionTombstoneIndexDDL = ` +CREATE INDEX IF NOT EXISTS __marmot_row_version_tombstones ON __marmot_row_version (wall) WHERE deleted = 1` + +// stampRowVersionSQL stores a version unless the stored one is newer. Equal +// versions belong to the same transaction, which may change a row twice, so +// they win too. Zero rows changed means the stored version is newer. +const stampRowVersionSQL = stampRowVersionInsert + stampRowVersionTuple + stampRowVersionUpsert + +const ( + stampRowVersionInsert = "INSERT INTO __marmot_row_version (tbl, pk, wall, logical, node, deleted) VALUES " + stampRowVersionTuple = "(?, ?, ?, ?, ?, ?)" + stampRowVersionUpsert = ` +ON CONFLICT (tbl, pk) DO UPDATE SET + wall = excluded.wall, logical = excluded.logical, node = excluded.node, deleted = excluded.deleted +WHERE (excluded.wall, excluded.logical, excluded.node) >= (wall, logical, node)` +) + +// stampRowVersionsSQL is stampRowVersionSQL for n rows at once. +func stampRowVersionsSQL(n int) string { + values := strings.TrimSuffix(strings.Repeat(stampRowVersionTuple+", ", n), ", ") + return stampRowVersionInsert + values + stampRowVersionUpsert +} + +// ensureRowVersionTable creates __marmot_row_version and its tombstone +// index in db if they are absent. +func ensureRowVersionTable(db *sql.DB) error { + if db == nil { + return nil + } + if _, err := db.Exec(rowVersionTableDDL); err != nil { + return fmt.Errorf("create row version table: %w", err) + } + if _, err := db.Exec(rowVersionTombstoneIndexDDL); err != nil { + return fmt.Errorf("create row version tombstone index: %w", err) + } + return nil +} + +// versionedApplier applies replicated row changes inside one SQLite +// transaction, each versioned per row, with the version stamp prepared once +// for the transaction. +type versionedApplier struct { + tx *sql.Tx + schema CDCSchemaProvider + stamp *sql.Stmt +} + +// newVersionedApplier prepares the version stamp on tx. The caller closes +// the applier before tx ends. +func newVersionedApplier(tx *sql.Tx, schema CDCSchemaProvider) (*versionedApplier, error) { + stamp, err := tx.Prepare(stampRowVersionSQL) + if err != nil { + return nil, fmt.Errorf("prepare row version stamp: %w", err) + } + return &versionedApplier{tx: tx, schema: schema, stamp: stamp}, nil +} + +// Close releases the prepared stamp. +func (a *versionedApplier) Close() error { + return a.stamp.Close() +} + +// stampVersion stores version for (table, key) unless a newer version is +// already stored, and reports whether it was stored: whether the change +// carrying it may be applied. +func (a *versionedApplier) stampVersion(table string, key []byte, version hlc.Timestamp, deleted bool) (bool, error) { + res, err := a.stamp.Exec(table, key, version.WallTime, version.Logical, version.NodeID, deleted) + if err != nil { + return false, fmt.Errorf("stamp row version for %s: %w", table, err) + } + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("stamp row version for %s: %w", table, err) + } + return n > 0, nil +} + +// pkKeyFromValues encodes the key of the row whose column values are values +// exactly as the preupdate hook encodes it at capture (extractPKFromValues): +// the hook sees TEXT and BLOB alike as []byte, so both map to PKTypeBytes, +// and every integer width decoded from msgpack maps to PKTypeInt64. rowid +// tables carry their rowid under rowidColumnKey, which is also their only +// primary key name. A key column values lacks is read from fallback, as +// ApplyCDCUpdate's WHERE clause does. +func pkKeyFromValues(table string, primaryKeys []string, values, fallback map[string][]byte) ([]byte, error) { + pkValues := make([]filter.TypedPKValue, len(primaryKeys)) + for i, col := range primaryKeys { + encoded, ok := values[col] + if !ok { + encoded, ok = fallback[col] + } + if !ok { + return nil, fmt.Errorf("row version %s: primary key column %s missing", table, col) + } + v, err := unmarshalCDCValue(encoded) + if err != nil { + return nil, fmt.Errorf("row version %s: decode primary key column %s: %w", table, col, err) + } + pkValues[i] = capturedPKValue(v) + } + return filter.EncodeIntentKey(table, pkValues), nil +} + +// capturedPKValue maps a decoded CDC value to the typed PK value the hook +// captured for it. +func capturedPKValue(v interface{}) filter.TypedPKValue { + rv := reflect.ValueOf(v) + switch { + case v == nil: + return filter.TypedPKValue{Type: filter.PKTypeNull} + case rv.CanInt(): + return valueToTypedPK(rv.Int()) + case rv.CanUint(): + return valueToTypedPK(int64(rv.Uint())) + case rv.CanFloat(): + return valueToTypedPK(rv.Float()) + } + if s, ok := v.(string); ok { + return valueToTypedPK([]byte(s)) + } + return valueToTypedPK(v) +} + +// apply applies one replicated DML row change, versioned +// by the committing transaction's commit timestamp: it stamps the row's +// version first and applies the change only if no newer version is stored, +// reporting whether it did. A DELETE leaves a tombstone. An UPDATE whose +// row is absent inserts its after-image; one that moves the row to a new +// primary key also tombstones the old key, and leaves the old row alone if +// a newer change owns it. +func (a *versionedApplier) apply(opType OpType, table string, intentKey []byte, oldValues, newValues map[string][]byte, version hlc.Timestamp) (bool, error) { + if len(intentKey) == 0 { + return false, fmt.Errorf("row version %s: change carries no intent key", table) + } + if opType == OpTypeUpdate { + return a.applyUpdate(table, intentKey, oldValues, newValues, version) + } + won, err := a.stampVersion(table, intentKey, version, opType == OpTypeDelete) + if err != nil || !won { + return false, err + } + switch opType { + case OpTypeInsert, OpTypeReplace: + return true, ApplyCDCInsert(a.tx, table, newValues) + case OpTypeDelete: + return true, ApplyCDCDelete(a.tx, a.schema, table, oldValues) + default: + return false, fmt.Errorf("unsupported operation type for CDC: %v", opType) + } +} + +// applyUpdate applies an UPDATE whose new image's key is newKey. +// The new key is stamped at version. If the UPDATE moved the row to another +// primary key, the old key is also tombstoned at version, and each key's +// outcome is applied on its own: with both stamps won the row is updated +// in place; with only the old key's won the old row is removed; with only +// the new key's won the after-image is written there. An update in place +// that matches no row inserts the after-image. It reports whether any part +// of the change was applied. +func (a *versionedApplier) applyUpdate(table string, newKey []byte, oldValues, newValues map[string][]byte, version hlc.Timestamp) (bool, error) { + primaryKeys, err := a.schema.GetPrimaryKeys(table) + if err != nil { + return false, fmt.Errorf("ApplyCDCUpdate %s: failed to get primary keys: %w", table, err) + } + newWon, err := a.stampVersion(table, newKey, version, false) + if err != nil { + return false, err + } + oldWon := newWon + if primaryKeyMoved(primaryKeys, oldValues, newValues) { + oldKey, err := pkKeyFromValues(table, primaryKeys, oldValues, newValues) + if err != nil { + return false, err + } + if !bytes.Equal(oldKey, newKey) { + if oldWon, err = a.stampVersion(table, oldKey, version, true); err != nil { + return false, err + } + } + } + switch { + case oldWon && newWon: + return true, updateInPlaceOrInsert(a.tx, a.schema, table, oldValues, newValues) + case oldWon: + return true, ApplyCDCDelete(a.tx, a.schema, table, oldValues) + case newWon: + return true, ApplyCDCInsert(a.tx, table, newValues) + default: + return false, nil + } +} + +// updateInPlaceOrInsert runs the UPDATE and, when it matches no row, +// inserts the after-image instead. Like ApplyCDCInsert's INSERT OR REPLACE, +// it replaces another row that holds the new key: the new key's stamp won, +// so that row is older than this change, and in commit order it was gone +// before it. +func updateInPlaceOrInsert(exec CDCExecutor, schema CDCSchemaProvider, table string, oldValues, newValues map[string][]byte) error { + result, err := execCDCUpdate(exec, schema, "UPDATE OR REPLACE", table, oldValues, newValues) + if err != nil { + return err + } + if n, err := result.RowsAffected(); err != nil || n > 0 { + return err + } + return ApplyCDCInsert(exec, table, newValues) +} + +// applyEntry applies a committing transaction's captured row through +// apply. A row a newer version already owns is skipped, as on replay. +func (a *versionedApplier) applyEntry(entry *IntentEntry, commitTS hlc.Timestamp) error { + _, err := a.apply(OpType(entry.Operation), entry.Table, entry.IntentKey, entry.OldValues, entry.NewValues, commitTS) + return err +} + +// tablePrimaryKeysSQL lists every user table's primary-key columns in key +// order; a table without a declared key lists one row with a NULL column +// (its key is the rowid). +const tablePrimaryKeysSQL = ` +SELECT m.name, p.name FROM sqlite_master m +LEFT JOIN pragma_table_info(m.name) p ON p.pk > 0 +WHERE m.type = 'table' AND m.name NOT LIKE 'sqlite\_%' ESCAPE '\' AND m.name NOT LIKE '\_\_marmot%' ESCAPE '\' +ORDER BY m.name, p.pk` + +// tablePrimaryKeys returns every user table's primary-key column list, +// joined into one comparable string. +func tablePrimaryKeys(ctx context.Context, q schemaQuerier) (map[string]string, error) { + rows, err := q.QueryContext(ctx, tablePrimaryKeysSQL) + if err != nil { + return nil, fmt.Errorf("list primary keys: %w", err) + } + defer rows.Close() + + keys := make(map[string]string) + for rows.Next() { + var table string + var col sql.NullString + if err := rows.Scan(&table, &col); err != nil { + return nil, fmt.Errorf("list primary keys: %w", err) + } + keys[table] += col.String + "\x00" + } + return keys, rows.Err() +} + +// reconcileRowVersions keeps the version table in step with one DDL +// statement, given every user table's primary key before and after it: a +// single table replaced by a single new one with the same key is a rename, +// and its versions move to the new name; any other removed table loses its +// versions, as does a kept table whose key columns changed, since its +// stored keys no longer name its rows. Those rows fall back to version +// zero, so the next change to each applies unconditionally. Each statement +// is reconciled alone, so a rebuild spread over several (create a copy, +// drop the original, rename the copy) drops the original's versions: the +// rebuilt table's rows start at version zero. +func reconcileRowVersions(ctx context.Context, q sqlExecQuerier, before, after map[string]string) error { + var removed, created []string + for name := range before { + if _, ok := after[name]; !ok { + removed = append(removed, name) + } + } + for name := range after { + if _, ok := before[name]; !ok { + created = append(created, name) + } + } + if len(removed) == 1 && len(created) == 1 && before[removed[0]] == after[created[0]] { + return renameRowVersions(ctx, q, removed[0], created[0]) + } + for name, key := range before { + if newKey, kept := after[name]; kept && newKey == key { + continue + } + if err := execRowVersionDDL(ctx, q, "DELETE FROM __marmot_row_version WHERE tbl = ?", name); err != nil { + return err + } + } + return nil +} + +func execRowVersionDDL(ctx context.Context, q sqlExecQuerier, stmt string, args ...interface{}) error { + if _, err := q.ExecContext(ctx, stmt, args...); err != nil { + return fmt.Errorf("update row versions after DDL: %w", err) + } + return nil +} + +const ( + // tombstonePurgeChunk is how many tombstones one purge statement + // deletes: each chunk is its own short write, so a GC tick never holds + // the database's single writer for long. + tombstonePurgeChunk = 1000 + // tombstonePurgeMaxChunks bounds one GC tick's purge; the rest waits + // for the next tick. + tombstonePurgeMaxChunks = 100 + // purgeTombstonesSQL deletes up to a chunk of tombstones older than a + // horizon, found through the partial tombstone index. + purgeTombstonesSQL = `DELETE FROM __marmot_row_version WHERE (tbl, pk) IN ( + SELECT tbl, pk FROM __marmot_row_version WHERE deleted = 1 AND wall < ? LIMIT ?)` +) + +// purgeTombstones deletes the tombstones no replayed change can still be +// older than. A log entry stays in a member's log at most gcMaxRetention +// after that member recorded it, and a member records an entry only at its +// commit or by pulling it from a log that still holds it, at most once +// (the applied marker stops a second time); so n members hold it at most +// n*gcMaxRetention past its commit. A tombstone older than that outlives +// every change it could have to beat. n is every member this node has +// known, REMOVED ones included, from the persisted registry +// (SetTombstoneMemberCountFunc); while it is unknown (0) nothing is purged. +// The bound assumes clock skew plus commit timeout well under +// gcMaxRetention. With unlimited retention (gcMaxRetention 0) no bound +// exists and tombstones are kept. Tombstones go in chunks +// (tombstonePurgeChunk), each its own short write. It returns how many were +// deleted. +func (tm *TransactionManager) purgeTombstones() (int64, error) { + tm.mu.RLock() + members := tm.memberCount + dbName := tm.databaseName + tm.mu.RUnlock() + if tm.gcMaxRetention <= 0 || members == nil || dbName == SystemDatabaseName { + return 0, nil + } + n := members() + if n <= 0 { + return 0, nil + } + horizon := time.Now().Add(-time.Duration(n) * tm.gcMaxRetention).UnixNano() + var purged int64 + for chunk := 0; chunk < tombstonePurgeMaxChunks; chunk++ { + res, err := tm.db.Exec(purgeTombstonesSQL, horizon, tombstonePurgeChunk) + if err != nil { + return purged, fmt.Errorf("purge row version tombstones: %w", err) + } + deleted, err := res.RowsAffected() + if err != nil { + return purged, fmt.Errorf("purge row version tombstones: %w", err) + } + purged += deleted + if deleted < tombstonePurgeChunk { + break + } + } + return purged, nil +} + +// renamedVersion is one version row being moved to a renamed table. +type renamedVersion struct { + key []byte + wall, logical, node int64 + deleted bool +} + +// renameRowVersions moves from's versions to to, re-encoding each intent +// key, which names its table, for the new name. +func renameRowVersions(ctx context.Context, q sqlExecQuerier, from, to string) error { + rows, err := q.QueryContext(ctx, "SELECT pk, wall, logical, node, deleted FROM __marmot_row_version WHERE tbl = ?", from) + if err != nil { + return fmt.Errorf("read row versions of %s: %w", from, err) + } + var moved []renamedVersion + for rows.Next() { + var v renamedVersion + if err := rows.Scan(&v.key, &v.wall, &v.logical, &v.node, &v.deleted); err != nil { + rows.Close() + return fmt.Errorf("read row versions of %s: %w", from, err) + } + _, pkValues, _, err := filter.DecodeIntentKey(v.key) + if err != nil { + rows.Close() + return fmt.Errorf("re-key row version of %s: %w", from, err) + } + v.key = filter.EncodeIntentKey(to, pkValues) + moved = append(moved, v) + } + iterErr := rows.Err() + if err := rows.Close(); err != nil || iterErr != nil { + return fmt.Errorf("read row versions of %s: %w", from, errors.Join(iterErr, err)) + } + if err := execRowVersionDDL(ctx, q, "DELETE FROM __marmot_row_version WHERE tbl = ?", from); err != nil { + return err + } + for _, v := range moved { + if err := execRowVersionDDL(ctx, q, "INSERT INTO __marmot_row_version (tbl, pk, wall, logical, node, deleted) VALUES (?, ?, ?, ?, ?, ?)", + to, v.key, v.wall, v.logical, v.node, v.deleted); err != nil { + return err + } + } + return nil +} + +// primaryKeyMoved reports whether an UPDATE's old and new images differ in +// any primary-key column's encoded value; a column the old image lacks is +// taken from the new one, as ApplyCDCUpdate does. +func primaryKeyMoved(primaryKeys []string, oldValues, newValues map[string][]byte) bool { + for _, col := range primaryKeys { + oldV, ok := oldValues[col] + if ok && !bytes.Equal(oldV, newValues[col]) { + return true + } + } + return false +} + +// seedClockFromLog advances clock past the largest commit wall time +// metaStore has logged, so a node restarted with its wall clock set back +// never issues a commit timestamp at or below one it already committed or +// applied. +func seedClockFromLog(clock *hlc.Clock, metaStore MetaStore) error { + if clock == nil || metaStore == nil { + return nil + } + wall, err := metaStore.MaxCommitWall() + if err != nil { + return fmt.Errorf("read largest logged commit time: %w", err) + } + if wall > 0 { + clock.Update(hlc.Timestamp{WallTime: wall + 1}) + } + return nil +} diff --git a/db/row_version_load.go b/db/row_version_load.go new file mode 100644 index 00000000..efb96020 --- /dev/null +++ b/db/row_version_load.go @@ -0,0 +1,234 @@ +package db + +import ( + "database/sql" + "fmt" + "strings" + + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/maxpert/marmot/protocol/filter" +) + +const ( + // loadBatchSavepoint scopes one batch of loaded rows, undone whole if + // any of its rows loses its version stamp. + loadBatchSavepoint = "marmot_load_batch" + // loadRowSavepoint scopes one loaded row, so a row that loses its + // version stamp is undone alone. + loadRowSavepoint = "marmot_load_row" + // loadBatchMaxRows and loadBatchMaxParams bound one batch's statement. + loadBatchMaxRows = 500 + loadBatchMaxParams = 30_000 +) + +// tablePrimaryKeyColumns reads table's primary-key columns in key order +// through tx, which sees any DDL earlier in the same transaction; a table +// without a declared key is keyed by its rowid, named as the CDC hook +// names it. +func tablePrimaryKeyColumns(tx *sql.Tx, table string) ([]string, error) { + rows, err := tx.Query("SELECT name FROM pragma_table_info(?) WHERE pk > 0 ORDER BY pk", table) + if err != nil { + return nil, fmt.Errorf("read primary key of %s: %w", table, err) + } + defer rows.Close() + var cols []string + for rows.Next() { + var col string + if err := rows.Scan(&col); err != nil { + return nil, fmt.Errorf("read primary key of %s: %w", table, err) + } + cols = append(cols, col) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read primary key of %s: %w", table, err) + } + if len(cols) == 0 { + cols = []string{rowidColumnKey} + } + return cols, nil +} + +// loadApply is one LOAD DATA payload being applied by a versionedApplier. +type loadApply struct { + *versionedApplier + load *protocol.LoadDataRows + primaryKeys []string + version hlc.Timestamp +} + +// applyLoadData applies a LOAD DATA LOCAL payload inside the applier's tx, +// each row versioned at version like a replicated INSERT: a row is kept +// only if its stored key's version stamp wins, so a replayed load never +// overwrites a newer row and an older change replayed later never removes +// a loaded row. A row landing on the key of an older row replaces it, as +// ApplyCDCInsert does: the origin inserted it into a key that was free in +// commit order. Rows go in batches under a savepoint; a batch in which any +// row loses is undone and applied again one row at a time. +func (a *versionedApplier) applyLoadData(loadSQL string, data []byte, version hlc.Timestamp) error { + load, err := protocol.ParseLoadDataRows(loadSQL, data) + if err != nil || len(load.Rows) == 0 { + return err + } + primaryKeys, err := tablePrimaryKeyColumns(a.tx, load.Table) + if err != nil { + return err + } + l := &loadApply{versionedApplier: a, load: load, primaryKeys: primaryKeys, version: version} + for start := 0; start < len(load.Rows); { + end := loadBatchEnd(load.Rows, start) + if err := l.applyBatch(load.Rows[start:end]); err != nil { + return err + } + start = end + } + return nil +} + +// loadBatchEnd returns the end of the batch starting at start: rows of the +// same field count, within loadBatchMaxRows and loadBatchMaxParams. +func loadBatchEnd(rows [][]string, start int) int { + fields := len(rows[start]) + limit := loadBatchMaxRows + if fields > 0 && loadBatchMaxParams/fields < limit { + limit = loadBatchMaxParams / fields + } + end := start + 1 + for end < len(rows) && end-start < limit && len(rows[end]) == fields { + end++ + } + return end +} + +// applyBatch writes rows under loadBatchSavepoint and stamps every key they +// were stored under. If any stamp loses, the batch is undone and its rows +// applied one at a time instead. +func (l *loadApply) applyBatch(rows [][]string) error { + if err := l.savepoint(loadBatchSavepoint); err != nil { + return err + } + keys, err := l.insertReturningKeys(rows) + allWon := false + if err == nil { + allWon, err = l.stampAll(keys) + } + if err == nil && !allWon { + err = l.rollbackTo(loadBatchSavepoint) + } + if err == nil { + err = l.release(loadBatchSavepoint) + } + if err != nil || allWon { + return err + } + for _, row := range rows { + if err := l.applyRow(row); err != nil { + return err + } + } + return nil +} + +// applyRow writes one row under loadRowSavepoint and keeps it only if its +// key's version stamp wins. +func (l *loadApply) applyRow(row []string) error { + if err := l.savepoint(loadRowSavepoint); err != nil { + return err + } + keys, err := l.insertReturningKeys([][]string{row}) + won := false + if err == nil { + won, err = l.stampAll(keys) + } + if err == nil && !won { + err = l.rollbackTo(loadRowSavepoint) + } + if err != nil { + return err + } + return l.release(loadRowSavepoint) +} + +func (l *loadApply) savepoint(name string) error { + return l.execLoad("SAVEPOINT " + name) +} + +func (l *loadApply) rollbackTo(name string) error { + return l.execLoad("ROLLBACK TO " + name) +} + +func (l *loadApply) release(name string) error { + return l.execLoad("RELEASE " + name) +} + +func (l *loadApply) execLoad(stmt string) error { + if _, err := l.tx.Exec(stmt); err != nil { + return fmt.Errorf("LOAD DATA into %s: %s: %w", l.load.Table, stmt, err) + } + return nil +} + +// insertReturningKeys writes rows with one INSERT OR REPLACE and returns the +// intent key of each row as stored, after SQLite's type affinity, encoded +// as the CDC hook encodes it. +func (l *loadApply) insertReturningKeys(rows [][]string) ([][]byte, error) { + var cols string + if len(l.load.Columns) > 0 { + cols = " (" + strings.Join(quoteSQLiteIdentList(l.load.Columns), ", ") + ")" + } + tuple := "(" + strings.TrimSuffix(strings.Repeat("?, ", len(rows[0])), ", ") + ")" + stmt := fmt.Sprintf("INSERT OR REPLACE INTO %s%s VALUES %s RETURNING %s", quoteSQLiteIdent(l.load.Table), cols, + strings.TrimSuffix(strings.Repeat(tuple+", ", len(rows)), ", "), strings.Join(quoteSQLiteIdentList(l.primaryKeys), ", ")) + args := make([]interface{}, 0, len(rows)*len(rows[0])) + for _, row := range rows { + for _, field := range row { + args = append(args, field) + } + } + result, err := l.tx.Query(stmt, args...) + if err != nil { + return nil, fmt.Errorf("LOAD DATA into %s: %w", l.load.Table, err) + } + defer result.Close() + keys := make([][]byte, 0, len(rows)) + pk := make([]interface{}, len(l.primaryKeys)) + ptrs := make([]interface{}, len(pk)) + for i := range pk { + ptrs[i] = &pk[i] + } + pkValues := make([]filter.TypedPKValue, len(pk)) + for result.Next() { + if err := result.Scan(ptrs...); err != nil { + return nil, fmt.Errorf("LOAD DATA key of %s: %w", l.load.Table, err) + } + for i, v := range pk { + pkValues[i] = capturedPKValue(v) + } + keys = append(keys, filter.EncodeIntentKey(l.load.Table, pkValues)) + } + return keys, result.Err() +} + +// stampAll stamps every key at the load's version with one upsert and +// reports whether every stamp won: the upsert changes one row per key it +// wrote, and none for a key whose stored version is newer. A key repeated +// in keys (a payload loading one key twice) conflicts with its own earlier +// row at an equal version, which wins, so it still counts once per row. +func (l *loadApply) stampAll(keys [][]byte) (bool, error) { + if len(keys) == 0 { + return true, nil + } + args := make([]interface{}, 0, len(keys)*6) + for _, key := range keys { + args = append(args, l.load.Table, key, l.version.WallTime, l.version.Logical, l.version.NodeID, false) + } + res, err := l.tx.Exec(stampRowVersionsSQL(len(keys)), args...) + if err != nil { + return false, fmt.Errorf("stamp row versions for %s: %w", l.load.Table, err) + } + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("stamp row versions for %s: %w", l.load.Table, err) + } + return n == int64(len(keys)), nil +} diff --git a/db/row_version_test.go b/db/row_version_test.go new file mode 100644 index 00000000..c947e611 --- /dev/null +++ b/db/row_version_test.go @@ -0,0 +1,650 @@ +package db + +import ( + "context" + "database/sql" + "strings" + "testing" + "time" + + "github.com/maxpert/marmot/coordinator" + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol" + "github.com/maxpert/marmot/protocol/filter" + "github.com/stretchr/testify/require" +) + +// docsKey is the intent key the CDC hook captures for docs.id = id. +func docsKey(id int64) []byte { + return filter.EncodeIntentKey("docs", []filter.TypedPKValue{{Type: filter.PKTypeInt64, Value: filter.EncodeInt64(id)}}) +} + +func docsInsert(id int64, title string) *EncodedCapturedRow { + return &EncodedCapturedRow{ + Table: "docs", + Op: uint8(OpTypeInsert), + IntentKey: docsKey(id), + NewValues: encodeTestValues(map[string]interface{}{"id": id, "title": title}), + } +} + +func docsUpdate(id int64, oldTitle, newTitle string) *EncodedCapturedRow { + return &EncodedCapturedRow{ + Table: "docs", + Op: uint8(OpTypeUpdate), + IntentKey: docsKey(id), + OldValues: encodeTestValues(map[string]interface{}{"id": id, "title": oldTitle}), + NewValues: encodeTestValues(map[string]interface{}{"id": id, "title": newTitle}), + } +} + +func docsDelete(id int64, title string) *EncodedCapturedRow { + return &EncodedCapturedRow{ + Table: "docs", + Op: uint8(OpTypeDelete), + IntentKey: docsKey(id), + OldValues: encodeTestValues(map[string]interface{}{"id": id, "title": title}), + } +} + +// replayAt applies rows as one pulled transaction committed at wall. +func replayAt(t *testing.T, mdb *ReplicatedDatabase, txnID uint64, wall int64, rows ...*EncodedCapturedRow) { + t.Helper() + applied, err := mdb.ApplyReplayedTxn(context.Background(), &ReplayTxn{ + TxnID: txnID, + OriginNodeID: 2, + CommitTS: hlc.Timestamp{WallTime: wall, NodeID: 2}, + Rows: rows, + }, true) + require.NoError(t, err) + require.True(t, applied, "txn %d must count as applied even when its rows lose", txnID) +} + +// docsTitle returns docs.title for id, and whether the row exists. +func docsTitle(t *testing.T, mdb *ReplicatedDatabase, id int64) (string, bool) { + t.Helper() + var title string + err := mdb.GetWriteDB().QueryRow("SELECT title FROM docs WHERE id = ?", id).Scan(&title) + if err != nil { + return "", false + } + return title, true +} + +func TestReplayOlderUpdateAfterNewerKeepsNewerImage(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + + replayAt(t, mdb, 1, 5, docsInsert(1, "v0")) + replayAt(t, mdb, 3, 20, docsUpdate(1, "v1", "v2")) + replayAt(t, mdb, 2, 10, docsUpdate(1, "v0", "v1")) + + title, ok := docsTitle(t, mdb, 1) + require.True(t, ok) + require.Equal(t, "v2", title) +} + +func TestReplayOlderInsertDoesNotResurrectNewerDelete(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + + replayAt(t, mdb, 5, 20, docsDelete(1, "v1")) + replayAt(t, mdb, 4, 10, docsInsert(1, "v1")) + + _, ok := docsTitle(t, mdb, 1) + require.False(t, ok, "an older INSERT must not resurrect a row a newer DELETE removed") +} + +func TestReplayUpdateBeforeItsInsertEndsWithNewestImage(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + + replayAt(t, mdb, 7, 20, docsUpdate(1, "v1", "v2")) + replayAt(t, mdb, 6, 10, docsInsert(1, "v1")) + + title, ok := docsTitle(t, mdb, 1) + require.True(t, ok) + require.Equal(t, "v2", title) +} + +// storedVersion reads the version table's row for (table, key). +func storedVersion(t *testing.T, mdb *ReplicatedDatabase, table string, key []byte) (hlc.Timestamp, bool, bool) { + t.Helper() + var v hlc.Timestamp + var deleted bool + err := mdb.GetWriteDB().QueryRow("SELECT wall, logical, node, deleted FROM __marmot_row_version WHERE tbl = ? AND pk = ?", table, key). + Scan(&v.WallTime, &v.Logical, &v.NodeID, &deleted) + if err == sql.ErrNoRows { + return hlc.Timestamp{}, false, false + } + require.NoError(t, err) + return v, deleted, true +} + +func TestCommitStampsRowsWithTheCoordinatorsCommitTimestamp(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + tm := mdb.GetTransactionManager() + txn := beginPreparedDMLTxn(t, tm, mdb.GetMetaStore(), docsInsert(1, "a")) + + decided := hlc.Timestamp{WallTime: time.Now().Add(time.Hour).UnixNano(), Logical: 3, NodeID: txn.NodeID} + require.NoError(t, tm.CommitTransactionAfter(txn, decided, nil)) + require.True(t, hlc.After(tm.clock.Now(), decided), "a node that commits a transaction must order what it prepares next after it") + + got, deleted, ok := storedVersion(t, mdb, "docs", docsKey(1)) + require.True(t, ok) + require.False(t, deleted) + require.Equal(t, decided, got) + rec, err := mdb.GetMetaStore().GetTransaction(txn.ID) + require.NoError(t, err) + require.Equal(t, decided.WallTime, rec.CommitTSWall, "the log must serve the same timestamp to pullers") + require.Equal(t, decided.Logical, rec.CommitTSLogical) +} + +// Both commit paths - the batch committer and the direct one used when +// batching is off - version rows the same way. +func TestCommitOlderThanStoredVersionLeavesRowAlone(t *testing.T) { + for _, batched := range []bool{true, false} { + _, mdb, _ := newReplayTestDatabase(t) + tm := mdb.GetTransactionManager() + if !batched { + tm.batchCommitter = nil + } + replayAt(t, mdb, 1, 50, docsInsert(1, "newer")) + + txn := beginPreparedDMLTxn(t, tm, mdb.GetMetaStore(), docsUpdate(1, "newer", "older")) + require.NoError(t, tm.CommitTransactionAfter(txn, hlc.Timestamp{WallTime: 10, NodeID: txn.NodeID}, nil)) + + title, ok := docsTitle(t, mdb, 1) + require.True(t, ok) + require.Equal(t, "newer", title, "batched=%v", batched) + } +} + +// The key computed from a row's decoded CDC values must equal the one the +// hook captured from the raw values, for every PK storage class. +func TestPKKeyFromValuesMatchesCapturedKey(t *testing.T) { + cases := []struct { + name string + raw interface{} // as the preupdate hook hands it over + cdc interface{} // as capture encodes it (TEXT becomes string) + }{ + {"integer", int64(7), int64(7)}, + {"small integer", int64(1), int64(1)}, + {"negative integer", int64(-300), int64(-300)}, + {"text", []byte("abc"), "abc"}, + {"blob", []byte{0, 1, 2}, []byte{0, 1, 2}}, + {"real", 2.5, 2.5}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + captured := filter.EncodeIntentKey("tb", []filter.TypedPKValue{valueToTypedPK(c.raw), valueToTypedPK(int64(9))}) + got, err := pkKeyFromValues("tb", []string{"a", "b"}, encodeTestValues(map[string]interface{}{"a": c.cdc}), encodeTestValues(map[string]interface{}{"b": int64(9)})) + require.NoError(t, err) + require.Equal(t, captured, got) + }) + } +} + +// An UPDATE that moves a row to a new key tombstones the old key, so an +// older change to the old key cannot bring the old row back. +func TestUpdateMovingPrimaryKeyTombstonesOldKey(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + replayAt(t, mdb, 1, 10, docsInsert(1, "a")) + move := &EncodedCapturedRow{ + Table: "docs", Op: uint8(OpTypeUpdate), IntentKey: docsKey(2), + OldValues: encodeTestValues(map[string]interface{}{"id": int64(1), "title": "a"}), + NewValues: encodeTestValues(map[string]interface{}{"id": int64(2), "title": "a"}), + } + replayAt(t, mdb, 3, 30, move) + replayAt(t, mdb, 2, 20, docsUpdate(1, "a", "stale")) + + _, ok := docsTitle(t, mdb, 1) + require.False(t, ok, "an older change must not bring the moved-away row back") + title, ok := docsTitle(t, mdb, 2) + require.True(t, ok) + require.Equal(t, "a", title) + _, deleted, ok := storedVersion(t, mdb, "docs", docsKey(1)) + require.True(t, ok) + require.True(t, deleted) +} + +func versionCount(t *testing.T, mdb *ReplicatedDatabase, table string) int { + t.Helper() + var n int + require.NoError(t, mdb.GetWriteDB().QueryRow("SELECT COUNT(*) FROM __marmot_row_version WHERE tbl = ?", table).Scan(&n)) + return n +} + +func TestDropTableRemovesItsVersions(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + replayAt(t, mdb, 1, 10, docsInsert(1, "a"), docsInsert(2, "b")) + require.Equal(t, 2, versionCount(t, mdb, "docs")) + + replayDDL(t, mdb, "docs", "DROP TABLE docs") + require.Zero(t, versionCount(t, mdb, "docs")) +} + +func TestRenameTableCarriesItsVersionsUnderTheNewKeys(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + replayAt(t, mdb, 1, 10, docsInsert(1, "a")) + + replayDDL(t, mdb, "docs", "ALTER TABLE docs RENAME TO papers") + require.Zero(t, versionCount(t, mdb, "docs")) + papersKey := filter.EncodeIntentKey("papers", []filter.TypedPKValue{{Type: filter.PKTypeInt64, Value: filter.EncodeInt64(1)}}) + got, deleted, ok := storedVersion(t, mdb, "papers", papersKey) + require.True(t, ok) + require.False(t, deleted) + require.Equal(t, hlc.Timestamp{WallTime: 10, NodeID: 2}, got) +} + +// A statement that keeps a table but changes its primary-key columns leaves +// its stored keys naming nothing, so its versions are cleared. +func TestPrimaryKeyChangeClearsVersions(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + replayAt(t, mdb, 1, 10, docsInsert(1, "a")) + tx, err := mdb.GetWriteDB().Begin() + require.NoError(t, err) + defer tx.Rollback() + + before := map[string]string{"docs": "id\x00"} + require.NoError(t, reconcileRowVersions(context.Background(), tx, before, map[string]string{"docs": "title\x00"})) + var n int + require.NoError(t, tx.QueryRow("SELECT COUNT(*) FROM __marmot_row_version WHERE tbl = 'docs'").Scan(&n)) + require.Zero(t, n) +} + +func TestPurgeTombstonesDeletesOnlyThoseOlderThanEveryMembersRetention(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + tm := mdb.GetTransactionManager() + now := time.Now() + replayAt(t, mdb, 1, now.Add(-3*time.Hour).UnixNano(), docsDelete(1, "old")) + replayAt(t, mdb, 2, now.Add(-time.Hour).UnixNano(), docsDelete(2, "recent")) + replayAt(t, mdb, 3, now.Add(-3*time.Hour).UnixNano(), docsInsert(3, "live")) + + tm.gcMaxRetention = 0 + tm.SetMemberCountFunc(func() int { return 2 }) + purged, err := tm.purgeTombstones() + require.NoError(t, err) + require.Zero(t, purged, "unlimited retention bounds nothing, so every tombstone stays") + + tm.gcMaxRetention = time.Hour + purged, err = tm.purgeTombstones() + require.NoError(t, err) + require.Equal(t, int64(1), purged) + _, _, ok := storedVersion(t, mdb, "docs", docsKey(1)) + require.False(t, ok, "a tombstone older than 2 members x 1h retention goes") + _, _, ok = storedVersion(t, mdb, "docs", docsKey(2)) + require.True(t, ok, "a younger tombstone stays") + _, _, ok = storedVersion(t, mdb, "docs", docsKey(3)) + require.True(t, ok, "a live row's version is never purged") +} + +// benchmarkApplyUpdate applies b.N updates of one row, each in its own +// transaction, through apply. +func benchmarkApplyUpdate(b *testing.B, apply func(tx *sql.Tx, schema CDCSchemaProvider, row *EncodedCapturedRow, version hlc.Timestamp) error) { + dbMgr, err := NewDatabaseManager(b.TempDir(), 1, hlc.NewClock(1)) + require.NoError(b, err) + defer dbMgr.Close() + require.NoError(b, dbMgr.CreateDatabase("app")) + mdb, err := dbMgr.GetDatabase("app") + require.NoError(b, err) + _, err = mdb.GetWriteDB().Exec(`CREATE TABLE docs (id INTEGER PRIMARY KEY, title TEXT)`) + require.NoError(b, err) + require.NoError(b, mdb.ReloadSchema()) + _, err = mdb.GetWriteDB().Exec(`INSERT INTO docs VALUES (1, 'v')`) + require.NoError(b, err) + schema := &schemaCacheAdapter{cache: mdb.schemaCache} + row := docsUpdate(1, "v", "w") + + b.ResetTimer() + for i := 0; i < b.N; i++ { + tx, err := mdb.GetWriteDB().Begin() + require.NoError(b, err) + require.NoError(b, apply(tx, schema, row, hlc.Timestamp{WallTime: int64(i + 1), NodeID: 1})) + require.NoError(b, tx.Commit()) + } +} + +func BenchmarkApplyUpdateVersioned(b *testing.B) { + benchmarkApplyUpdate(b, func(tx *sql.Tx, schema CDCSchemaProvider, row *EncodedCapturedRow, version hlc.Timestamp) error { + applier, err := newVersionedApplier(tx, schema) + if err != nil { + return err + } + defer applier.Close() + _, err = applier.apply(OpType(row.Op), row.Table, row.IntentKey, row.OldValues, row.NewValues, version) + return err + }) +} + +func BenchmarkApplyUpdateUnversioned(b *testing.B) { + benchmarkApplyUpdate(b, func(tx *sql.Tx, schema CDCSchemaProvider, row *EncodedCapturedRow, _ hlc.Timestamp) error { + return ApplyCDCValues(tx, schema, OpType(row.Op), row.Table, row.OldValues, row.NewValues) + }) +} + +// The coordinator's own node commits through LocalReplicator with the same +// decided commit timestamp it sends every remote participant. +func TestLocalReplicatorCommitsWithTheDecidedTimestamp(t *testing.T) { + replicator, dm, cleanup := setupTestLocalReplicator(t) + defer cleanup() + require.NoError(t, dm.CreateDatabase("app")) + mdb, err := dm.GetDatabase("app") + require.NoError(t, err) + _, err = mdb.GetWriteDB().Exec(`CREATE TABLE docs (id INTEGER PRIMARY KEY, title TEXT)`) + require.NoError(t, err) + require.NoError(t, mdb.ReloadSchema()) + + ctx := context.Background() + insert := testProtocolDMLStatement(protocol.StatementInsert, "app", "docs", docsKey(1), nil, + encodeTestValues(map[string]interface{}{"id": int64(1), "title": "a"})) + resp, err := replicator.ReplicateTransaction(ctx, 1, &coordinator.ReplicationRequest{ + TxnID: 9, NodeID: 1, Database: "app", Phase: coordinator.PhasePrep, + StartTS: hlc.Timestamp{WallTime: 1, NodeID: 1}, Statements: []protocol.Statement{insert}, + }) + require.NoError(t, err) + require.True(t, resp.Success, resp.Error) + + decided := hlc.Timestamp{WallTime: 888, Logical: 4, NodeID: 1} + resp, err = replicator.ReplicateTransaction(ctx, 1, &coordinator.ReplicationRequest{ + TxnID: 9, NodeID: 1, Database: "app", Phase: coordinator.PhaseCommit, + StartTS: hlc.Timestamp{WallTime: 1, NodeID: 1}, CommitTS: decided, + }) + require.NoError(t, err) + require.True(t, resp.Success, resp.Error) + + got, _, ok := storedVersion(t, mdb, "docs", docsKey(1)) + require.True(t, ok) + require.Equal(t, decided, got) +} + +// A replayed transaction's commit timestamp is merged into the clock, as a +// committed one is, so what this node prepares next is ordered after it. +func TestReplayMergesCommitTimestampIntoClock(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + future := time.Now().Add(time.Hour).UnixNano() + replayAt(t, mdb, 1, future, docsInsert(1, "a")) + require.True(t, hlc.After(mdb.GetTransactionManager().clock.Now(), hlc.Timestamp{WallTime: future, NodeID: 2})) +} + +// docsMove is an UPDATE that moves docs row oldID to newID. +func docsMove(oldID, newID int64, title string) *EncodedCapturedRow { + return &EncodedCapturedRow{ + Table: "docs", Op: uint8(OpTypeUpdate), IntentKey: docsKey(newID), + OldValues: encodeTestValues(map[string]interface{}{"id": oldID, "title": title}), + NewValues: encodeTestValues(map[string]interface{}{"id": newID, "title": title}), + } +} + +// docsRows reads every docs row as id -> title. +func docsRows(t *testing.T, mdb *ReplicatedDatabase) map[int64]string { + t.Helper() + rows, err := mdb.GetWriteDB().Query("SELECT id, title FROM docs") + require.NoError(t, err) + defer rows.Close() + got := map[int64]string{} + for rows.Next() { + var id int64 + var title string + require.NoError(t, rows.Scan(&id, &title)) + got[id] = title + } + require.NoError(t, rows.Err()) + return got +} + +// A key-moving UPDATE whose new key a newer change already owns must still +// retire the old key: insert 1 (v5), move 1->2 (v10), update 2 (v20), in +// commit order and with the move arriving last, end in the same state. +func TestKeyMoveArrivingAfterNewerChangeToNewKeyStillRetiresOldRow(t *testing.T) { + _, inOrder, _ := newReplayTestDatabase(t) + replayAt(t, inOrder, 1, 5, docsInsert(1, "x")) + replayAt(t, inOrder, 2, 10, docsMove(1, 2, "x")) + replayAt(t, inOrder, 3, 20, docsUpdate(2, "x", "y")) + + _, late, _ := newReplayTestDatabase(t) + replayAt(t, late, 1, 5, docsInsert(1, "x")) + replayAt(t, late, 3, 20, docsUpdate(2, "x", "y")) + replayAt(t, late, 2, 10, docsMove(1, 2, "x")) + + require.Equal(t, map[int64]string{2: "y"}, docsRows(t, inOrder)) + require.Equal(t, docsRows(t, inOrder), docsRows(t, late)) +} + +// A move whose new key is still held by an older row, because the delete +// that cleared it has not arrived yet, replaces that row instead of failing +// on the key: insert 2 (v3), delete 2 (v4), insert 1 (v5), move 1->2 (v10), +// with the delete arriving last. +func TestKeyMoveOntoAKeyStillHeldByAnOlderRowReplacesIt(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + replayAt(t, mdb, 1, 3, docsInsert(2, "stale")) + replayAt(t, mdb, 3, 5, docsInsert(1, "x")) + replayAt(t, mdb, 4, 10, docsMove(1, 2, "x")) + replayAt(t, mdb, 2, 4, docsDelete(2, "stale")) + + require.Equal(t, map[int64]string{2: "x"}, docsRows(t, mdb)) +} + +// applyEntryInTx applies entry at commitTS in its own transaction on sqlDB. +func applyEntryInTx(t testing.TB, sqlDB *sql.DB, schema CDCSchemaProvider, entry *IntentEntry, commitTS hlc.Timestamp) error { + t.Helper() + tx, err := sqlDB.Begin() + require.NoError(t, err) + defer tx.Rollback() + applier, err := newVersionedApplier(tx, schema) + require.NoError(t, err) + if err := applier.applyEntry(entry, commitTS); err != nil { + applier.Close() + return err + } + require.NoError(t, applier.Close()) + return tx.Commit() +} + +func docsLoad(lines string) *EncodedCapturedRow { + return &EncodedCapturedRow{ + Table: "docs", Op: uint8(OpTypeLoadData), + LoadSQL: "LOAD DATA LOCAL INFILE 'x.csv' INTO TABLE docs FIELDS TERMINATED BY ',' LINES TERMINATED BY '\\n' (id, title)", + LoadData: []byte(lines), + } +} + +// A loaded row is versioned like an inserted one: an older DELETE replayed +// after the load does not remove it. +func TestOlderDeleteReplayedAfterLoadDataKeepsLoadedRow(t *testing.T) { + _, inOrder, _ := newReplayTestDatabase(t) + replayAt(t, inOrder, 1, 10, docsDelete(5, "old")) + replayAt(t, inOrder, 2, 20, docsLoad("5,b\n")) + + _, late, _ := newReplayTestDatabase(t) + replayAt(t, late, 2, 20, docsLoad("5,b\n")) + replayAt(t, late, 1, 10, docsDelete(5, "old")) + + require.Equal(t, map[int64]string{5: "b"}, docsRows(t, inOrder)) + require.Equal(t, docsRows(t, inOrder), docsRows(t, late)) +} + +// A replayed LOAD DATA older than a row's stored version leaves that row +// alone, and still loads its other rows. +func TestOlderLoadDataReplayedAfterNewerRowSkipsOnlyThatRow(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + replayAt(t, mdb, 2, 30, docsInsert(5, "newer")) + replayAt(t, mdb, 1, 20, docsLoad("5,loaded\n6,loaded\n")) + + require.Equal(t, map[int64]string{5: "newer", 6: "loaded"}, docsRows(t, mdb)) + got, _, ok := storedVersion(t, mdb, "docs", docsKey(6)) + require.True(t, ok) + require.Equal(t, hlc.Timestamp{WallTime: 20, NodeID: 2}, got) +} + +// The commit path versions LOAD DATA rows as replay does. +func TestCommittedLoadDataIsVersionedPerRow(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + replayAt(t, mdb, 1, 30, docsInsert(5, "newer")) + load := docsLoad("5,loaded\n6,loaded\n") + snapshot, err := SerializeData(LoadDataSnapshot{Type: int(protocol.StatementLoadData), SQL: load.LoadSQL, TableName: "docs", Data: load.LoadData}) + require.NoError(t, err) + intents := []*WriteIntentRecord{{IntentType: IntentTypeDDL, TableName: "docs", NodeID: 2, SQLStatement: load.LoadSQL, DataSnapshot: snapshot}} + + commitTS := hlc.Timestamp{WallTime: 20, NodeID: 2} + require.NoError(t, mdb.GetTransactionManager().applyNonDMLIntents(2, commitTS, intents)) + + require.Equal(t, map[int64]string{5: "newer", 6: "loaded"}, docsRows(t, mdb)) + got, _, ok := storedVersion(t, mdb, "docs", docsKey(6)) + require.True(t, ok) + require.Equal(t, commitTS, got) +} + +// The purge finds tombstones through their index, never by scanning every +// row's version, and deletes past one chunk in a single tick. +func TestPurgeTombstonesUsesTheIndexAndSpansChunks(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + var plan strings.Builder + rows, err := mdb.GetWriteDB().Query("EXPLAIN QUERY PLAN SELECT tbl, pk FROM __marmot_row_version WHERE deleted = 1 AND wall < 1 LIMIT 1") + require.NoError(t, err) + for rows.Next() { + var id, parent, notused int + var detail string + require.NoError(t, rows.Scan(&id, &parent, ¬used, &detail)) + plan.WriteString(detail) + } + require.NoError(t, rows.Close()) + require.Contains(t, plan.String(), "__marmot_row_version_tombstones") + + old := time.Now().Add(-3 * time.Hour).UnixNano() + const n = tombstonePurgeChunk*2 + 7 + for i := 0; i < n; i++ { + _, err := mdb.GetWriteDB().Exec("INSERT INTO __marmot_row_version VALUES ('docs', ?, ?, 0, 1, 1)", docsKey(int64(i)), old) + require.NoError(t, err) + } + tm := mdb.GetTransactionManager() + tm.gcMaxRetention = time.Hour + tm.SetMemberCountFunc(func() int { return 2 }) + purged, err := tm.purgeTombstones() + require.NoError(t, err) + require.Equal(t, int64(n), purged) +} + +// A listed log entry carries its commit timestamp from the position itself, +// for a local commit and a replayed one alike. +func TestListedLogEntriesCarryTheirCommitTimestamps(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + tm := mdb.GetTransactionManager() + txn := beginPreparedDMLTxn(t, tm, mdb.GetMetaStore(), docsInsert(1, "a")) + local := hlc.Timestamp{WallTime: 7_000, Logical: 1, NodeID: txn.NodeID} + require.NoError(t, tm.CommitTransactionAfter(txn, local, nil)) + replayAt(t, mdb, 900, 8_000, docsInsert(2, "b")) + + entries, _, _, err := mdb.GetMetaStore().ListCommittedLog(LogPosition{}, 10) + require.NoError(t, err) + got := map[uint64]hlc.Timestamp{} + for _, e := range entries { + got[e.TxnID] = e.CommitTS + } + require.Equal(t, local, got[txn.ID]) + require.Equal(t, hlc.Timestamp{WallTime: 8_000, NodeID: 2}, got[900]) +} + +// A restart with the wall clock behind the last logged commit still issues +// later timestamps: opening a database seeds the clock from its log. +func TestReopenSeedsClockPastTheLastLoggedCommit(t *testing.T) { + dir := t.TempDir() + dbMgr, err := NewDatabaseManager(dir, 1, hlc.NewClock(1)) + require.NoError(t, err) + require.NoError(t, dbMgr.CreateDatabase("app")) + mdb, err := dbMgr.GetDatabase("app") + require.NoError(t, err) + _, err = mdb.GetWriteDB().Exec(`CREATE TABLE docs (id INTEGER PRIMARY KEY, title TEXT)`) + require.NoError(t, err) + require.NoError(t, mdb.ReloadSchema()) + future := time.Now().Add(time.Hour).UnixNano() + replayAt(t, mdb, 1, future, docsInsert(1, "a")) + require.NoError(t, dbMgr.Close()) + + clock := hlc.NewClock(1) + reopened, err := NewDatabaseManager(dir, 1, clock) + require.NoError(t, err) + defer reopened.Close() + _, err = reopened.GetDatabase("app") + require.NoError(t, err) + require.Greater(t, clock.Now().WallTime, future) +} + +// A replayed DML row without an intent key is refused before anything is +// logged or applied, never applied unversioned. +func TestReplayRefusesKeylessRowBeforeLoggingIt(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + keyless := docsInsert(1, "a") + keyless.IntentKey = nil + _, err := mdb.ApplyReplayedTxn(context.Background(), &ReplayTxn{ + TxnID: 5, OriginNodeID: 2, CommitTS: hlc.Timestamp{WallTime: 10, NodeID: 2}, Rows: []*EncodedCapturedRow{keyless}, + }, true) + require.ErrorIs(t, err, ErrReplayRowWithoutKey) + rec, err := mdb.GetMetaStore().GetTransaction(5) + require.NoError(t, err) + require.Nil(t, rec, "nothing may be logged") + require.Empty(t, docsRows(t, mdb)) +} + +// A move whose old key a newer change already owns still writes its row at +// the new key: insert 1 (v5), move 1->2 (v10), insert 1 (v15), delete 1 +// (v20), with the move arriving last. +func TestKeyMoveWhoseOldKeyIsNewerElsewhereStillWritesTheNewKey(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + replayAt(t, mdb, 1, 5, docsInsert(1, "x")) + replayAt(t, mdb, 3, 15, docsInsert(1, "y")) + replayAt(t, mdb, 4, 20, docsDelete(1, "y")) + replayAt(t, mdb, 2, 10, docsMove(1, 2, "x")) + + require.Equal(t, map[int64]string{2: "x"}, docsRows(t, mdb)) +} + +// A loaded row's version key is the key the CDC hook captures for the same +// row, TEXT primary keys included. +func TestLoadedRowVersionKeyMatchesCapturedKeyForTextKey(t *testing.T) { + _, mdb, _ := newReplayTestDatabase(t) + _, err := mdb.GetWriteDB().Exec(`CREATE TABLE tags (name TEXT PRIMARY KEY, n INTEGER)`) + require.NoError(t, err) + require.NoError(t, mdb.ReloadSchema()) + replayAt(t, mdb, 1, 10, &EncodedCapturedRow{ + Table: "tags", Op: uint8(OpTypeLoadData), LoadData: []byte("go,1\n"), + LoadSQL: "LOAD DATA LOCAL INFILE 'x.csv' INTO TABLE tags FIELDS TERMINATED BY ',' LINES TERMINATED BY '\\n' (name, n)", + }) + + captured := filter.EncodeIntentKey("tags", []filter.TypedPKValue{valueToTypedPK([]byte("go"))}) + _, _, ok := storedVersion(t, mdb, "tags", captured) + require.True(t, ok) +} + +// BenchmarkApplyVersionedTxn100Rows applies a 100-row transaction per op +// through one versionedApplier, each row an update of its own row. +func BenchmarkApplyVersionedTxn100Rows(b *testing.B) { + const rowsPerTxn = 100 + dbMgr, err := NewDatabaseManager(b.TempDir(), 1, hlc.NewClock(1)) + require.NoError(b, err) + defer dbMgr.Close() + require.NoError(b, dbMgr.CreateDatabase("app")) + mdb, err := dbMgr.GetDatabase("app") + require.NoError(b, err) + _, err = mdb.GetWriteDB().Exec(`CREATE TABLE docs (id INTEGER PRIMARY KEY, title TEXT)`) + require.NoError(b, err) + require.NoError(b, mdb.ReloadSchema()) + rows := make([]*EncodedCapturedRow, rowsPerTxn) + for i := range rows { + _, err = mdb.GetWriteDB().Exec(`INSERT INTO docs VALUES (?, 'v')`, i) + require.NoError(b, err) + rows[i] = docsUpdate(int64(i), "v", "w") + } + schema := &schemaCacheAdapter{cache: mdb.schemaCache} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + tx, err := mdb.GetWriteDB().Begin() + require.NoError(b, err) + applier, err := newVersionedApplier(tx, schema) + require.NoError(b, err) + for _, row := range rows { + _, err := applier.apply(OpType(row.Op), row.Table, row.IntentKey, row.OldValues, row.NewValues, hlc.Timestamp{WallTime: int64(i + 1), NodeID: 1}) + require.NoError(b, err) + } + require.NoError(b, applier.Close()) + require.NoError(b, tx.Commit()) + } +} diff --git a/db/rowid_sentinel_cdc_test.go b/db/rowid_sentinel_cdc_test.go index 8522f1f3..c49d04af 100644 --- a/db/rowid_sentinel_cdc_test.go +++ b/db/rowid_sentinel_cdc_test.go @@ -69,7 +69,7 @@ func applyEntries(t *testing.T, replica *ReplicatedDatabase, entries []*IntentEn t.Helper() adapter := &schemaCacheAdapter{cache: replica.schemaCache} for _, entry := range entries { - require.NoError(t, ApplyCDCEntry(replica.GetWriteDB(), adapter, entry)) + require.NoError(t, applyEntryInTx(t, replica.GetWriteDB(), adapter, entry, hlc.Timestamp{WallTime: 1, NodeID: 1})) } } diff --git a/db/schema_change.go b/db/schema_change.go index d37ca73a..dda57795 100644 --- a/db/schema_change.go +++ b/db/schema_change.go @@ -38,13 +38,19 @@ type SchemaChange struct { } // Exec runs one DDL statement on q, with the database's table definitions -// read on q just before and just after it, and records what it changed. -// table is the table the statement names, owner the node that authored it. +// and primary keys read on q just before and just after it, records what it +// changed, and brings the row version table in line with it +// (reconcileRowVersions). table is the table the statement names, owner the +// node that authored it. func (c *SchemaChange) Exec(ctx context.Context, q sqlExecQuerier, stmt, table string, owner uint64) error { before, err := tableDefinitions(ctx, q) if err != nil { return fmt.Errorf("failed to read tables before DDL: %w", err) } + keysBefore, err := tablePrimaryKeys(ctx, q) + if err != nil { + return err + } if _, err := q.ExecContext(ctx, stmt); err != nil { return fmt.Errorf("failed to execute DDL statement: %w", err) } @@ -52,6 +58,13 @@ func (c *SchemaChange) Exec(ctx context.Context, q sqlExecQuerier, stmt, table s if err != nil { return fmt.Errorf("failed to read tables after DDL: %w", err) } + keysAfter, err := tablePrimaryKeys(ctx, q) + if err != nil { + return err + } + if err := reconcileRowVersions(ctx, q, keysBefore, keysAfter); err != nil { + return err + } c.record(before, after, owner) c.tables = append(c.tables, ddlTableOwner{table: table, owner: owner}) return nil diff --git a/db/sqlite_batch_committer.go b/db/sqlite_batch_committer.go index 2193b222..5176957d 100644 --- a/db/sqlite_batch_committer.go +++ b/db/sqlite_batch_committer.go @@ -242,6 +242,10 @@ func (bc *SQLiteBatchCommitter) openOptimizedConnection() (*sql.DB, error) { db.Close() return nil, err } + if err := ensureRowVersionTable(db); err != nil { + db.Close() + return nil, err + } return db, nil } @@ -441,8 +445,14 @@ func (bc *SQLiteBatchCommitter) flush(batch map[uint64]*pendingCommit, trigger s return } - // Create schema adapter for the unified applier - schemaAdapter := &schemaCacheAdapter{cache: schemaCache} + applier, err := newVersionedApplier(tx, &schemaCacheAdapter{cache: schemaCache}) + if err != nil { + tx.Rollback() + for _, pc := range batch { + pc.promise.Set(nil, err) + } + return + } // Apply each logical transaction under a savepoint. A failed transaction // rolls back its own row changes without poisoning successful peers in the @@ -458,7 +468,7 @@ func (bc *SQLiteBatchCommitter) flush(batch map[uint64]*pendingCommit, trigger s continue } for _, entry := range pc.cdcEntries { - if err := ApplyCDCEntry(tx, schemaAdapter, entry); err != nil { + if err := applier.applyEntry(entry, pc.commitTS); err != nil { pc.err = err break } @@ -476,6 +486,7 @@ func (bc *SQLiteBatchCommitter) flush(batch map[uint64]*pendingCommit, trigger s } } + applier.Close() // Commit (single fsync for entire batch) if err := tx.Commit(); err != nil { tx.Rollback() diff --git a/db/stale_gc_commit_race_test.go b/db/stale_gc_commit_race_test.go index 1f82703f..69d42e0f 100644 --- a/db/stale_gc_commit_race_test.go +++ b/db/stale_gc_commit_race_test.go @@ -61,7 +61,7 @@ func TestStaleGCAndLocalCommitNeverCommitAnEmptyTransaction(t *testing.T) { go func() { defer wg.Done() for i := 0; i < staleGCRaceTxns; i++ { - _ = dm.CommitLocallyPrepared("testdb", uint64(firstTxn+i)) + _ = dm.CommitLocallyPrepared("testdb", uint64(firstTxn+i), hlc.Timestamp{}) } }() wg.Wait() diff --git a/db/test_helpers.go b/db/test_helpers.go index 1fef8a06..935b2006 100644 --- a/db/test_helpers.go +++ b/db/test_helpers.go @@ -68,6 +68,10 @@ func openTestDBWithMeta(t *testing.T, dbPath string) *testDBWithMetaStore { db.Close() t.Fatalf("Failed to create applied transaction marker table: %v", err) } + if err := ensureRowVersionTable(db); err != nil { + db.Close() + t.Fatalf("Failed to create row version table: %v", err) + } // applyNonDMLIntents bumps __marmot_schema_version for any real DDL // intent inside its one SQLite tx, same as NewReplicatedDatabase would // have created it; this raw test *sql.DB needs it too. diff --git a/db/transaction.go b/db/transaction.go index 1873f4f6..c79c1132 100644 --- a/db/transaction.go +++ b/db/transaction.go @@ -62,6 +62,7 @@ type TransactionManager struct { gcRunning bool databaseName string // Name of database this manager manages gcSafePosition GCSafePositionFunc // Callback for GC's safe deletion position + memberCount func() int // Current member count, for tombstone GC (SetMemberCountFunc) batchCommitter *SQLiteBatchCommitter // SQLite write batcher (nil if disabled) notifier CDCNotifier // Injected, can be nil vectorCDCNotifier VectorCDCNotifier // Injected, can be nil @@ -149,6 +150,14 @@ func (tm *TransactionManager) SetGCSafePositionFunc(fn GCSafePositionFunc) { tm.gcSafePosition = fn } +// SetMemberCountFunc sets the callback tombstone GC (purgeTombstones) uses +// to learn how many members the cluster has. +func (tm *TransactionManager) SetMemberCountFunc(fn func() int) { + tm.mu.Lock() + defer tm.mu.Unlock() + tm.memberCount = fn +} + // SetNotifier sets the CDC notifier for signaling after commits func (tm *TransactionManager) SetNotifier(n CDCNotifier) { tm.mu.Lock() @@ -400,7 +409,7 @@ func (tm *TransactionManager) WriteIntent(txn *Transaction, intentType IntentTyp // idempotence would otherwise mask a duplicate finalizeCommit writing a // second log entry). func (tm *TransactionManager) CommitTransaction(txn *Transaction) error { - return tm.CommitTransactionAfter(txn, nil) + return tm.CommitTransactionAfter(txn, hlc.Timestamp{}, nil) } // CommitTransactionAfter is CommitTransaction with before run first, under @@ -408,7 +417,12 @@ func (tm *TransactionManager) CommitTransaction(txn *Transaction) error { // happen exactly once before the commit (applying an AUTO_INCREMENT claim) // never runs for a transaction another caller already committed. An error // from before aborts the commit and is returned as is. -func (tm *TransactionManager) CommitTransactionAfter(txn *Transaction, before func() error) error { +// +// commitTS is the commit timestamp the transaction's coordinator decided, +// the same on every node; every row the commit writes is versioned with it. +// A zero commitTS (a coordinator or peer too old to send one) falls back to +// a timestamp computed here, as every commit did before. +func (tm *TransactionManager) CommitTransactionAfter(txn *Transaction, commitTS hlc.Timestamp, before func() error) error { txn.mu.Lock() defer txn.mu.Unlock() @@ -437,8 +451,7 @@ func (tm *TransactionManager) CommitTransactionAfter(txn *Transaction, before fu return err } - // Calculate commit timestamp - txn.CommitTS = tm.calculateCommitTS(txn.StartTS) + txn.CommitTS = tm.resolveCommitTS(txn, commitTS) if len(cdcEntries) > 0 { // DML path: apply CDC entries @@ -641,6 +654,22 @@ func (tm *TransactionManager) notifyVectorCDC(txn *Transaction, entries []*Inten } } +// resolveCommitTS returns txn's commit timestamp: decided, the +// coordinator's, merged into this node's clock so any transaction this node +// prepares later is ordered after it, or, when decided is zero, +// calculateCommitTS. Its NodeID is always txn's origin (its coordinator), +// the tie-break every node applying txn uses for its row versions. +func (tm *TransactionManager) resolveCommitTS(txn *Transaction, decided hlc.Timestamp) hlc.Timestamp { + commitTS := decided + if decided.IsZero() { + commitTS = tm.calculateCommitTS(txn.StartTS) + } else { + tm.clock.Update(decided) + } + commitTS.NodeID = txn.NodeID + return commitTS +} + // calculateCommitTS determines the commit timestamp (must be > start_ts). func (tm *TransactionManager) calculateCommitTS(startTS hlc.Timestamp) hlc.Timestamp { commitTS := tm.clock.Now() @@ -722,10 +751,14 @@ func (tm *TransactionManager) applyCDCEntries(txnID uint64, commitTS hlc.Timesta } defer tx.Rollback() - schemaAdapter := &schemaCacheAdapter{cache: tm.schemaCache} + applier, err := newVersionedApplier(tx, &schemaCacheAdapter{cache: tm.schemaCache}) + if err != nil { + return err + } + defer applier.Close() for _, entry := range entries { - if err := ApplyCDCEntry(tx, schemaAdapter, entry); err != nil { + if err := applier.applyEntry(entry, commitTS); err != nil { return fmt.Errorf("failed to write CDC data for %s: %w", entry.Table, err) } } @@ -841,7 +874,7 @@ func (tm *TransactionManager) applyNonDMLIntents(txnID uint64, commitTS hlc.Time } defer tx.Rollback() - change, hasDDL, err := tm.execNonDMLIntentsInTx(ctx, tx, txnID, kinds) + change, hasDDL, err := tm.execNonDMLIntentsInTx(ctx, tx, txnID, commitTS, kinds) if err != nil { return err } @@ -891,15 +924,27 @@ func (tm *TransactionManager) applyNonDMLVectorControl(kinds []nonDMLIntentKind) // (LOAD DATA and DDL, vector control having already run outside it), seeds // AUTO_INCREMENT bases for any DDL this txn ran, and reports the accumulated // schema change and whether any DDL ran. -func (tm *TransactionManager) execNonDMLIntentsInTx(ctx context.Context, tx *sql.Tx, txnID uint64, kinds []nonDMLIntentKind) (SchemaChange, bool, error) { +func (tm *TransactionManager) execNonDMLIntentsInTx(ctx context.Context, tx *sql.Tx, txnID uint64, commitTS hlc.Timestamp, kinds []nonDMLIntentKind) (SchemaChange, bool, error) { var change SchemaChange hasDDL := false + var applier *versionedApplier + defer func() { + if applier != nil { + applier.Close() + } + }() for _, k := range kinds { switch { case k.vector != nil: continue // already applied above case k.load != nil: - if _, err := ApplyLoadDataInTx(tx, k.load.SQL, k.load.Data); err != nil { + if applier == nil { + var err error + if applier, err = newVersionedApplier(tx, &schemaCacheAdapter{cache: tm.schemaCache}); err != nil { + return change, false, err + } + } + if err := applier.applyLoadData(k.load.SQL, k.load.Data, commitTS); err != nil { return change, false, fmt.Errorf("failed to execute LOAD DATA statement: %w", err) } log.Debug().Uint64("txn_id", txnID).Msg("LOAD DATA statement executed") @@ -1245,6 +1290,14 @@ func (tm *TransactionManager) runGarbageCollection() { Int("old_txn_records", oldTxnCount). Msg("GC Phase 2: Cleaned up old data") } + + tombstones, err := tm.purgeTombstones() + if err != nil { + log.Error().Err(err).Msg("GC Phase 2: Failed to purge row version tombstones") + } + if tombstones > 0 { + log.Info().Int64("tombstones", tombstones).Msg("GC Phase 2: Purged row version tombstones") + } } // cleanupStaleTransactions aborts transactions that haven't had a heartbeat within the timeout diff --git a/docs/src/pages/replication.mdx b/docs/src/pages/replication.mdx index 064f0eda..c704ea2c 100644 --- a/docs/src/pages/replication.mdx +++ b/docs/src/pages/replication.mdx @@ -298,6 +298,21 @@ The DML wire payload is the encoded msgpack bytes plus a codec id. Decoded maps | UPDATE | `UPDATE table SET ... WHERE pk=...` using the decoded old primary key image | | DELETE | `DELETE FROM table WHERE pk=...` using the decoded old primary key image | +Every applied row change is versioned per primary key in `__marmot_row_version`, in the same SQLite +transaction, with its transaction's commit timestamp: the HLC the coordinator decides after PREPARE and +sends with COMMIT, tie-broken by the coordinator's node id. It is the same on every node. A change older +than the row's stored version is skipped, so replaying peers' logs in any order ends with the newest image. +A DELETE leaves a tombstone, so an older INSERT or UPDATE cannot bring the row back, and an UPDATE whose row +is absent inserts its after-image. Tombstones are removed once older than `gc_max_retention_hours` times +the member count; with unlimited retention they are kept. DROP TABLE removes a table's versions, RENAME +TABLE carries them, and a change of primary-key columns clears them. LOAD DATA rows are versioned the same +way, each at its transaction's commit timestamp. + +Known limits: a table rebuilt by DDL (create a new table, copy the rows, drop the old one, rename) loses the +old table's versions, so changes to its rows replayed afterwards apply as if the rows were unversioned. Read +replicas apply their source's stream in order without row versions, so a replica that switches to another +source may apply that source's order. + ### Example: Encoded Row for Our UPDATE When Node 1 executes `UPDATE users SET balance = 75 WHERE id = 1`: diff --git a/grpc/client.go b/grpc/client.go index e864be9f..ead0113c 100644 --- a/grpc/client.go +++ b/grpc/client.go @@ -280,6 +280,7 @@ func (c *Client) TransactionStream( statements []*Statement, chunkSize int, timestamp *HLC, + commitTimestamp *HLC, sourceNodeID uint64, ) (*TransactionResponse, error) { conn, err := c.getConn(nodeID) @@ -331,10 +332,11 @@ func (c *Client) TransactionStream( commitMsg := &TransactionStreamMessage{ Payload: &TransactionStreamMessage_Commit{ Commit: &TransactionCommit{ - TxnId: txnID, - Database: database, - Timestamp: timestamp, - SourceNodeId: sourceNodeID, + TxnId: txnID, + Database: database, + Timestamp: timestamp, + CommitTimestamp: commitTimestamp, + SourceNodeId: sourceNodeID, }, }, } diff --git a/grpc/client_test.go b/grpc/client_test.go index 99663899..b37a6057 100644 --- a/grpc/client_test.go +++ b/grpc/client_test.go @@ -108,6 +108,7 @@ func TestClient_TransactionStream_ChunksSentCorrectly(t *testing.T) { statements, 1024, // 1KB chunks timestamp, + nil, 1, ) @@ -176,6 +177,7 @@ func TestClient_TransactionStream_SingleChunk(t *testing.T) { statements, 1024*1024, // 1MB chunks timestamp, + nil, 5, ) @@ -240,6 +242,7 @@ func TestClient_TransactionStream_ExactlyFiftyStatements(t *testing.T) { statements, 1024*1024, // 1MB chunks timestamp, + nil, 10, ) @@ -294,6 +297,7 @@ func TestClient_TransactionStream_EmptyStatements(t *testing.T) { []*Statement{}, 1024*1024, // 1MB chunks timestamp, + nil, 7, ) @@ -319,6 +323,7 @@ func TestClient_TransactionStream_NotConnected(t *testing.T) { []*Statement{}, 1024*1024, // 1MB chunks &HLC{}, + nil, 1, ) diff --git a/grpc/log_puller.go b/grpc/log_puller.go index e9280912..115f1cb9 100644 --- a/grpc/log_puller.go +++ b/grpc/log_puller.go @@ -405,7 +405,7 @@ func (lp *LogPuller) applyPage(ctx context.Context, st *pullState, entries []*Lo st.result.Stuck++ continue } - fetch, err := lp.resolveLocalPending(st, e.TxnId, &covered[i]) + fetch, err := lp.resolveLocalPending(st, e, &covered[i]) if err != nil { return 0, err } @@ -426,8 +426,8 @@ func (lp *LogPuller) applyPage(ctx context.Context, st *pullState, entries []*Lo return prefix, nil } -// resolveLocalPending resolves this node's own record for txnID, which the -// peer's log proves COMMITTED, and reports whether the entry must be fetched +// resolveLocalPending resolves this node's own record for e's transaction, +// which the peer's log proves COMMITTED (at e's commit timestamp), and reports whether the entry must be fetched // and replayed from the peer: // - no PENDING record: fetch; // - durably prepared: commit it through the local commit path @@ -436,7 +436,8 @@ func (lp *LogPuller) applyPage(ctx context.Context, st *pullState, entries []*Lo // - begun but abandoned (its PREPARE died with an earlier process): it // promised nothing, so discard it and fetch; // - begun and live (a PREPARE executing now): leave it for the next call. -func (lp *LogPuller) resolveLocalPending(st *pullState, txnID uint64, covered *bool) (fetch bool, err error) { +func (lp *LogPuller) resolveLocalPending(st *pullState, e *LogEntry, covered *bool) (fetch bool, err error) { + txnID := e.TxnId kind, err := st.metaStore.ClassifyPending(txnID) if err != nil { return false, fmt.Errorf("check local status for txn %d: %w", txnID, err) @@ -448,7 +449,7 @@ func (lp *LogPuller) resolveLocalPending(st *pullState, txnID uint64, covered *b if st.localCommitFailed { return false, nil } - if err := lp.dbMgr.CommitLocallyPrepared(st.database, txnID); err != nil { + if err := lp.dbMgr.CommitLocallyPrepared(st.database, txnID, HLCToTimestamp(e.CommitTimestamp)); err != nil { st.localCommitFailed = true // A busy writer (a pinned session holding it, say) is not this // transaction's fault and must not push it toward STUCK, whose diff --git a/grpc/log_puller_row_version_test.go b/grpc/log_puller_row_version_test.go new file mode 100644 index 00000000..586f85d2 --- /dev/null +++ b/grpc/log_puller_row_version_test.go @@ -0,0 +1,148 @@ +package grpc + +import ( + "context" + "testing" + + "github.com/maxpert/marmot/coordinator" + "github.com/maxpert/marmot/db" + pb "github.com/maxpert/marmot/grpc/common" + "github.com/maxpert/marmot/hlc" + "github.com/maxpert/marmot/protocol/filter" + "github.com/stretchr/testify/require" +) + +// tRow builds a captured change of table t's row id, keyed exactly as the +// CDC hook keys it. +func tRow(t *testing.T, op db.OpType, id int64, oldV, newV string) *db.EncodedCapturedRow { + row := &db.EncodedCapturedRow{ + Table: "t", + Op: uint8(op), + IntentKey: filter.EncodeIntentKey("t", []filter.TypedPKValue{{Type: filter.PKTypeInt64, Value: filter.EncodeInt64(id)}}), + } + if oldV != "" { + row.OldValues = encodeSeedValues(t, map[string]interface{}{"id": id, "v": oldV}) + } + if newV != "" { + row.NewValues = encodeSeedValues(t, map[string]interface{}{"id": id, "v": newV}) + } + return row +} + +// seedRows commits rows as txnID, coordinated by origin at wall, into +// database d, logging it in n's own commit log in call order. +func (n *pullTestNode) seedRows(t *testing.T, d string, txnID, origin uint64, wall int64, rows ...*db.EncodedCapturedRow) { + t.Helper() + mdb, err := n.dm.GetDatabase(d) + require.NoError(t, err) + _, err = mdb.ApplyReplayedTxn(context.Background(), &db.ReplayTxn{ + TxnID: txnID, OriginNodeID: origin, CommitTS: hlc.Timestamp{WallTime: wall, NodeID: origin}, Rows: rows, + }, true) + require.NoError(t, err) +} + +// pullBoth pulls database d from peers a then b into local. +func pullBoth(t *testing.T, local, a, b *pullTestNode, d string) { + t.Helper() + lp := NewLogPuller(LogPullerConfig{NodeID: local.id, Client: NewClient(local.id), DBManager: local.dm}) + for _, p := range []*pullTestNode{a, b} { + res, err := lp.PullPair(context.Background(), PeerRef{NodeID: p.id, Address: p.addr}, d) + require.NoError(t, err) + require.True(t, res.CaughtUp) + } +} + +// Peer 2 logged T2 (row 1 = v2) before T1 (row 1 = v1), as a peer that +// missed T1's COMMIT and pulled it later does; peer 3 logged them in commit +// order. Whichever order the puller meets them in, row 1 ends as v2. +func TestLogPuller_PeersListingSameTxnsInDifferentOrdersConvergeOnNewestImage(t *testing.T) { + late, inOrder := newPullTestNode(t, 2, "app"), newPullTestNode(t, 3, "app") + late.serve(t, nil) + inOrder.serve(t, nil) + const t0, t1, t2 = 100, 200, 300 + for _, p := range []*pullTestNode{late, inOrder} { + p.seedRows(t, "app", t0, 3, 10, tRow(t, db.OpTypeInsert, 1, "", "v0")) + } + late.seedRows(t, "app", t2, 3, 30, tRow(t, db.OpTypeUpdate, 1, "v1", "v2")) + late.seedRows(t, "app", t1, 2, 20, tRow(t, db.OpTypeUpdate, 1, "v0", "v1")) + inOrder.seedRows(t, "app", t1, 2, 20, tRow(t, db.OpTypeUpdate, 1, "v0", "v1")) + inOrder.seedRows(t, "app", t2, 3, 30, tRow(t, db.OpTypeUpdate, 1, "v1", "v2")) + + for _, order := range [][2]*pullTestNode{{late, inOrder}, {inOrder, late}} { + local := newPullTestNode(t, 1, "app") + pullBoth(t, local, order[0], order[1], "app") + require.Equal(t, map[int64]string{1: "v2"}, local.rows(t, "app"), "pulled from node %d first", order[0].id) + } +} + +// A peer that logged a DELETE before the older INSERT it follows must not +// make a puller resurrect the row. +func TestLogPuller_OlderInsertListedAfterDeleteDoesNotResurrectRow(t *testing.T) { + late, inOrder := newPullTestNode(t, 2, "app"), newPullTestNode(t, 3, "app") + late.serve(t, nil) + inOrder.serve(t, nil) + late.seedRows(t, "app", 200, 3, 30, tRow(t, db.OpTypeDelete, 1, "v1", "")) + late.seedRows(t, "app", 100, 2, 20, tRow(t, db.OpTypeInsert, 1, "", "v1")) + inOrder.seedRows(t, "app", 100, 2, 20, tRow(t, db.OpTypeInsert, 1, "", "v1")) + inOrder.seedRows(t, "app", 200, 3, 30, tRow(t, db.OpTypeDelete, 1, "v1", "")) + + for _, order := range [][2]*pullTestNode{{late, inOrder}, {inOrder, late}} { + local := newPullTestNode(t, 1, "app") + pullBoth(t, local, order[0], order[1], "app") + require.Empty(t, local.rows(t, "app"), "pulled from node %d first", order[0].id) + } +} + +// A COMMIT's decided timestamp and a PREPARE answer's clock survive the +// wire in both directions; a request without one sends none. +func TestCommitTimestampAndAppliedAtCrossTheWire(t *testing.T) { + commitTS := hlc.Timestamp{WallTime: 42, Logical: 3, NodeID: 7} + req, err := transactionRequestToProto(&coordinator.ReplicationRequest{TxnID: 1, Phase: coordinator.PhaseCommit, CommitTS: commitTS}) + require.NoError(t, err) + require.Equal(t, commitTS, HLCToTimestamp(req.CommitTimestamp)) + + req, err = transactionRequestToProto(&coordinator.ReplicationRequest{TxnID: 1, Phase: coordinator.PhasePrep}) + require.NoError(t, err) + require.Nil(t, req.CommitTimestamp) + + resp := convertTransactionResponse(&TransactionResponse{Success: true, AppliedAt: &HLC{WallTime: 9, Logical: 1, NodeId: 2}}) + require.Equal(t, hlc.Timestamp{WallTime: 9, Logical: 1, NodeID: 2}, resp.AppliedAt) +} + +// prepareOneRow PREPAREs txnID on rh as coordinator 2's insert of t row id. +func prepareOneRow(t *testing.T, rh *ReplicationHandler, mdb *db.ReplicatedDatabase, txnID uint64, id int64) { + t.Helper() + resp, err := rh.HandleReplicateTransaction(context.Background(), &TransactionRequest{ + TxnId: txnID, SourceNodeId: 2, Database: "app", Phase: TransactionPhase_PREPARE, + Timestamp: &HLC{WallTime: 1, NodeId: 2}, RequiredSchemaVersion: mdb.SchemaVersion(), + Statements: []*Statement{{Type: pb.StatementType_INSERT, TableName: "t", Database: "app", + Payload: &Statement_RowChange{RowChange: testInsertRowChange("t", tRow(t, db.OpTypeInsert, id, "", "x").IntentKey, map[string][]byte{ + "id": mustMarshalMsgpack(t, id), "v": mustMarshalMsgpack(t, "x"), + })}}}, + }) + require.NoError(t, err) + require.True(t, resp.Success, resp.ErrorMessage) +} + +// A participant commits with the coordinator's commit timestamp, so its log +// serves the same one every other node's does. +func TestCommitRPCCommitsWithTheCoordinatorsTimestamp(t *testing.T) { + node := newPullTestNode(t, 1, "app") + mdb, err := node.dm.GetDatabase("app") + require.NoError(t, err) + rh := NewReplicationHandler(1, node.dm, hlc.NewClock(1), db.NewSchemaVersionManager(node.dm)) + prepareOneRow(t, rh, mdb, 4000, 1) + + decided := &HLC{WallTime: 777, Logical: 2, NodeId: 2} + resp, err := rh.HandleReplicateTransaction(context.Background(), &TransactionRequest{ + TxnId: 4000, SourceNodeId: 2, Database: "app", Phase: TransactionPhase_COMMIT, + Timestamp: &HLC{WallTime: 1, NodeId: 2}, CommitTimestamp: decided, + }) + require.NoError(t, err) + require.True(t, resp.Success, resp.ErrorMessage) + + rec, err := mdb.GetMetaStore().GetTransaction(4000) + require.NoError(t, err) + require.Equal(t, decided.WallTime, rec.CommitTSWall) + require.Equal(t, decided.Logical, rec.CommitTSLogical) +} diff --git a/grpc/log_puller_test.go b/grpc/log_puller_test.go index 1838850e..e33a301e 100644 --- a/grpc/log_puller_test.go +++ b/grpc/log_puller_test.go @@ -187,6 +187,7 @@ func TestLogPuller_PreparedLocalTxnCommittedLocallyInSamePull(t *testing.T) { require.NoError(t, err) require.NotNil(t, rec) require.Equal(t, db.TxnStatusCommitted, rec.Status, "the prepared txn must be committed locally") + require.Equal(t, int64(2000), rec.CommitTSWall, "the local commit must take the commit timestamp the peer's log records") cursor, err := mdb.GetMetaStore().GetPullCursor(2) require.NoError(t, err) diff --git a/grpc/log_server.go b/grpc/log_server.go index 6a2d6743..bc739b40 100644 --- a/grpc/log_server.go +++ b/grpc/log_server.go @@ -7,6 +7,7 @@ import ( "github.com/maxpert/marmot/common" "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/hlc" "github.com/rs/zerolog/log" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -85,9 +86,9 @@ func (s *Server) ListCommittedLog(ctx context.Context, req *LogListRequest) (*Lo return nil, status.Errorf(codes.Internal, "read truncation point: %v", err) } - pbEntries := make([]*LogEntry, 0, len(entries)) - for _, e := range entries { - pbEntries = append(pbEntries, &LogEntry{Seq: e.Seq, TxnId: e.TxnID}) + pbEntries, err := logEntriesToProto(metaStore, entries) + if err != nil { + return nil, err } return &LogListResponse{ @@ -100,6 +101,33 @@ func (s *Server) ListCommittedLog(ctx context.Context, req *LogListRequest) (*Lo }, nil } +// logEntriesToProto converts listed log entries to LogEntry messages, each +// carrying the commit timestamp stored with its position, which a puller +// that holds the transaction prepared commits it with. Only an entry +// written before positions carried a timestamp reads it from the +// transaction's record. +func logEntriesToProto(metaStore db.MetaStore, entries []db.CommittedLogEntry) ([]*LogEntry, error) { + pbEntries := make([]*LogEntry, 0, len(entries)) + for _, e := range entries { + commitTS := e.CommitTS + if commitTS.IsZero() { + rec, err := metaStore.GetTransaction(e.TxnID) + if err != nil { + return nil, status.Errorf(codes.Internal, "read transaction %d: %v", e.TxnID, err) + } + if rec != nil { + commitTS = hlc.Timestamp{WallTime: rec.CommitTSWall, Logical: rec.CommitTSLogical, NodeID: rec.NodeID} + } + } + entry := &LogEntry{Seq: e.Seq, TxnId: e.TxnID} + if !commitTS.IsZero() { + entry.CommitTimestamp = &HLC{WallTime: commitTS.WallTime, Logical: commitTS.Logical, NodeId: commitTS.NodeID} + } + pbEntries = append(pbEntries, entry) + } + return pbEntries, nil +} + // FetchTransactions streams the named committed transactions of one // database, in the order requested, then ends (EOF). Unlike StreamChanges' // existing lenient semantics (kept unchanged for read replicas), diff --git a/grpc/marmot.pb.go b/grpc/marmot.pb.go index 7c2ebd1f..60c59704 100644 --- a/grpc/marmot.pb.go +++ b/grpc/marmot.pb.go @@ -710,8 +710,11 @@ type TransactionRequest struct { Database string `protobuf:"bytes,7,opt,name=database,proto3" json:"database,omitempty"` // Minimum schema version required to execute this transaction RequiredSchemaVersion uint64 `protobuf:"varint,9,opt,name=required_schema_version,json=requiredSchemaVersion,proto3" json:"required_schema_version,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // COMMIT only: the commit timestamp the coordinator decided, which every + // participant commits with. Unset from an older coordinator. + CommitTimestamp *HLC `protobuf:"bytes,10,opt,name=commit_timestamp,json=commitTimestamp,proto3" json:"commit_timestamp,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *TransactionRequest) Reset() { @@ -800,6 +803,13 @@ func (x *TransactionRequest) GetRequiredSchemaVersion() uint64 { return 0 } +func (x *TransactionRequest) GetCommitTimestamp() *HLC { + if x != nil { + return x.CommitTimestamp + } + return nil +} + // Statement represents either a DML row change or a DDL schema change // Following industry standard CDC approach (MySQL binlog, TiDB TiCDC, CockroachDB) type Statement struct { @@ -2055,11 +2065,12 @@ func (x *LogListRequest) GetConsumedTxnId() uint64 { } type LogEntry struct { - state protoimpl.MessageState `protogen:"open.v1"` - Seq uint64 `protobuf:"varint,1,opt,name=seq,proto3" json:"seq,omitempty"` - TxnId uint64 `protobuf:"varint,2,opt,name=txn_id,json=txnId,proto3" json:"txn_id,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Seq uint64 `protobuf:"varint,1,opt,name=seq,proto3" json:"seq,omitempty"` + TxnId uint64 `protobuf:"varint,2,opt,name=txn_id,json=txnId,proto3" json:"txn_id,omitempty"` + CommitTimestamp *HLC `protobuf:"bytes,3,opt,name=commit_timestamp,json=commitTimestamp,proto3" json:"commit_timestamp,omitempty"` // The entry's commit timestamp; unset from an older peer + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *LogEntry) Reset() { @@ -2106,6 +2117,13 @@ func (x *LogEntry) GetTxnId() uint64 { return 0 } +func (x *LogEntry) GetCommitTimestamp() *HLC { + if x != nil { + return x.CommitTimestamp + } + return nil +} + type LogListResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Entries []*LogEntry `protobuf:"bytes,1,rep,name=entries,proto3" json:"entries,omitempty"` @@ -3649,13 +3667,14 @@ func (x *TransactionChunk) GetChunkIndex() uint32 { // Final commit signal for streaming type TransactionCommit struct { - state protoimpl.MessageState `protogen:"open.v1"` - TxnId uint64 `protobuf:"varint,1,opt,name=txn_id,json=txnId,proto3" json:"txn_id,omitempty"` - Database string `protobuf:"bytes,2,opt,name=database,proto3" json:"database,omitempty"` - Timestamp *HLC `protobuf:"bytes,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - SourceNodeId uint64 `protobuf:"varint,4,opt,name=source_node_id,json=sourceNodeId,proto3" json:"source_node_id,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + TxnId uint64 `protobuf:"varint,1,opt,name=txn_id,json=txnId,proto3" json:"txn_id,omitempty"` + Database string `protobuf:"bytes,2,opt,name=database,proto3" json:"database,omitempty"` + Timestamp *HLC `protobuf:"bytes,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + SourceNodeId uint64 `protobuf:"varint,4,opt,name=source_node_id,json=sourceNodeId,proto3" json:"source_node_id,omitempty"` + CommitTimestamp *HLC `protobuf:"bytes,5,opt,name=commit_timestamp,json=commitTimestamp,proto3" json:"commit_timestamp,omitempty"` // As TransactionRequest.commit_timestamp; unset from an older coordinator + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *TransactionCommit) Reset() { @@ -3716,6 +3735,13 @@ func (x *TransactionCommit) GetSourceNodeId() uint64 { return 0 } +func (x *TransactionCommit) GetCommitTimestamp() *HLC { + if x != nil { + return x.CommitTimestamp + } + return nil +} + // Wrapper for streaming (oneof for chunk vs commit) type TransactionStreamMessage struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -4268,7 +4294,7 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x0esource_node_id\x18\x01 \x01(\x04R\fsourceNodeId\"V\n" + "\fPingResponse\x12\x17\n" + "\anode_id\x18\x01 \x01(\x04R\x06nodeId\x12-\n" + - "\x06status\x18\x02 \x01(\x0e2\x15.marmot.v2.NodeStatusR\x06status\"\x81\x03\n" + + "\x06status\x18\x02 \x01(\x0e2\x15.marmot.v2.NodeStatusR\x06status\"\xbc\x03\n" + "\x12TransactionRequest\x12\x15\n" + "\x06txn_id\x18\x01 \x01(\x04R\x05txnId\x12$\n" + "\x0esource_node_id\x18\x02 \x01(\x04R\fsourceNodeId\x124\n" + @@ -4279,7 +4305,9 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x05phase\x18\x05 \x01(\x0e2\x1b.marmot.v2.TransactionPhaseR\x05phase\x12=\n" + "\vconsistency\x18\x06 \x01(\x0e2\x1b.marmot.v2.ConsistencyLevelR\vconsistency\x12\x1a\n" + "\bdatabase\x18\a \x01(\tR\bdatabase\x126\n" + - "\x17required_schema_version\x18\t \x01(\x04R\x15requiredSchemaVersionJ\x04\b\b\x10\t\"\xc7\x04\n" + + "\x17required_schema_version\x18\t \x01(\x04R\x15requiredSchemaVersion\x129\n" + + "\x10commit_timestamp\x18\n" + + " \x01(\v2\x0e.marmot.v2.HLCR\x0fcommitTimestampJ\x04\b\b\x10\t\"\xc7\x04\n" + "\tStatement\x120\n" + "\x04type\x18\x01 \x01(\x0e2\x1c.marmot.common.StatementTypeR\x04type\x12\x1d\n" + "\n" + @@ -4398,10 +4426,11 @@ const file_grpc_marmot_proto_rawDesc = "" + "afterTxnId\x12\x14\n" + "\x05limit\x18\x05 \x01(\rR\x05limit\x12!\n" + "\fconsumed_seq\x18\x06 \x01(\x04R\vconsumedSeq\x12&\n" + - "\x0fconsumed_txn_id\x18\a \x01(\x04R\rconsumedTxnId\"3\n" + + "\x0fconsumed_txn_id\x18\a \x01(\x04R\rconsumedTxnId\"n\n" + "\bLogEntry\x12\x10\n" + "\x03seq\x18\x01 \x01(\x04R\x03seq\x12\x15\n" + - "\x06txn_id\x18\x02 \x01(\x04R\x05txnId\"\x92\x02\n" + + "\x06txn_id\x18\x02 \x01(\x04R\x05txnId\x129\n" + + "\x10commit_timestamp\x18\x03 \x01(\v2\x0e.marmot.v2.HLCR\x0fcommitTimestamp\"\x92\x02\n" + "\x0fLogListResponse\x12-\n" + "\aentries\x18\x01 \x03(\v2\x13.marmot.v2.LogEntryR\aentries\x12\x1d\n" + "\n" + @@ -4522,12 +4551,13 @@ const file_grpc_marmot_proto_rawDesc = "" + "statements\x18\x03 \x03(\v2\x14.marmot.v2.StatementR\n" + "statements\x12\x1f\n" + "\vchunk_index\x18\x04 \x01(\rR\n" + - "chunkIndex\"\x9a\x01\n" + + "chunkIndex\"\xd5\x01\n" + "\x11TransactionCommit\x12\x15\n" + "\x06txn_id\x18\x01 \x01(\x04R\x05txnId\x12\x1a\n" + "\bdatabase\x18\x02 \x01(\tR\bdatabase\x12,\n" + "\ttimestamp\x18\x03 \x01(\v2\x0e.marmot.v2.HLCR\ttimestamp\x12$\n" + - "\x0esource_node_id\x18\x04 \x01(\x04R\fsourceNodeId\"\x92\x01\n" + + "\x0esource_node_id\x18\x04 \x01(\x04R\fsourceNodeId\x129\n" + + "\x10commit_timestamp\x18\x05 \x01(\v2\x0e.marmot.v2.HLCR\x0fcommitTimestamp\"\x92\x01\n" + "\x18TransactionStreamMessage\x123\n" + "\x05chunk\x18\x01 \x01(\v2\x1b.marmot.v2.TransactionChunkH\x00R\x05chunk\x126\n" + "\x06commit\x18\x02 \x01(\v2\x1c.marmot.v2.TransactionCommitH\x00R\x06commitB\t\n" + @@ -4720,82 +4750,85 @@ var file_grpc_marmot_proto_depIdxs = []int32{ 19, // 7: marmot.v2.TransactionRequest.timestamp:type_name -> marmot.v2.HLC 1, // 8: marmot.v2.TransactionRequest.phase:type_name -> marmot.v2.TransactionPhase 2, // 9: marmot.v2.TransactionRequest.consistency:type_name -> marmot.v2.ConsistencyLevel - 63, // 10: marmot.v2.Statement.type:type_name -> marmot.common.StatementType - 15, // 11: marmot.v2.Statement.row_change:type_name -> marmot.v2.RowChange - 17, // 12: marmot.v2.Statement.ddl_change:type_name -> marmot.v2.DDLChange - 18, // 13: marmot.v2.Statement.load_data_change:type_name -> marmot.v2.LoadDataChange - 14, // 14: marmot.v2.Statement.vector_index_change:type_name -> marmot.v2.VectorIndexChange - 16, // 15: marmot.v2.Statement.dml_intent:type_name -> marmot.v2.DMLIntent - 3, // 16: marmot.v2.VectorIndexChange.action:type_name -> marmot.v2.VectorIndexAction - 19, // 17: marmot.v2.TransactionResponse.applied_at:type_name -> marmot.v2.HLC - 19, // 18: marmot.v2.ReadRequest.snapshot_ts:type_name -> marmot.v2.HLC - 2, // 19: marmot.v2.ReadRequest.consistency:type_name -> marmot.v2.ConsistencyLevel - 23, // 20: marmot.v2.ReadResponse.rows:type_name -> marmot.v2.Row - 19, // 21: marmot.v2.ReadResponse.timestamp:type_name -> marmot.v2.HLC - 61, // 22: marmot.v2.Row.columns:type_name -> marmot.v2.Row.ColumnsEntry - 13, // 23: marmot.v2.ChangeEvent.statements:type_name -> marmot.v2.Statement - 19, // 24: marmot.v2.ChangeEvent.timestamp:type_name -> marmot.v2.HLC - 27, // 25: marmot.v2.LogListResponse.entries:type_name -> marmot.v2.LogEntry - 31, // 26: marmot.v2.DatabaseRegistryResponse.entries:type_name -> marmot.v2.DatabaseRegistryEntry - 35, // 27: marmot.v2.ReplicationStateResponse.states:type_name -> marmot.v2.DatabaseReplicationState - 19, // 28: marmot.v2.DatabaseReplicationState.last_applied_timestamp:type_name -> marmot.v2.HLC - 19, // 29: marmot.v2.SnapshotInfoResponse.timestamp:type_name -> marmot.v2.HLC - 38, // 30: marmot.v2.SnapshotInfoResponse.databases:type_name -> marmot.v2.DatabaseFileInfo - 39, // 31: marmot.v2.SnapshotInfoResponse.database_metadata:type_name -> marmot.v2.DatabaseSnapshotMetadata - 62, // 32: marmot.v2.LatestTxnIDsResponse.database_txn_ids:type_name -> marmot.v2.LatestTxnIDsResponse.DatabaseTxnIdsEntry - 43, // 33: marmot.v2.LatestTxnIDsResponse.database_info:type_name -> marmot.v2.DatabaseInfo - 46, // 34: marmot.v2.AutoIncBasesResponse.bases:type_name -> marmot.v2.AutoIncBase - 7, // 35: marmot.v2.GetClusterNodesResponse.nodes:type_name -> marmot.v2.NodeState - 13, // 36: marmot.v2.TransactionChunk.statements:type_name -> marmot.v2.Statement - 19, // 37: marmot.v2.TransactionCommit.timestamp:type_name -> marmot.v2.HLC - 52, // 38: marmot.v2.TransactionStreamMessage.chunk:type_name -> marmot.v2.TransactionChunk - 53, // 39: marmot.v2.TransactionStreamMessage.commit:type_name -> marmot.v2.TransactionCommit - 4, // 40: marmot.v2.ForwardQueryRequest.txn_control:type_name -> marmot.v2.ForwardTxnControl - 5, // 41: marmot.v2.MarmotService.Gossip:input_type -> marmot.v2.GossipRequest - 8, // 42: marmot.v2.MarmotService.Join:input_type -> marmot.v2.JoinRequest - 10, // 43: marmot.v2.MarmotService.Ping:input_type -> marmot.v2.PingRequest - 12, // 44: marmot.v2.MarmotService.ReplicateTransaction:input_type -> marmot.v2.TransactionRequest - 21, // 45: marmot.v2.MarmotService.Read:input_type -> marmot.v2.ReadRequest - 24, // 46: marmot.v2.MarmotService.StreamChanges:input_type -> marmot.v2.StreamRequest - 33, // 47: marmot.v2.MarmotService.GetReplicationState:input_type -> marmot.v2.ReplicationStateRequest - 36, // 48: marmot.v2.MarmotService.GetSnapshotInfo:input_type -> marmot.v2.SnapshotInfoRequest - 40, // 49: marmot.v2.MarmotService.StreamSnapshot:input_type -> marmot.v2.SnapshotRequest - 42, // 50: marmot.v2.MarmotService.GetLatestTxnIDs:input_type -> marmot.v2.LatestTxnIDsRequest - 50, // 51: marmot.v2.MarmotService.GetClusterNodes:input_type -> marmot.v2.GetClusterNodesRequest - 55, // 52: marmot.v2.MarmotService.ForwardQuery:input_type -> marmot.v2.ForwardQueryRequest - 57, // 53: marmot.v2.MarmotService.ForwardLoadData:input_type -> marmot.v2.ForwardLoadDataRequest - 58, // 54: marmot.v2.MarmotService.GetLoadDataChunk:input_type -> marmot.v2.LoadDataChunkRequest - 45, // 55: marmot.v2.MarmotService.GetAutoIncBases:input_type -> marmot.v2.AutoIncBasesRequest - 48, // 56: marmot.v2.MarmotService.SyncAutoIncBases:input_type -> marmot.v2.AutoIncSyncRequest - 26, // 57: marmot.v2.MarmotService.ListCommittedLog:input_type -> marmot.v2.LogListRequest - 29, // 58: marmot.v2.MarmotService.FetchTransactions:input_type -> marmot.v2.FetchTransactionsRequest - 30, // 59: marmot.v2.MarmotService.ListDatabaseRegistry:input_type -> marmot.v2.DatabaseRegistryRequest - 54, // 60: marmot.v2.MarmotService.TransactionStream:input_type -> marmot.v2.TransactionStreamMessage - 6, // 61: marmot.v2.MarmotService.Gossip:output_type -> marmot.v2.GossipResponse - 9, // 62: marmot.v2.MarmotService.Join:output_type -> marmot.v2.JoinResponse - 11, // 63: marmot.v2.MarmotService.Ping:output_type -> marmot.v2.PingResponse - 20, // 64: marmot.v2.MarmotService.ReplicateTransaction:output_type -> marmot.v2.TransactionResponse - 22, // 65: marmot.v2.MarmotService.Read:output_type -> marmot.v2.ReadResponse - 25, // 66: marmot.v2.MarmotService.StreamChanges:output_type -> marmot.v2.ChangeEvent - 34, // 67: marmot.v2.MarmotService.GetReplicationState:output_type -> marmot.v2.ReplicationStateResponse - 37, // 68: marmot.v2.MarmotService.GetSnapshotInfo:output_type -> marmot.v2.SnapshotInfoResponse - 41, // 69: marmot.v2.MarmotService.StreamSnapshot:output_type -> marmot.v2.SnapshotChunk - 44, // 70: marmot.v2.MarmotService.GetLatestTxnIDs:output_type -> marmot.v2.LatestTxnIDsResponse - 51, // 71: marmot.v2.MarmotService.GetClusterNodes:output_type -> marmot.v2.GetClusterNodesResponse - 56, // 72: marmot.v2.MarmotService.ForwardQuery:output_type -> marmot.v2.ForwardQueryResponse - 56, // 73: marmot.v2.MarmotService.ForwardLoadData:output_type -> marmot.v2.ForwardQueryResponse - 59, // 74: marmot.v2.MarmotService.GetLoadDataChunk:output_type -> marmot.v2.LoadDataChunkResponse - 47, // 75: marmot.v2.MarmotService.GetAutoIncBases:output_type -> marmot.v2.AutoIncBasesResponse - 49, // 76: marmot.v2.MarmotService.SyncAutoIncBases:output_type -> marmot.v2.AutoIncSyncResponse - 28, // 77: marmot.v2.MarmotService.ListCommittedLog:output_type -> marmot.v2.LogListResponse - 25, // 78: marmot.v2.MarmotService.FetchTransactions:output_type -> marmot.v2.ChangeEvent - 32, // 79: marmot.v2.MarmotService.ListDatabaseRegistry:output_type -> marmot.v2.DatabaseRegistryResponse - 20, // 80: marmot.v2.MarmotService.TransactionStream:output_type -> marmot.v2.TransactionResponse - 61, // [61:81] is the sub-list for method output_type - 41, // [41:61] is the sub-list for method input_type - 41, // [41:41] is the sub-list for extension type_name - 41, // [41:41] is the sub-list for extension extendee - 0, // [0:41] is the sub-list for field type_name + 19, // 10: marmot.v2.TransactionRequest.commit_timestamp:type_name -> marmot.v2.HLC + 63, // 11: marmot.v2.Statement.type:type_name -> marmot.common.StatementType + 15, // 12: marmot.v2.Statement.row_change:type_name -> marmot.v2.RowChange + 17, // 13: marmot.v2.Statement.ddl_change:type_name -> marmot.v2.DDLChange + 18, // 14: marmot.v2.Statement.load_data_change:type_name -> marmot.v2.LoadDataChange + 14, // 15: marmot.v2.Statement.vector_index_change:type_name -> marmot.v2.VectorIndexChange + 16, // 16: marmot.v2.Statement.dml_intent:type_name -> marmot.v2.DMLIntent + 3, // 17: marmot.v2.VectorIndexChange.action:type_name -> marmot.v2.VectorIndexAction + 19, // 18: marmot.v2.TransactionResponse.applied_at:type_name -> marmot.v2.HLC + 19, // 19: marmot.v2.ReadRequest.snapshot_ts:type_name -> marmot.v2.HLC + 2, // 20: marmot.v2.ReadRequest.consistency:type_name -> marmot.v2.ConsistencyLevel + 23, // 21: marmot.v2.ReadResponse.rows:type_name -> marmot.v2.Row + 19, // 22: marmot.v2.ReadResponse.timestamp:type_name -> marmot.v2.HLC + 61, // 23: marmot.v2.Row.columns:type_name -> marmot.v2.Row.ColumnsEntry + 13, // 24: marmot.v2.ChangeEvent.statements:type_name -> marmot.v2.Statement + 19, // 25: marmot.v2.ChangeEvent.timestamp:type_name -> marmot.v2.HLC + 19, // 26: marmot.v2.LogEntry.commit_timestamp:type_name -> marmot.v2.HLC + 27, // 27: marmot.v2.LogListResponse.entries:type_name -> marmot.v2.LogEntry + 31, // 28: marmot.v2.DatabaseRegistryResponse.entries:type_name -> marmot.v2.DatabaseRegistryEntry + 35, // 29: marmot.v2.ReplicationStateResponse.states:type_name -> marmot.v2.DatabaseReplicationState + 19, // 30: marmot.v2.DatabaseReplicationState.last_applied_timestamp:type_name -> marmot.v2.HLC + 19, // 31: marmot.v2.SnapshotInfoResponse.timestamp:type_name -> marmot.v2.HLC + 38, // 32: marmot.v2.SnapshotInfoResponse.databases:type_name -> marmot.v2.DatabaseFileInfo + 39, // 33: marmot.v2.SnapshotInfoResponse.database_metadata:type_name -> marmot.v2.DatabaseSnapshotMetadata + 62, // 34: marmot.v2.LatestTxnIDsResponse.database_txn_ids:type_name -> marmot.v2.LatestTxnIDsResponse.DatabaseTxnIdsEntry + 43, // 35: marmot.v2.LatestTxnIDsResponse.database_info:type_name -> marmot.v2.DatabaseInfo + 46, // 36: marmot.v2.AutoIncBasesResponse.bases:type_name -> marmot.v2.AutoIncBase + 7, // 37: marmot.v2.GetClusterNodesResponse.nodes:type_name -> marmot.v2.NodeState + 13, // 38: marmot.v2.TransactionChunk.statements:type_name -> marmot.v2.Statement + 19, // 39: marmot.v2.TransactionCommit.timestamp:type_name -> marmot.v2.HLC + 19, // 40: marmot.v2.TransactionCommit.commit_timestamp:type_name -> marmot.v2.HLC + 52, // 41: marmot.v2.TransactionStreamMessage.chunk:type_name -> marmot.v2.TransactionChunk + 53, // 42: marmot.v2.TransactionStreamMessage.commit:type_name -> marmot.v2.TransactionCommit + 4, // 43: marmot.v2.ForwardQueryRequest.txn_control:type_name -> marmot.v2.ForwardTxnControl + 5, // 44: marmot.v2.MarmotService.Gossip:input_type -> marmot.v2.GossipRequest + 8, // 45: marmot.v2.MarmotService.Join:input_type -> marmot.v2.JoinRequest + 10, // 46: marmot.v2.MarmotService.Ping:input_type -> marmot.v2.PingRequest + 12, // 47: marmot.v2.MarmotService.ReplicateTransaction:input_type -> marmot.v2.TransactionRequest + 21, // 48: marmot.v2.MarmotService.Read:input_type -> marmot.v2.ReadRequest + 24, // 49: marmot.v2.MarmotService.StreamChanges:input_type -> marmot.v2.StreamRequest + 33, // 50: marmot.v2.MarmotService.GetReplicationState:input_type -> marmot.v2.ReplicationStateRequest + 36, // 51: marmot.v2.MarmotService.GetSnapshotInfo:input_type -> marmot.v2.SnapshotInfoRequest + 40, // 52: marmot.v2.MarmotService.StreamSnapshot:input_type -> marmot.v2.SnapshotRequest + 42, // 53: marmot.v2.MarmotService.GetLatestTxnIDs:input_type -> marmot.v2.LatestTxnIDsRequest + 50, // 54: marmot.v2.MarmotService.GetClusterNodes:input_type -> marmot.v2.GetClusterNodesRequest + 55, // 55: marmot.v2.MarmotService.ForwardQuery:input_type -> marmot.v2.ForwardQueryRequest + 57, // 56: marmot.v2.MarmotService.ForwardLoadData:input_type -> marmot.v2.ForwardLoadDataRequest + 58, // 57: marmot.v2.MarmotService.GetLoadDataChunk:input_type -> marmot.v2.LoadDataChunkRequest + 45, // 58: marmot.v2.MarmotService.GetAutoIncBases:input_type -> marmot.v2.AutoIncBasesRequest + 48, // 59: marmot.v2.MarmotService.SyncAutoIncBases:input_type -> marmot.v2.AutoIncSyncRequest + 26, // 60: marmot.v2.MarmotService.ListCommittedLog:input_type -> marmot.v2.LogListRequest + 29, // 61: marmot.v2.MarmotService.FetchTransactions:input_type -> marmot.v2.FetchTransactionsRequest + 30, // 62: marmot.v2.MarmotService.ListDatabaseRegistry:input_type -> marmot.v2.DatabaseRegistryRequest + 54, // 63: marmot.v2.MarmotService.TransactionStream:input_type -> marmot.v2.TransactionStreamMessage + 6, // 64: marmot.v2.MarmotService.Gossip:output_type -> marmot.v2.GossipResponse + 9, // 65: marmot.v2.MarmotService.Join:output_type -> marmot.v2.JoinResponse + 11, // 66: marmot.v2.MarmotService.Ping:output_type -> marmot.v2.PingResponse + 20, // 67: marmot.v2.MarmotService.ReplicateTransaction:output_type -> marmot.v2.TransactionResponse + 22, // 68: marmot.v2.MarmotService.Read:output_type -> marmot.v2.ReadResponse + 25, // 69: marmot.v2.MarmotService.StreamChanges:output_type -> marmot.v2.ChangeEvent + 34, // 70: marmot.v2.MarmotService.GetReplicationState:output_type -> marmot.v2.ReplicationStateResponse + 37, // 71: marmot.v2.MarmotService.GetSnapshotInfo:output_type -> marmot.v2.SnapshotInfoResponse + 41, // 72: marmot.v2.MarmotService.StreamSnapshot:output_type -> marmot.v2.SnapshotChunk + 44, // 73: marmot.v2.MarmotService.GetLatestTxnIDs:output_type -> marmot.v2.LatestTxnIDsResponse + 51, // 74: marmot.v2.MarmotService.GetClusterNodes:output_type -> marmot.v2.GetClusterNodesResponse + 56, // 75: marmot.v2.MarmotService.ForwardQuery:output_type -> marmot.v2.ForwardQueryResponse + 56, // 76: marmot.v2.MarmotService.ForwardLoadData:output_type -> marmot.v2.ForwardQueryResponse + 59, // 77: marmot.v2.MarmotService.GetLoadDataChunk:output_type -> marmot.v2.LoadDataChunkResponse + 47, // 78: marmot.v2.MarmotService.GetAutoIncBases:output_type -> marmot.v2.AutoIncBasesResponse + 49, // 79: marmot.v2.MarmotService.SyncAutoIncBases:output_type -> marmot.v2.AutoIncSyncResponse + 28, // 80: marmot.v2.MarmotService.ListCommittedLog:output_type -> marmot.v2.LogListResponse + 25, // 81: marmot.v2.MarmotService.FetchTransactions:output_type -> marmot.v2.ChangeEvent + 32, // 82: marmot.v2.MarmotService.ListDatabaseRegistry:output_type -> marmot.v2.DatabaseRegistryResponse + 20, // 83: marmot.v2.MarmotService.TransactionStream:output_type -> marmot.v2.TransactionResponse + 64, // [64:84] is the sub-list for method output_type + 44, // [44:64] is the sub-list for method input_type + 44, // [44:44] is the sub-list for extension type_name + 44, // [44:44] is the sub-list for extension extendee + 0, // [0:44] is the sub-list for field type_name } func init() { file_grpc_marmot_proto_init() } diff --git a/grpc/marmot.proto b/grpc/marmot.proto index 396b9d47..21c62e9c 100644 --- a/grpc/marmot.proto +++ b/grpc/marmot.proto @@ -161,6 +161,9 @@ message TransactionRequest { reserved 8; // Minimum schema version required to execute this transaction uint64 required_schema_version = 9; + // COMMIT only: the commit timestamp the coordinator decided, which every + // participant commits with. Unset from an older coordinator. + HLC commit_timestamp = 10; } enum TransactionPhase { @@ -369,6 +372,7 @@ message LogListRequest { message LogEntry { uint64 seq = 1; uint64 txn_id = 2; + HLC commit_timestamp = 3; // The entry's commit timestamp; unset from an older peer } message LogListResponse { @@ -553,6 +557,7 @@ message TransactionCommit { string database = 2; HLC timestamp = 3; uint64 source_node_id = 4; + HLC commit_timestamp = 5; // As TransactionRequest.commit_timestamp; unset from an older coordinator } // Wrapper for streaming (oneof for chunk vs commit) diff --git a/grpc/node_registry.go b/grpc/node_registry.go index 7ba47dd3..12ed4d89 100644 --- a/grpc/node_registry.go +++ b/grpc/node_registry.go @@ -419,6 +419,21 @@ func (nr *NodeRegistry) GetAll() []*NodeState { return nodes } +// KnownMemberCount returns how many members this node has ever known and +// still records, in any status, REMOVED included: every member whose log +// could still hold an entry. The registry restores the persisted +// membership at boot, so the count survives a restart. A node configured +// with seeds (seeded) that knows only itself has not learned its +// membership yet and returns 0: unknown. +func (nr *NodeRegistry) KnownMemberCount(seeded bool) int { + nr.mu.RLock() + defer nr.mu.RUnlock() + if seeded && len(nr.nodes) < 2 { + return 0 + } + return len(nr.nodes) +} + // GetAlive returns all alive nodes (returns copies to avoid race conditions) func (nr *NodeRegistry) GetAlive() []*NodeState { nr.mu.RLock() diff --git a/grpc/node_registry_test.go b/grpc/node_registry_test.go index 13ac2f74..5b277924 100644 --- a/grpc/node_registry_test.go +++ b/grpc/node_registry_test.go @@ -1189,3 +1189,36 @@ func TestNodeRegistry_DiscoveringAnAliveNodeConnectsToIt(t *testing.T) { t.Fatalf("a peer that turned ALIVE was never connected: %v", connected) } } + +// A REMOVED member still counts toward the tombstone horizon, and the count +// survives a restart through the persisted membership. +func TestNodeRegistry_KnownMemberCountKeepsRemovedMembersAcrossRestart(t *testing.T) { + dir := t.TempDir() + nr := NewNodeRegistryWithDataDir(1, "localhost:8081", dir) + nr.Add(&NodeState{NodeId: 2, Status: NodeStatus_ALIVE}) + nr.Add(&NodeState{NodeId: 3, Status: NodeStatus_ALIVE}) + if err := nr.MarkRemoved(3); err != nil { + t.Fatalf("MarkRemoved: %v", err) + } + if got := nr.KnownMemberCount(true); got != 3 { + t.Fatalf("KnownMemberCount = %d, want 3 (a REMOVED member counts)", got) + } + + restarted := NewNodeRegistryWithDataDir(1, "localhost:8081", dir) + if got := restarted.KnownMemberCount(true); got != 3 { + t.Fatalf("KnownMemberCount after restart = %d, want 3", got) + } +} + +// A seeded node that knows only itself has not learned its membership: the +// count is unknown (0) until it does. An unseeded, standalone node is a +// membership of one. +func TestNodeRegistry_KnownMemberCountUnknownUntilASeededNodeLearnsPeers(t *testing.T) { + nr := NewNodeRegistry(1, "localhost:8081") + if got := nr.KnownMemberCount(true); got != 0 { + t.Fatalf("seeded, alone: KnownMemberCount = %d, want 0", got) + } + if got := nr.KnownMemberCount(false); got != 1 { + t.Fatalf("standalone: KnownMemberCount = %d, want 1", got) + } +} diff --git a/grpc/replication_handler.go b/grpc/replication_handler.go index b5bf987c..e8fadad1 100644 --- a/grpc/replication_handler.go +++ b/grpc/replication_handler.go @@ -236,6 +236,7 @@ func (rh *ReplicationHandler) handleCommit(ctx context.Context, req *Transaction TxnID: req.TxnId, Database: req.Database, Statements: statements, + CommitTS: HLCToTimestamp(req.CommitTimestamp), } result := rh.engine.Commit(ctx, engineReq) diff --git a/grpc/replicator_adapter.go b/grpc/replicator_adapter.go index d3bee2af..5e4a51b1 100644 --- a/grpc/replicator_adapter.go +++ b/grpc/replicator_adapter.go @@ -54,7 +54,7 @@ func transactionRequestToProto(req *coordinator.ReplicationRequest) (*Transactio if err != nil { return nil, err } - return &TransactionRequest{ + protoReq := &TransactionRequest{ TxnId: req.TxnID, SourceNodeId: req.NodeID, Statements: statements, @@ -66,7 +66,11 @@ func transactionRequestToProto(req *coordinator.ReplicationRequest) (*Transactio Phase: convertPhaseToProto(req.Phase), Database: req.Database, RequiredSchemaVersion: req.RequiredSchemaVersion, - }, nil + } + if !req.CommitTS.IsZero() { + protoReq.CommitTimestamp = convertTimestampToHLC(req.CommitTS) + } + return protoReq, nil } // convertTransactionResponse converts a gRPC TransactionResponse into @@ -78,6 +82,7 @@ func convertTransactionResponse(resp *TransactionResponse) *coordinator.Replicat return &coordinator.ReplicationResponse{ Success: resp.Success, Error: resp.ErrorMessage, + AppliedAt: HLCToTimestamp(resp.AppliedAt), ConflictDetected: resp.ConflictDetected, ConflictDetails: resp.ConflictDetails, Rejected: resp.Rejected, @@ -244,7 +249,11 @@ func (gr *GRPCReplicator) StreamReplicateTransaction(ctx context.Context, nodeID // Call streaming client with configured chunk size chunkSize := coordinator.GetStreamChunkSize() - grpcResp, err := gr.client.TransactionStream(ctx, nodeID, req.TxnID, req.Database, statements, chunkSize, timestamp, req.NodeID) + var commitTimestamp *HLC + if !req.CommitTS.IsZero() { + commitTimestamp = convertTimestampToHLC(req.CommitTS) + } + grpcResp, err := gr.client.TransactionStream(ctx, nodeID, req.TxnID, req.Database, statements, chunkSize, timestamp, commitTimestamp, req.NodeID) if err != nil { return nil, fmt.Errorf("streaming gRPC call failed: %w", err) } diff --git a/grpc/replicator_adapter_test.go b/grpc/replicator_adapter_test.go index 853e8672..b1181a82 100644 --- a/grpc/replicator_adapter_test.go +++ b/grpc/replicator_adapter_test.go @@ -106,3 +106,47 @@ func TestGRPCReplicator_PrepareRejectionCarriesOverRealGRPCConnection(t *testing t.Errorf("MySQL error code: got %d, want %d (ErrCodeDupFieldName)", mysqlErr.Code, protocol.ErrCodeDupFieldName) } } + +// commitCapturingStreamServer records the TransactionCommit a streamed +// transaction ends with. +type commitCapturingStreamServer struct { + UnimplementedMarmotServiceServer + commit chan *TransactionCommit +} + +func (s *commitCapturingStreamServer) TransactionStream(stream grpc.ClientStreamingServer[TransactionStreamMessage, TransactionResponse]) error { + for { + msg, err := stream.Recv() + if err != nil { + return err + } + if commit := msg.GetCommit(); commit != nil { + s.commit <- commit + return stream.SendAndClose(&TransactionResponse{Success: true}) + } + } +} + +// A streamed COMMIT carries the decided commit timestamp, as a unary one does. +func TestGRPCReplicator_StreamedCommitCarriesCommitTimestamp(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + server := &commitCapturingStreamServer{commit: make(chan *TransactionCommit, 1)} + grpcServer := grpc.NewServer() + RegisterMarmotServiceServer(grpcServer, server) + go func() { _ = grpcServer.Serve(listener) }() + defer grpcServer.Stop() + + client := NewClient(1) + require.NoError(t, client.Connect(2, listener.Addr().String())) + defer client.Close() + + decided := hlc.Timestamp{WallTime: 555, Logical: 6, NodeID: 1} + resp, err := NewGRPCReplicator(client).StreamReplicateTransaction(context.Background(), 2, &coordinator.ReplicationRequest{ + TxnID: 77, NodeID: 1, Database: "app", Phase: coordinator.PhaseCommit, + StartTS: hlc.Timestamp{WallTime: 1, NodeID: 1}, CommitTS: decided, + }) + require.NoError(t, err) + require.True(t, resp.Success) + require.Equal(t, decided, HLCToTimestamp((<-server.commit).CommitTimestamp)) +} diff --git a/grpc/server.go b/grpc/server.go index 83cc83d7..d6d34f81 100644 --- a/grpc/server.go +++ b/grpc/server.go @@ -1544,7 +1544,7 @@ func (s *Server) TransactionStream(stream grpc.ClientStreamingServer[Transaction } // Commit the transaction - if err := txnMgr.CommitTransaction(txn); err != nil { + if err := txnMgr.CommitTransactionAfter(txn, HLCToTimestamp(commit.CommitTimestamp), nil); err != nil { log.Error().Err(err). Uint64("txn_id", txnID). Str("database", database). diff --git a/grpc/transaction_stream_handler_test.go b/grpc/transaction_stream_handler_test.go index 5db409fe..8c95eb73 100644 --- a/grpc/transaction_stream_handler_test.go +++ b/grpc/transaction_stream_handler_test.go @@ -173,6 +173,7 @@ func TestTransactionStream_BasicFlow(t *testing.T) { Logical: 0, NodeId: 2, }, + CommitTimestamp: &HLC{WallTime: startTS.WallTime + 2000, Logical: 5, NodeId: 2}, }, }, }, @@ -200,6 +201,13 @@ func TestTransactionStream_BasicFlow(t *testing.T) { require.NoError(t, err) assert.Equal(t, "Alice", name) assert.Equal(t, "alice@example.com", email) + + // The participant committed with the coordinator's decided timestamp. + rec, err := dbInstance.GetMetaStore().GetTransaction(txnID) + require.NoError(t, err) + require.NotNil(t, rec) + assert.Equal(t, startTS.WallTime+2000, rec.CommitTSWall) + assert.Equal(t, int32(5), rec.CommitTSLogical) } func TestTransactionStream_CommitNotFound(t *testing.T) { diff --git a/hlc/clock.go b/hlc/clock.go index 81be5d32..ff4c420a 100644 --- a/hlc/clock.go +++ b/hlc/clock.go @@ -191,6 +191,11 @@ func After(a, b Timestamp) bool { return Compare(a, b) > 0 } +// IsZero reports whether t is the zero Timestamp, which no clock issues. +func (t Timestamp) IsZero() bool { + return t == Timestamp{} +} + // PhysicalTime returns the physical time component as time.Time func (t Timestamp) PhysicalTime() time.Time { return time.Unix(0, t.WallTime) diff --git a/marmot.go b/marmot.go index f1c7266d..8b38683d 100644 --- a/marmot.go +++ b/marmot.go @@ -411,6 +411,11 @@ func main() { return ids }) + // The tombstone horizon counts every member this node has known + // (NodeRegistry.KnownMemberCount). + seeded := len(cfg.Config.Cluster.SeedNodes) > 0 + dbMgr.SetTombstoneMemberCountFunc(func() int { return registry.KnownMemberCount(seeded) }) + // Start anti-entropy service antiEntropy.Start() diff --git a/protocol/load_data_local.go b/protocol/load_data_local.go index 7892be9d..b1dff86d 100644 --- a/protocol/load_data_local.go +++ b/protocol/load_data_local.go @@ -355,3 +355,30 @@ func executeLocalLoadDataRowsWithHandler(session *ConnectionSession, handler Con } return inserted, lastInsertID, committedTxnID, nil } + +// LoadDataRows is what a LOAD DATA LOCAL statement inserts: its target +// table (unquoted), its column list (empty for every column in table +// order), and the payload's rows, split exactly as ExecuteLoadDataLocal +// splits them. +type LoadDataRows struct { + Table string + Columns []string + Rows [][]string +} + +// ParseLoadDataRows parses query and splits data into the rows it inserts. +// A row whose field count differs from the column list is an error, as in +// ExecuteLoadDataLocal. +func ParseLoadDataRows(query string, data []byte) (*LoadDataRows, error) { + spec, err := parseLoadDataLocalSpec(query) + if err != nil { + return nil, err + } + rows := splitLoadDataRows(data, spec) + for _, row := range rows { + if len(spec.Columns) > 0 && len(row) != len(spec.Columns) { + return nil, fmt.Errorf("LOAD DATA LOCAL row column mismatch: got %d, expected %d", len(row), len(spec.Columns)) + } + } + return &LoadDataRows{Table: spec.Table, Columns: spec.Columns, Rows: rows}, nil +} diff --git a/replica/stream_client.go b/replica/stream_client.go index ab00e3ca..6e8edb65 100644 --- a/replica/stream_client.go +++ b/replica/stream_client.go @@ -1050,7 +1050,9 @@ func (s *StreamClient) applyStatement(ctx context.Context, tx *sql.Tx, mdb *db.R } // applyCDCStatement applies a CDC statement using unified CDC applier. -// Uses cached schema from mdb - does NOT query SQLite PRAGMA. +// Uses cached schema from mdb - does NOT query SQLite PRAGMA. It applies +// rows unversioned, in the order its one source streams them; a replica +// that switches to another source applies that source's order. func (s *StreamClient) applyCDCStatement(tx *sql.Tx, mdb *db.ReplicatedDatabase, stmt *marmotgrpc.Statement) error { rowChange := stmt.GetRowChange() if rowChange == nil { diff --git a/test/node_outage_convergence_cluster_test.go b/test/node_outage_convergence_cluster_test.go index c9a1a022..a334e9e4 100644 --- a/test/node_outage_convergence_cluster_test.go +++ b/test/node_outage_convergence_cluster_test.go @@ -6,11 +6,9 @@ package test // catch-up, convergence with GC actually running, and transactions a killed // node had prepared but never committed. // -// These check row presence (missing=0) and duplicate ids only, never row -// values. A value mismatch across nodes - an older row image overwriting a -// newer one - is a separate, known limitation of replaying a peer's log over -// rows a node already changed; comparing values here would conflate it with -// the presence guarantees these tests exist to check. +// These check row presence (missing=0) and duplicate ids only. Row values +// converging across nodes is checked separately +// (row_version_convergence_cluster_test.go). // // A transaction a node holds locally PENDING (its own coordinator killed // before its local commit, or a participant COMMIT that failed against a diff --git a/test/row_version_convergence_cluster_test.go b/test/row_version_convergence_cluster_test.go new file mode 100644 index 00000000..b0ef6b7a --- /dev/null +++ b/test/row_version_convergence_cluster_test.go @@ -0,0 +1,146 @@ +package test + +// Cluster test for row-value convergence: nodes 1 and 2 keep overwriting the +// same rows while node 3 is down and for a moment after it returns, then +// node 3 writes each row once more. Every node must end with the same value +// for every row. The order in which a node meets older and newer images of +// a row is pinned deterministically by grpc's log puller tests; this checks +// the end-to-end outcome. + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "sync" + "testing" + "time" +) + +const ( + // hotRows is how many rows the writers keep overwriting. + hotRows = 10 + // hotOutage is how long node 3 stays down under load. + hotOutage = 10 * time.Second + // hotAfterRestart is how long the load continues once node 3 is back. + hotAfterRestart = 2 * time.Second +) + +// tableChecksum hashes every (id, v) of table on node, in id order. +func tableChecksum(t *testing.T, h *ClusterHarness, node int, database, table string) (string, error) { + a, err := queryAnswer(openTimedNodeDatabase(t, h, node, database), "SELECT id, v FROM "+table+" ORDER BY id") + if err != nil { + return "", fmt.Errorf("node %d: %v", node, err) + } + sum := sha256.New() + for _, r := range a.rows { + fmt.Fprintf(sum, "%s=%s\n", r[0], r[1]) + } + return hex.EncodeToString(sum.Sum(nil))[:16], nil +} + +// waitChecksumsEqual polls until table's checksum is identical on every +// node, within deadline. +func waitChecksumsEqual(t *testing.T, h *ClusterHarness, database, table string, deadline time.Duration) { + t.Helper() + start := time.Now() + for { + h.Progress() + sums := make([]string, 0, numNodes) + for node := 1; node <= numNodes; node++ { + sum, err := tableChecksum(t, h, node, database, table) + if err != nil { + sum = err.Error() + } + sums = append(sums, sum) + } + equal := true + for _, s := range sums[1:] { + equal = equal && s == sums[0] + } + if equal { + t.Logf("checksums equal on every node %s after load stopped", time.Since(start).Round(time.Millisecond)) + return + } + if time.Since(start) > deadline { + h.dumpNodeLogs("cluster_" + strings.ReplaceAll(t.Name(), "/", "_")) + t.Fatalf("%s.%s checksums still differ %s after load stopped: %v", database, table, deadline, sums) + } + time.Sleep(convergencePoll) + } +} + +// runHotWriters has each node in nodes overwrite rows 1..hotRows in turn +// until stop closes. A statement that fails is retried on the next turn. +func runHotWriters(t *testing.T, h *ClusterHarness, database, table string, nodes []int, stop <-chan struct{}) *sync.WaitGroup { + var wg sync.WaitGroup + for _, n := range nodes { + wg.Add(1) + go func(n int) { + defer wg.Done() + conn := openTimedNodeDatabase(t, h, n, database) + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + id := i%hotRows + 1 + if _, err := execTimed(conn, "UPDATE "+table+" SET v = ? WHERE id = ?", fmt.Sprintf("n%d#%d", n, i), id); err != nil { + time.Sleep(convergencePoll) + continue + } + h.Progress() + } + }(n) + } + return &wg +} + +func TestSameRowsOverwrittenAcrossOutageConvergeByValue(t *testing.T) { + h := NewClusterHarness(t) + defer h.Cleanup() + const database, table = "hotrows", "t" + startWithTables(t, h, map[string][]string{database: {table}}) + conn := openTimedNodeDatabase(t, h, 1, database) + for id := 1; id <= hotRows; id++ { + if _, err := execTimed(conn, "INSERT INTO "+table+" (id, v) VALUES (?, 'seed')", id); err != nil { + t.Fatalf("seed row %d: %v", id, err) + } + } + waitChecksumsEqual(t, h, database, table, convergenceDeadline) + h.Phase("seeded") + + if err := h.StopNode(3); err != nil { + t.Fatal(err) + } + h.Phase("node3-stopped") + stop := make(chan struct{}) + wg := runHotWriters(t, h, database, table, []int{1, 2}, stop) + sleepWithProgress(h, hotOutage) + + if err := h.StartNode(3); err != nil { + t.Fatal(err) + } + if err := h.WaitForAlive(3, restartDeadline); err != nil { + t.Fatal(err) + } + h.Phase("node3-alive") + sleepWithProgress(h, hotAfterRestart) + close(stop) + wg.Wait() + h.Phase("load-stopped") + + // One last live write per row, coordinated by node 3 itself, while the + // writes it missed during the outage may still be waiting to be pulled. + conn3 := openTimedNodeDatabase(t, h, 3, database) + for id := 1; id <= hotRows; id++ { + if _, err := execTimed(conn3, "UPDATE "+table+" SET v = ? WHERE id = ?", fmt.Sprintf("final#%d", id), id); err != nil { + t.Fatalf("final update of row %d: %v", id, err) + } + } + h.Phase("final-writes") + + waitChecksumsEqual(t, h, database, table, convergenceDeadline) + h.Phase("converged") +} From 5ebafa09e71a42baa3d339f2b60da33caabd7175 Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Tue, 29 Sep 2026 11:10:13 -0500 Subject: [PATCH 08/17] fix(cluster): a gracefully stopped node rejoins on restart; document narrow AUTO_INCREMENT ids A node restarted after a graceful stop shut itself down again about half a second later. Its peers kept the LEAVING record of its own departure, the node came back at incarnation 0, and it took that record for an admin decommission. The cluster harness never saw it because it only killed nodes. A restarting node now resumes one incarnation above the one it persisted, so its ALIVE supersedes its own earlier LEAVING. Incarnation bumps from refutation and from turning ALIVE are persisted, so a restart never comes back at or below an incarnation it already announced. A REMOVED view of self is never refuted, and an admin decommission of a live node still takes effect, including while the node is still catching up. A node restarted on a wiped data directory after a graceful stop keeps the old behaviour; the operations guide says to treat it as a replaced node. Docs: compatibility.mdx describes narrow AUTO_INCREMENT columns (per-node ranges, the startup vote hold and its retryable 1205, explicit-id rules with 1235/1264/1062, no reset after DROP/CREATE); sql-reference.mdx no longer says INT AUTO_INCREMENT behaves as BIGINT; operations.mdx covers graceful restart, decommission versus removal, and changing membership one node at a time; a new application page (LLDAP 0.6.3) records the tested version, the DATABASE_URL form and the verified 1-node and 3-node scenarios. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- docs/src/pages/_meta.json | 1 + docs/src/pages/compatibility.mdx | 31 ++++++ docs/src/pages/lldap.mdx | 38 +++++++ docs/src/pages/operations.mdx | 29 +++++ docs/src/pages/sql-reference.mdx | 3 +- grpc/node_registry.go | 29 ++++- grpc/self_refutation_test.go | 147 ++++++++++++++++++++++++++ test/crash_recovery_test.go | 41 +++++++ test/graceful_restart_cluster_test.go | 87 +++++++++++++++ 9 files changed, 403 insertions(+), 3 deletions(-) create mode 100644 docs/src/pages/lldap.mdx create mode 100644 grpc/self_refutation_test.go create mode 100644 test/graceful_restart_cluster_test.go diff --git a/docs/src/pages/_meta.json b/docs/src/pages/_meta.json index e28bdc11..10fa2614 100644 --- a/docs/src/pages/_meta.json +++ b/docs/src/pages/_meta.json @@ -5,6 +5,7 @@ "replication": "Replication", "integrations": "Integrations", "compatibility": "Compatibility", + "lldap": "LLDAP", "sql-reference": "SQL Reference", "vector-search": "Vector Search", "operations": "Operations", diff --git a/docs/src/pages/compatibility.mdx b/docs/src/pages/compatibility.mdx index 43382bd0..a42f1444 100644 --- a/docs/src/pages/compatibility.mdx +++ b/docs/src/pages/compatibility.mdx @@ -121,6 +121,37 @@ Marmot generates unique distributed IDs for `AUTO_INCREMENT` columns. Two modes Configure via `[mysql].auto_id_mode` in config. The compact mode uses: 41 bits timestamp + 6 bits node ID + 6 bits sequence. +These modes apply to `BIGINT AUTO_INCREMENT` columns. + +### Narrow AUTO_INCREMENT Columns + +An `AUTO_INCREMENT` column declared narrower than `BIGINT` (`TINYINT`, `SMALLINT`, `MEDIUMINT`, `INT`, signed or +`UNSIGNED`) gets ids that fit its declared width. Each node takes its ids from a range of the column it claims +through a cluster quorum, so no two nodes hand out the same id while the cluster membership is stable (see +Membership Changes and Narrow AUTO_INCREMENT Ids in [Operations](/operations)). + +- **Single node:** set `cluster.standalone = true` with `seed_nodes` empty. The node claims ranges without peers. +- **Cluster:** a node whose system database is new holds its claim votes until it has merged claim bases from + enough cluster members; until its log says `AUTO_INCREMENT claim votes released after merging claim bases`, narrow inserts can return the + retryable 1205. Start applications that insert at boot after that line. +- Ids increase within one node's range, not across nodes: two nodes inserting into the same table issue ids from + different ranges. Ids a range did not issue are skipped for good, for example after a restart. +- `LAST_INSERT_ID()` reports the generated id as usual. + +Explicit ids in a narrow column: + +| Explicit id | Result | +|-------------|--------| +| Beyond the declared width | Refused with 1264 (`Out of range value for column`) | +| Above the cluster's allocation base | Accepted; the base moves past it, so later generated ids are higher | +| At or below the base, inside a range this node claimed | Accepted | +| At or below the base, outside this node's ranges | Refused with 1235: another node may issue that id, and replicated inserts replace rows by key | + +A column whose ids are used up returns 1062 on the column's maximum, as MySQL does. + +Ids are not reset by `DROP TABLE` followed by `CREATE TABLE`, or by `DROP DATABASE` followed by `CREATE DATABASE`: +the recreated table continues above the ids the old one had claimed. + ## Limitations - **Selective Table Watching**: All tables in a database are replicated. Selective table replication is not supported. diff --git a/docs/src/pages/lldap.mdx b/docs/src/pages/lldap.mdx new file mode 100644 index 00000000..4172abbb --- /dev/null +++ b/docs/src/pages/lldap.mdx @@ -0,0 +1,38 @@ +--- +title: LLDAP +--- + +# LLDAP + +[LLDAP](https://github.com/lldap/lldap) runs on Marmot through its MySQL backend. Tested version: **0.6.3** +(`lldap/lldap:v0.6.3`). + +## Configuration + +Create the database once, on any node: + +```sql +CREATE DATABASE lldap; +``` + +Point LLDAP at a node's MySQL port: + +```bash +LLDAP_DATABASE_URL=mysql://root@marmot-node-1:3306/lldap +``` + +LLDAP's migrations run through Marmot and replicate like any other DDL. Its `groups` table uses an +`INT AUTO_INCREMENT` key, so group ids fit 32 bits (see Narrow AUTO_INCREMENT Columns in +[Compatibility](/compatibility)): + +- A single node needs `cluster.standalone = true`. +- On a new cluster, start LLDAP after the node it connects to logs `AUTO_INCREMENT claim votes released after merging claim bases`. Each node + issues group ids from its own range, so groups created through different nodes get ids from different ranges. + +## Verified Scenarios + +| Scenario | Verified | +|----------|----------| +| Single node | LLDAP boots and runs its migrations; users, groups and memberships created through the GraphQL API; a password set with `lldap_set_password`; LDAP bind as that user; all ids fit their columns; data and login survive a restart of LLDAP and then of Marmot | +| Three nodes, LLDAP on two of them | LLDAP instances on node 1 and node 2 see each other's users and groups and bind the same users; a node stopped gracefully while both keep writing, then restarted, holds identical rows in every LLDAP table (checked with a 30-second bound); no id collisions | +| Cold start on a cluster | LLDAP started against a new database on one node of three: its migrations reach every node, and the schema is identical on all nodes | diff --git a/docs/src/pages/operations.mdx b/docs/src/pages/operations.mdx index fb5d1ef2..a0991bcb 100644 --- a/docs/src/pages/operations.mdx +++ b/docs/src/pages/operations.mdx @@ -178,6 +178,35 @@ curl -X POST -H "X-Marmot-Secret: $SECRET" http://node1:8080/admin/cluster/remov # 5-node cluster → 3-node cluster, quorum: 3 → 2 ``` +**Restarting a Node:** a node stopped with SIGTERM announces that it is leaving, and its peers keep that record. +Started again, it supersedes the record and rejoins as ALIVE; no admin call is needed. This relies on the +membership record in its data directory: a node restarted on a wiped data directory after a graceful stop +still takes its old departure as a decommission and shuts down again. Handle it like a replaced node (see +Replacing a Failed Node): remove it, and allow it before reusing its `node_id`, or bring it back with a new one. + +A node decommissioned through `/admin/cluster/decommission` while ALIVE also rejoins if it is started again +(for example by a service manager that restarts it). `/admin/cluster/remove` (REMOVED) is the permanent exclusion. + +### Membership Changes and Narrow AUTO_INCREMENT Ids + +Ids of `AUTO_INCREMENT` columns narrower than `BIGINT` come from ranges each node claims through a quorum (see +Narrow AUTO_INCREMENT Columns in [Compatibility](/compatibility)). They are unique while the membership is stable. +Every node also raises its claim bases to the highest base every ALIVE peer reports, every 10 seconds, which closes +the window a membership change opens. To add or remove nodes, change the membership one node at a time: + +1. Add or remove **one** node. +2. Wait until it is ALIVE and its claim votes are released: its log says `AUTO_INCREMENT claim votes released after merging claim bases`, + or it never held them. +3. Run the sync on any node: + ```bash + curl -X POST -H "X-Marmot-Secret: $SECRET" http://node1:8080/admin/cluster/autoinc/sync + ``` +4. Make the next change only when the answer's `complete` is `true`: every member answered, is not holding its + votes, and reached every member. A sync gathers only from the peers a node sees ALIVE, so right after a change + it can be incomplete; re-run it until it is complete. + +The sync endpoint requires `cluster_secret`; on a cluster without one it answers 403. + ## Rolling Upgrades This release replaces how nodes catch up with each other: every node now diff --git a/docs/src/pages/sql-reference.mdx b/docs/src/pages/sql-reference.mdx index 0c22df3d..c0440e51 100644 --- a/docs/src/pages/sql-reference.mdx +++ b/docs/src/pages/sql-reference.mdx @@ -894,7 +894,8 @@ See the [Vector Search](/vector-search) page for the full startup guide, RAG sch | `INT UNSIGNED` | `INTEGER` | UNSIGNED removed | | `VARCHAR(n)` | `TEXT` | Length constraint removed | | `DATETIME` | `TEXT` | Date stored as ISO string | -| `INT AUTO_INCREMENT` | `BIGINT` | Auto-generated distributed IDs (see Auto ID Mode) | +| `INT AUTO_INCREMENT` | `INTEGER` | Ids fit the declared width (see Narrow AUTO_INCREMENT Columns in [Compatibility](/compatibility)) | +| `BIGINT AUTO_INCREMENT` | `INTEGER` | Auto-generated distributed IDs (see Auto ID Mode) | | `BOOLEAN` | `INTEGER` | Type changed | | `HIGH_PRIORITY` | (removed) | MySQL-only hint | | `SQL_NO_CACHE` | (removed) | MySQL-only hint | diff --git a/grpc/node_registry.go b/grpc/node_registry.go index 12ed4d89..6c45c1bd 100644 --- a/grpc/node_registry.go +++ b/grpc/node_registry.go @@ -311,8 +311,16 @@ func (nr *NodeRegistry) handleSelfUpdateLocked(node *NodeState) { return } + // REMOVED is terminal: every peer keeps it sticky, so refuting it would only + // climb our incarnation. Only the admin allow endpoint clears it. + if node.Status == NodeStatus_REMOVED { + return + } + // Accept LEAVING from higher incarnation — this is a remote decommission - // via admin API, not a failure detection. Don't refute it. + // via admin API, not a failure detection. Don't refute it. The departure + // a previous run of this node gossiped never reaches here: a restart + // starts above the incarnation it was persisted at (restoreMembershipLocked). if node.Status == NodeStatus_LEAVING && node.Incarnation > self.Incarnation { oldStatus := self.Status oldIncarnation := self.Incarnation @@ -356,6 +364,8 @@ func (nr *NodeRegistry) handleSelfUpdateLocked(node *NodeState) { if self.Status != NodeStatus_JOINING { self.Status = NodeStatus_ALIVE } + // Persist the bump, so a restart starts above what this run announced. + nr.updateClusterMetricsLocked() log.Debug(). Uint64("refuted_incarnation", node.Incarnation). Uint64("new_incarnation", self.Incarnation). @@ -686,8 +696,9 @@ func (nr *NodeRegistry) MarkAlive(nodeID uint64) { // Record state transition if status changed if oldStatus != NodeStatus_ALIVE { telemetry.NodeStateTransitionsTotal.With(oldStatus.String(), NodeStatus_ALIVE.String()).Inc() - nr.updateClusterMetricsLocked() } + // The incarnation changed even when the status did not: persist it. + nr.updateClusterMetricsLocked() } } @@ -1111,9 +1122,12 @@ func (nr *NodeRegistry) restoreMembershipLocked() int { } restored := 0 + var selfIncarnation uint64 + selfRecorded := false for _, record := range snapshot.Nodes { if record.NodeID == nr.localNodeID { // Self is ALIVE by construction and was added by the caller. + selfIncarnation, selfRecorded = record.Incarnation, true continue } @@ -1137,6 +1151,17 @@ func (nr *NodeRegistry) restoreMembershipLocked() int { // The snapshot we just read is what is on disk; record it so an unchanged // membership does not rewrite the same bytes on the first mutation. nr.persistedFingerprint = nr.membershipFingerprintLocked() + + // Start above every incarnation the previous run announced: every change + // of our own incarnation is persisted (MarkLeaving, MarkAlive and SWIM + // refutation all reach persistMembershipLocked). Peers keep that run's + // last record of us - LEAVING after a graceful stop - and a restart at or + // below it would lose to it. Set after the fingerprint so the next + // membership change persists the new value. A failed persist is logged + // and leaves the older incarnation on disk. + if selfRecorded { + nr.nodes[nr.localNodeID].Incarnation = selfIncarnation + 1 + } return restored } diff --git a/grpc/self_refutation_test.go b/grpc/self_refutation_test.go new file mode 100644 index 00000000..bccb4f16 --- /dev/null +++ b/grpc/self_refutation_test.go @@ -0,0 +1,147 @@ +package grpc + +import ( + "sync/atomic" + "testing" +) + +// leavingWatch counts how often the registry asks this node to shut down. +func leavingWatch(nr *NodeRegistry) *atomic.Int32 { + var fired atomic.Int32 + nr.SetOnNodeLeaving(func() { fired.Add(1) }) + return &fired +} + +// TestGracefulStopThenRestartStaysUp: a node that stopped gracefully announced +// itself LEAVING, and its peers keep that record. When it restarts, that record +// is its own previous departure, not an admin decommission, and must not shut +// the new process down. +func TestGracefulStopThenRestartStaysUp(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + first := registryAt(t, 3, "10.0.0.3:7100", dir) + first.Add(&NodeState{NodeId: 1, Address: "10.0.0.1:7100", Status: NodeStatus_ALIVE, Incarnation: 2}) + first.MarkAlive(3) + if err := first.MarkSelfLeaving(); err != nil { + t.Fatalf("MarkSelfLeaving: %v", err) + } + departed, _ := first.Get(3) + + restarted := registryAt(t, 3, "10.0.0.3:7100", dir) + fired := leavingWatch(restarted) + // A node the startup pull finds current keeps its status, so this restart + // is ALIVE from the first gossip it hears. The peers' record of it is the + // departure the previous process gossiped. + restarted.Update(&NodeState{NodeId: 3, Status: NodeStatus_LEAVING, Incarnation: departed.Incarnation}) + + self, _ := restarted.Get(3) + if self.Status != NodeStatus_ALIVE || fired.Load() != 0 { + t.Fatalf("restarted node took its own departure as a decommission: status=%v shutdowns=%d", + self.Status, fired.Load()) + } + if self.Incarnation <= departed.Incarnation { + t.Fatalf("restarted incarnation %d does not supersede the departure at %d", + self.Incarnation, departed.Incarnation) + } +} + +// TestDecommissionWhileLocallyJoiningTakesEffect: the startup catch-up marks a +// node JOINING without an incarnation bump, so peers still see it ALIVE and an +// admin may decommission it. The node must accept that decommission. +func TestDecommissionWhileLocallyJoiningTakesEffect(t *testing.T) { + t.Parallel() + admin := NewNodeRegistry(1, "10.0.0.1:7100") + node := NewNodeRegistry(3, "10.0.0.3:7100") + fired := leavingWatch(node) + + gossiped, _ := node.Get(3) + admin.Update(copyNodeState(gossiped)) + node.MarkJoining(3) + if v, _ := admin.Get(3); v.Status != NodeStatus_ALIVE { + t.Fatalf("fixture: admin view %v, want ALIVE", v.Status) + } + if err := admin.MarkLeaving(3); err != nil { + t.Fatal(err) + } + leaving, _ := admin.Get(3) + node.Update(copyNodeState(leaving)) + + self, _ := node.Get(3) + if self.Status != NodeStatus_LEAVING || fired.Load() != 1 { + t.Fatalf("decommission refuted: status=%v@%d shutdowns=%d", self.Status, self.Incarnation, fired.Load()) + } +} + +// TestOwnIncarnationBumpsSurviveRestart: a restart starts above every +// incarnation the previous run announced, including a SWIM refutation and a +// MarkAlive that left the status unchanged. +func TestOwnIncarnationBumpsSurviveRestart(t *testing.T) { + t.Parallel() + cases := map[string]func(nr *NodeRegistry){ + "refutation": func(nr *NodeRegistry) { + nr.Update(&NodeState{NodeId: 3, Status: NodeStatus_DEAD, Incarnation: 5}) + }, + "mark-alive-while-alive": func(nr *NodeRegistry) { nr.MarkAlive(3) }, + } + for name, bump := range cases { + dir := t.TempDir() + first := registryAt(t, 3, "10.0.0.3:7100", dir) + bump(first) + announced, _ := first.Get(3) + + restarted := registryAt(t, 3, "10.0.0.3:7100", dir) + self, _ := restarted.Get(3) + if self.Incarnation <= announced.Incarnation { + t.Errorf("%s: restart at incarnation %d, not above the announced %d", + name, self.Incarnation, announced.Incarnation) + } + } +} + +// TestRemovedViewOfSelfIsNotRefuted: REMOVED is terminal. Refuting it would +// only climb this node's incarnation against a record every peer keeps sticky. +func TestRemovedViewOfSelfIsNotRefuted(t *testing.T) { + t.Parallel() + for _, joining := range []bool{false, true} { + nr := NewNodeRegistry(3, "10.0.0.3:7100") + if joining { + nr.MarkJoining(3) + } + before, _ := nr.Get(3) + + nr.Update(&NodeState{NodeId: 3, Status: NodeStatus_REMOVED, Incarnation: before.Incarnation + 1}) + + self, _ := nr.Get(3) + if self.Incarnation != before.Incarnation { + t.Fatalf("joining=%v: REMOVED view refuted, incarnation %d -> %d", + joining, before.Incarnation, self.Incarnation) + } + } +} + +// TestLiveAdminDecommissionStillTakesEffect: a peer's admin decommission of this +// node while it is ALIVE marks it LEAVING at the node's incarnation + 1; the node +// accepts it and starts shutting down, also after a restart. +func TestLiveAdminDecommissionStillTakesEffect(t *testing.T) { + t.Parallel() + dir := t.TempDir() + first := registryAt(t, 3, "10.0.0.3:7100", dir) + first.MarkAlive(3) + if err := first.MarkSelfLeaving(); err != nil { + t.Fatalf("MarkSelfLeaving: %v", err) + } + + nr := registryAt(t, 3, "10.0.0.3:7100", dir) + fired := leavingWatch(nr) + nr.MarkJoining(3) + nr.MarkAlive(3) + live, _ := nr.Get(3) + + nr.Update(&NodeState{NodeId: 3, Status: NodeStatus_LEAVING, Incarnation: live.Incarnation + 1}) + + self, _ := nr.Get(3) + if self.Status != NodeStatus_LEAVING || fired.Load() != 1 { + t.Fatalf("live decommission not accepted: status=%v shutdowns=%d", self.Status, fired.Load()) + } +} diff --git a/test/crash_recovery_test.go b/test/crash_recovery_test.go index 917f5289..d5edfcd9 100644 --- a/test/crash_recovery_test.go +++ b/test/crash_recovery_test.go @@ -582,6 +582,47 @@ func (h *ClusterHarness) StopNode(nodeID int) error { return nil } +// GracefulStopNode sends SIGTERM and waits for the node's own shutdown - the +// path that announces LEAVING to its peers - unlike StopNode, which kills it. +func (h *ClusterHarness) GracefulStopNode(nodeID int, timeout time.Duration) error { + node := h.Nodes[nodeID-1] + node.mu.Lock() + defer node.mu.Unlock() + + if !node.isRunning { + return fmt.Errorf("node %d is not running", nodeID) + } + if node.DB != nil { + node.DB.Close() + node.DB = nil + } + if err := node.Cmd.Process.Signal(syscall.SIGTERM); err != nil { + return fmt.Errorf("SIGTERM node %d: %w", nodeID, err) + } + + exited := make(chan error, 1) + go func() { exited <- node.Cmd.Wait() }() + var err error + select { + case waitErr := <-exited: + if waitErr != nil { + err = fmt.Errorf("node %d exited with an error after SIGTERM: %w", nodeID, waitErr) + } else { + h.t.Logf("Node %d stopped gracefully", nodeID) + } + case <-time.After(timeout): + err = fmt.Errorf("node %d did not shut down within %v of SIGTERM", nodeID, timeout) + if killErr := node.Cmd.Process.Kill(); killErr != nil { + err = fmt.Errorf("%w; kill: %v", err, killErr) + } + <-exited + } + node.isRunning = false + node.Cmd = nil + os.Remove(node.PIDFile) + return err +} + func (h *ClusterHarness) KillNode(nodeID int) error { node := h.Nodes[nodeID-1] node.mu.Lock() diff --git a/test/graceful_restart_cluster_test.go b/test/graceful_restart_cluster_test.go new file mode 100644 index 00000000..30d05a3d --- /dev/null +++ b/test/graceful_restart_cluster_test.go @@ -0,0 +1,87 @@ +package test + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "testing" + "time" +) + +// gracefulStopDeadline bounds a node's own shutdown after SIGTERM. +const gracefulStopDeadline = 30 * time.Second + +// memberStatus returns nodeID's status in observer's membership view. +func memberStatus(h *ClusterHarness, observer int, nodeID uint64) (string, error) { + url := fmt.Sprintf("http://localhost:%d/admin/cluster/members", h.Nodes[observer-1].GRPCPort) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return "", err + } + req.Header.Set("X-Marmot-Secret", "test-secret") + resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + var wrapped struct { + Data []struct { + NodeID uint64 `json:"node_id"` + Status string `json:"status"` + } `json:"data"` + } + if err := json.Unmarshal(body, &wrapped); err != nil { + return "", fmt.Errorf("members on node %d: %w: %s", observer, err, body) + } + for _, m := range wrapped.Data { + if m.NodeID == nodeID { + return m.Status, nil + } + } + return "", fmt.Errorf("node %d absent from node %d's membership", nodeID, observer) +} + +// TestGracefulStopThenRestartRejoins: a node stopped with SIGTERM announces +// LEAVING and its peers keep that record. Restarted, it must rejoin as ALIVE and +// serve writes, not read its own departure as a decommission and shut down. +func TestGracefulStopThenRestartRejoins(t *testing.T) { + h := NewClusterHarness(t) + defer h.Cleanup() + tables := map[string][]string{"marmot": {"t"}} + startWithTables(t, h, tables) + + if err := h.GracefulStopNode(3, gracefulStopDeadline); err != nil { + t.Fatal(err) + } + h.Phase("node-3-stopped-gracefully") + if err := h.StartNode(3); err != nil { + t.Fatal(err) + } + if err := h.WaitForAlive(3, restartDeadline); err != nil { + t.Fatal(err) + } + + deadline := time.Now().Add(restartDeadline) + for { + s1, err1 := memberStatus(h, 1, 3) + s3, err3 := memberStatus(h, 3, 3) + if err1 == nil && err3 == nil && s1 == "ALIVE" && s3 == "ALIVE" { + break + } + if time.Now().After(deadline) { + t.Fatalf("node 3 not ALIVE within %v of restart: node1 view=%q (%v), own view=%q (%v); log tail:\n%s", + restartDeadline, s1, err1, s3, err3, h.getNodeLogTail(3, 30)) + } + time.Sleep(convergencePoll) + } + h.Phase("node-3-alive") + + l := newKeyLedger(h) + insertFromEveryNode(t, h, l, "after-restart") + waitKeysConverged(t, h, l, tables, convergenceDeadline) +} From 98eafbc0758f5366c0c63da30e644c127fe41f48 Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Tue, 29 Sep 2026 16:05:21 -0500 Subject: [PATCH 09/17] fix(replication): recover cleanly from a kill mid-commit; end a rolled-back explicit transaction Defects found while rewriting the cluster suite, each fixed test first: - A local prepare whose captured rows were lost to SIGKILL wedged catch-up: the puller retried CommitLocallyPrepared until the txn was marked stuck and the node stayed JOINING. It now discards the local prepare under the commit guard and replays the peer's committed copy. - A txn committed just before SIGKILL had its commit record rewritten at open without releasing the row locks restored for it, so every later change to those rows was refused on replay and the node kept a stale value. repairAppliedTxnMetadata now cleans up after the commit it rewrites. - A narrow AUTO_INCREMENT claim inside an explicit transaction deadlocked on the batch committer flush, which waited on the writer the same transaction's pinned session holds. A claim-only commit has no SQLite work and no longer flushes. - A pinned statement whose SQLite transaction the lock wait rolled back now answers 1213 and ends the transaction, so a later COMMIT commits nothing. - XsyncTransactionStore mutated shared TxnState in place while the stale transaction GC read it. Updates are now copy-on-write. The AUTO_INCREMENT merge and base-sync intervals become config knobs with unchanged defaults. The operations guide says to rebuild wiped nodes one at a time. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- cfg/autoinc_interval_validation_test.go | 50 +++++++++++ cfg/config.go | 39 +++++++++ config.toml.example | 10 +++ coordinator/handler.go | 8 ++ coordinator/pinned_txn_rollback_test.go | 58 +++++++++++++ db/applied_txn_marker.go | 6 ++ db/memory_stores.go | 7 +- db/memory_stores_xsync.go | 34 +++++--- db/memory_stores_xsync_test.go | 43 +++++++++ db/prepared_intent_test.go | 110 ++++++++++++++++++++++++ db/prepared_recovery_test.go | 71 +++++++++++++++ db/replication_engine.go | 12 +++ db/transaction.go | 33 +++++-- docs/src/pages/operations.mdx | 11 ++- grpc/autoinc_merge.go | 64 +++++++------- grpc/autoinc_merge_test.go | 11 ++- grpc/log_puller.go | 20 ++++- grpc/log_puller_gc_race_test.go | 26 ++---- grpc/log_puller_test.go | 58 +++++++++++++ marmot.go | 6 +- protocol/error_mapper.go | 10 +++ protocol/error_mapper_test.go | 14 +++ 22 files changed, 631 insertions(+), 70 deletions(-) create mode 100644 cfg/autoinc_interval_validation_test.go create mode 100644 coordinator/pinned_txn_rollback_test.go diff --git a/cfg/autoinc_interval_validation_test.go b/cfg/autoinc_interval_validation_test.go new file mode 100644 index 00000000..b0115772 --- /dev/null +++ b/cfg/autoinc_interval_validation_test.go @@ -0,0 +1,50 @@ +package cfg + +import ( + "testing" + "time" +) + +// TestValidate_AutoIncIntervals: an unset AUTO_INCREMENT merge or base sync +// interval takes its default, a set one is kept, and a negative one is +// refused. +func TestValidate_AutoIncIntervals(t *testing.T) { + tests := []struct { + name string + merge, sync int + wantMerge, wantSync time.Duration + wantErr bool + }{ + {name: "unset takes the defaults", wantMerge: 2 * time.Second, wantSync: 10 * time.Second}, + {name: "set values are kept", merge: 100, sync: 500, wantMerge: 100 * time.Millisecond, wantSync: 500 * time.Millisecond}, + {name: "negative merge is refused", merge: -1, wantErr: true}, + {name: "negative sync is refused", sync: -1, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + original := Config + defer func() { Config = original }() + conf := *original + conf.Cluster.AutoIncMergeIntervalMS = tt.merge + conf.Cluster.AutoIncBaseSyncIntervalMS = tt.sync + Config = &conf + + err := Validate() + if tt.wantErr { + if err == nil { + t.Fatal("Validate accepted a negative interval") + } + return + } + if err != nil { + t.Fatalf("Validate: %v", err) + } + if got := Config.Cluster.GetAutoIncMergeInterval(); got != tt.wantMerge { + t.Fatalf("merge interval = %s, want %s", got, tt.wantMerge) + } + if got := Config.Cluster.GetAutoIncBaseSyncInterval(); got != tt.wantSync { + t.Fatalf("base sync interval = %s, want %s", got, tt.wantSync) + } + }) + } +} diff --git a/cfg/config.go b/cfg/config.go index b67c809c..666b8ff0 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -37,6 +37,32 @@ type ClusterConfiguration struct { DeadTimeoutMS int `toml:"dead_timeout_ms"` ShutdownGracePeriodMS int `toml:"shutdown_grace_period_ms"` // Grace period for in-flight queries during shutdown (default: 15000ms) Promotion PromotionConfiguration `toml:"promotion"` + + // AutoIncMergeIntervalMS is how often a node whose AUTO_INCREMENT claim + // votes are held retries merging claim bases from its peers: the most a + // node declines claims after enough of its peers became reachable + // (default: 2000ms). + AutoIncMergeIntervalMS int `toml:"autoinc_merge_interval_ms"` + // AutoIncBaseSyncIntervalMS is how often a node raises its AUTO_INCREMENT + // claim bases to every alive peer's: the longest a claim committed before + // a membership change can stay unknown to a new quorum (default: 10000ms). + AutoIncBaseSyncIntervalMS int `toml:"autoinc_base_sync_interval_ms"` +} + +// Defaults of ClusterConfiguration's AUTO_INCREMENT claim base intervals. +const ( + DefaultAutoIncMergeIntervalMS = 2000 + DefaultAutoIncBaseSyncIntervalMS = 10000 +) + +// GetAutoIncMergeInterval returns AutoIncMergeIntervalMS as a time.Duration. +func (c *ClusterConfiguration) GetAutoIncMergeInterval() time.Duration { + return time.Duration(c.AutoIncMergeIntervalMS) * time.Millisecond +} + +// GetAutoIncBaseSyncInterval returns AutoIncBaseSyncIntervalMS as a time.Duration. +func (c *ClusterConfiguration) GetAutoIncBaseSyncInterval() time.Duration { + return time.Duration(c.AutoIncBaseSyncIntervalMS) * time.Millisecond } // GetShutdownGracePeriod returns the configured shutdown grace period as a time.Duration. @@ -296,6 +322,9 @@ var Config = &Configuration{ CheckIntervalSeconds: 2, // Check every 2 seconds MinHealthyDurationSec: 3, // Must be healthy for 3 seconds }, + + AutoIncMergeIntervalMS: DefaultAutoIncMergeIntervalMS, + AutoIncBaseSyncIntervalMS: DefaultAutoIncBaseSyncIntervalMS, }, Replication: ReplicationConfiguration{ @@ -810,6 +839,16 @@ func Validate() error { if Config.Cluster.ShutdownGracePeriodMS < 1000 || Config.Cluster.ShutdownGracePeriodMS > 300000 { return fmt.Errorf("cluster.shutdown_grace_period_ms must be between 1000ms and 300000ms (5 minutes)") } + // 0 means the default, as for the grace period above. + if Config.Cluster.AutoIncMergeIntervalMS == 0 { + Config.Cluster.AutoIncMergeIntervalMS = DefaultAutoIncMergeIntervalMS + } + if Config.Cluster.AutoIncBaseSyncIntervalMS == 0 { + Config.Cluster.AutoIncBaseSyncIntervalMS = DefaultAutoIncBaseSyncIntervalMS + } + if Config.Cluster.AutoIncMergeIntervalMS < 0 || Config.Cluster.AutoIncBaseSyncIntervalMS < 0 { + return fmt.Errorf("cluster.autoinc_merge_interval_ms and cluster.autoinc_base_sync_interval_ms must be positive") + } // Validate publisher configuration if Config.Publisher.Enabled { diff --git a/config.toml.example b/config.toml.example index e8bab624..c7a33fef 100644 --- a/config.toml.example +++ b/config.toml.example @@ -63,6 +63,16 @@ seed_nodes = [] # seed_nodes. standalone = false +# How often a node whose AUTO_INCREMENT claim votes are held retries merging +# claim bases from its peers: after enough peers are reachable, the node +# declines claims for at most this long. +autoinc_merge_interval_ms = 2000 + +# How often every node raises its AUTO_INCREMENT claim bases to every alive +# peer's: after a membership change, a claim committed under the old +# membership stays unknown to a new quorum for at most this long. +autoinc_base_sync_interval_ms = 10000 + # PSK for cluster authentication (or set MARMOT_CLUSTER_SECRET env var) # Leave empty to disable authentication. The AUTO_INCREMENT endpoints # (/admin/cluster/autoinc/release-votes and /admin/cluster/autoinc/sync) are diff --git a/coordinator/handler.go b/coordinator/handler.go index ca62c240..388ed7eb 100644 --- a/coordinator/handler.go +++ b/coordinator/handler.go @@ -3,6 +3,7 @@ package coordinator import ( "context" "database/sql" + "errors" "fmt" "math" "reflect" @@ -927,6 +928,13 @@ func (h *CoordinatorHandler) executeEagerDML(session *protocol.ConnectionSession defer cancel() rowsAffected, lastInsertId, err := pinned.ExecuteStatement(execCtx, stmt.SQL, execParams) if err != nil { + if errors.Is(err, sql.ErrTxDone) { + // The pinned session's SQLite transaction is gone: it outlived + // the lock wait and was rolled back. The client is answered 1213 + // (protocol.ConvertToMySQLError), so end the whole transaction as + // MySQL does: a later COMMIT must find nothing of it to commit. + _, _ = h.handleRollback(session) + } return nil, fmt.Errorf("DML execution failed: %w", err) } diff --git a/coordinator/pinned_txn_rollback_test.go b/coordinator/pinned_txn_rollback_test.go new file mode 100644 index 00000000..c55738be --- /dev/null +++ b/coordinator/pinned_txn_rollback_test.go @@ -0,0 +1,58 @@ +//go:build sqlite_preupdate_hook + +package coordinator_test + +import ( + "testing" + "time" + + "github.com/maxpert/marmot/cfg" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// TestRolledBackExplicitTransactionLeavesNothingToCommit: an explicit +// transaction's pinned session outlives the lock wait, so its SQLite +// transaction is rolled back and the next statement is answered 1213 - +// MySQL's "the transaction was rolled back, run it again". The transaction +// must then be over, as in MySQL: a later COMMIT applies and replicates +// nothing of it, and a later BEGIN starts a fresh transaction that commits. +// +// Mutation: answer the ended pinned session with 1213 without ending the +// transaction. The COMMIT applies 'first' and "applied the rolled-back +// statement" fires. +func TestRolledBackExplicitTransactionLeavesNothingToCommit(t *testing.T) { + old := cfg.Config.Transaction.LockWaitTimeoutSeconds + cfg.Config.Transaction.LockWaitTimeoutSeconds = 1 + t.Cleanup(func() { cfg.Config.Transaction.LockWaitTimeoutSeconds = old }) + + s := setupNoopDML(t) + _, err := s.handler.HandleQuery(s.session, "BEGIN", nil) + require.NoError(t, err) + _, err = s.handler.HandleQuery(s.session, "INSERT INTO t (name) VALUES ('first')", nil) + require.NoError(t, err) + // Outlive the pinned session's lock wait. + time.Sleep(1500 * time.Millisecond) + _, err = s.handler.HandleQuery(s.session, "INSERT INTO t (name) VALUES ('second')", nil) + require.Error(t, err) + me := protocol.ConvertToMySQLError(err) + require.Equal(t, uint16(protocol.ErrCodeDeadlock), me.Code, "statement after the rollback: %s", me.Message) + require.False(t, s.session.InTransaction(), "a 1213 must end the transaction") + + commitsBefore := s.replicator.commits + _, err = s.handler.HandleQuery(s.session, "COMMIT", nil) + require.NoError(t, err, "COMMIT outside a transaction is a no-op in MySQL") + var count int + require.NoError(t, s.conn.QueryRow("SELECT COUNT(*) FROM t WHERE name IN ('first', 'second')").Scan(&count)) + require.Zero(t, count, "COMMIT after 1213 applied the rolled-back statement") + require.Equal(t, commitsBefore, s.replicator.commits, "COMMIT after 1213 replicated the rolled-back statement") + + _, err = s.handler.HandleQuery(s.session, "BEGIN", nil) + require.NoError(t, err) + _, err = s.handler.HandleQuery(s.session, "INSERT INTO t (name) VALUES ('retry')", nil) + require.NoError(t, err, "a fresh BEGIN must not reuse the ended session") + _, err = s.handler.HandleQuery(s.session, "COMMIT", nil) + require.NoError(t, err) + require.NoError(t, s.conn.QueryRow("SELECT COUNT(*) FROM t WHERE name = 'retry'").Scan(&count)) + require.Equal(t, 1, count, "the retried transaction must commit") +} diff --git a/db/applied_txn_marker.go b/db/applied_txn_marker.go index 5aeb352f..c3234af0 100644 --- a/db/applied_txn_marker.go +++ b/db/applied_txn_marker.go @@ -81,6 +81,12 @@ func repairAppliedTxnMetadata(db *sql.DB, metaStore MetaStore, dbName string) er if err := metaStore.CommitTransaction(txnID, commitTS, nil, dbName, "", 0, uint32(len(entries))); err != nil { return fmt.Errorf("commit txn %d: %w", txnID, err) } + // The open recovered this transaction as prepared, with its row + // locks and intents; free them as any commit does, or every later + // replay touching those rows is refused. + if err := metaStore.CleanupAfterCommit(txnID); err != nil { + return fmt.Errorf("clean up repaired commit of txn %d: %w", txnID, err) + } continue } if err := metaStore.StoreReplayedTransaction(txnID, nodeID, commitTS, dbName, uint32(len(entries)), 0); err != nil { diff --git a/db/memory_stores.go b/db/memory_stores.go index 97bd5c75..7ba84a92 100644 --- a/db/memory_stores.go +++ b/db/memory_stores.go @@ -14,10 +14,12 @@ type TxnState struct { // TransactionStore manages in-memory transaction state for active transactions. // This interface provides fast lookups and updates without disk I/O overhead. type TransactionStore interface { - // Begin registers a new transaction with initial state. + // Begin registers a new transaction with initial state. The store owns + // state from then on: the caller must not write it again. Begin(txnID uint64, state *TxnState) // Get retrieves transaction state. Returns false if transaction not found. + // The state is a snapshot that is never written again: updates replace it. Get(txnID uint64) (*TxnState, bool) // UpdateStatus updates transaction status (Pending, Committed, Aborted). @@ -34,7 +36,8 @@ type TransactionStore interface { RangePending(fn func(txnID uint64) bool) // RangeAll iterates over all transactions regardless of status. - // Iterator returns true to continue, false to stop. + // Iterator returns true to continue, false to stop. Each state is a + // snapshot, as from Get. RangeAll(fn func(txnID uint64, state *TxnState) bool) // CountPending returns the number of pending transactions. diff --git a/db/memory_stores_xsync.go b/db/memory_stores_xsync.go index 16e2f942..5fc03909 100644 --- a/db/memory_stores_xsync.go +++ b/db/memory_stores_xsync.go @@ -30,20 +30,34 @@ func (s *XsyncTransactionStore) Get(txnID uint64) (*TxnState, bool) { } func (s *XsyncTransactionStore) UpdateStatus(txnID uint64, status TxnStatus) { - if state, ok := s.txns.Load(txnID); ok { - state.Status = status - if status != TxnStatusPending { - s.pending.Delete(txnID) - } else { - s.pending.Store(txnID, struct{}{}) - } + if !s.update(txnID, func(state *TxnState) { state.Status = status }) { + return + } + if status != TxnStatusPending { + s.pending.Delete(txnID) + } else { + s.pending.Store(txnID, struct{}{}) } } func (s *XsyncTransactionStore) UpdateHeartbeat(txnID uint64, ts int64) { - if state, ok := s.txns.Load(txnID); ok { - state.LastHeartbeat = ts - } + s.update(txnID, func(state *TxnState) { state.LastHeartbeat = ts }) +} + +// update replaces txnID's state with a copy that change modified, atomically +// with any other update of it, and reports whether txnID was present. A +// TxnState a reader got from Get or RangeAll is never written again, so the +// reader needs no lock. +func (s *XsyncTransactionStore) update(txnID uint64, change func(*TxnState)) bool { + _, ok := s.txns.Compute(txnID, func(old *TxnState, loaded bool) (*TxnState, bool) { + if !loaded { + return nil, true + } + next := *old + change(&next) + return &next, false + }) + return ok } func (s *XsyncTransactionStore) Remove(txnID uint64) { diff --git a/db/memory_stores_xsync_test.go b/db/memory_stores_xsync_test.go index 29a70571..83128667 100644 --- a/db/memory_stores_xsync_test.go +++ b/db/memory_stores_xsync_test.go @@ -545,3 +545,46 @@ func TestXsyncCDCLockStore_HasRowLocks(t *testing.T) { t.Error("should have no locks after release") } } + +// TestXsyncTransactionStore_UpdatesDoNotRaceReaders: a transaction's +// heartbeat and status are updated by one goroutine (the heartbeat RPC, the +// commit path) while another reads the state it got from Get or RangeAll (the +// stale-transaction GC). Run under -race, this fails if an update writes into +// a TxnState a reader already holds. +func TestXsyncTransactionStore_UpdatesDoNotRaceReaders(t *testing.T) { + store := NewXsyncTransactionStore() + const txnID = 1 + store.Begin(txnID, &TxnState{NodeID: 1, Status: TxnStatusPending, LastHeartbeat: 1}) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := int64(2); i < 2000; i++ { + store.UpdateHeartbeat(txnID, i) + } + store.UpdateStatus(txnID, TxnStatusAborted) + }() + var lastSeen int64 + go func() { + defer wg.Done() + for i := 0; i < 2000; i++ { + if state, ok := store.Get(txnID); ok && state.Status == TxnStatusPending { + lastSeen = max(lastSeen, state.LastHeartbeat) + } + store.RangeAll(func(_ uint64, state *TxnState) bool { + lastSeen = max(lastSeen, state.LastHeartbeat) + return true + }) + } + }() + wg.Wait() + + state, ok := store.Get(txnID) + if !ok || state.Status != TxnStatusAborted || state.LastHeartbeat != 1999 { + t.Fatalf("final state = %+v (found %v), want ABORTED with heartbeat 1999", state, ok) + } + if lastSeen > 1999 { + t.Fatalf("a reader saw heartbeat %d, never written", lastSeen) + } +} diff --git a/db/prepared_intent_test.go b/db/prepared_intent_test.go index 16e71205..42321d2e 100644 --- a/db/prepared_intent_test.go +++ b/db/prepared_intent_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/maxpert/marmot/cfg" "github.com/maxpert/marmot/hlc" "github.com/maxpert/marmot/protocol" "github.com/stretchr/testify/require" @@ -407,3 +408,112 @@ func TestCommitRefusesADMLTransactionWhoseRowsTheGCDeleted(t *testing.T) { require.NoError(t, mdb.GetReadDB().QueryRow("SELECT COUNT(*) FROM log").Scan(&n)) require.Zero(t, n) } + +// TestClaimCommitDoesNotWaitOnQueuedDMLCommits: an explicit transaction's +// session holds this node's SQLite writer; one peer DML commit is being +// flushed and waits for that writer, and a second one is queued behind it in +// the batch committer. The session's next narrow insert claims a range. The +// claim writes nothing to the user database, so its local commit must not +// wait for the queued DML: that DML waits for the writer the session holds, +// and the session waits for the claim, so waiting deadlocked until the lock +// wait ran out and the whole transaction failed. +// +// Mutation: flush the batch committer before every non-DML commit (the old +// behaviour). The claim's COMMIT takes the queued DML and waits on the writer. +func TestClaimCommitDoesNotWaitOnQueuedDMLCommits(t *testing.T) { + // A long batch window keeps a queued commit observable before a flush + // takes it. + oldWait := cfg.Config.BatchCommit.MaxWaitMS + cfg.Config.BatchCommit.MaxWaitMS = 200 + t.Cleanup(func() { cfg.Config.BatchCommit.MaxWaitMS = oldWait }) + + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE users (id INTEGER /*M:32a*/ PRIMARY KEY, v TEXT)") + _, err := mdb.GetWriteDB().Exec("CREATE TABLE log (id INTEGER PRIMARY KEY, v TEXT)") + require.NoError(t, err) + seedClaimBase(t, dm, "testdb", "users", 500, 1) + bc := mdb.batchCommitter + require.NotNil(t, bc, "the test needs the batch committer") + + const flushing, claim, queued = 9300, 9301, 9302 + prepareRow := func(txnID uint64, key string, id int64) { + prep := engine.Prepare(context.Background(), &PrepareRequest{ + TxnID: txnID, NodeID: 2, StartTS: hlc.Timestamp{WallTime: int64(txnID)}, Database: "testdb", + Statements: []protocol.Statement{dmlLogInsertStatement("testdb", key, id, "peer")}, + }) + require.True(t, prep.Success, "prepare %d: %s", txnID, prep.Error) + } + commitRow := func(txnID uint64, key string) <-chan *CommitResult { + done := make(chan *CommitResult, 1) + go func() { + done <- engine.Commit(context.Background(), &CommitRequest{TxnID: txnID, Database: "testdb", + Statements: []protocol.Statement{dmlLogCommitStatement("testdb", key)}}) + }() + return done + } + prepareRow(flushing, "log:1", 1) + prepareRow(queued, "log:3", 3) + prep := prepareClaim(engine, t, claim, 1, 500, 10) + require.True(t, prep.Success, "claim prepare: %s", prep.Error) + + // The session's open transaction holds the writer. + session, err := mdb.GetWriteDB().Begin() + require.NoError(t, err) + _, err = session.Exec("INSERT INTO log (id, v) VALUES (2, 'session')") + require.NoError(t, err) + + // The first DML is queued, then taken by the timer's flush, which waits + // on the writer; the second is then queued and stays there. + flushingDone := commitRow(flushing, "log:1") + require.Eventually(t, func() bool { return pendingInBatch(bc, flushing) }, 5*time.Second, time.Millisecond) + require.Eventually(t, func() bool { return !pendingInBatch(bc, flushing) }, 5*time.Second, time.Millisecond) + queuedDone := commitRow(queued, "log:3") + require.Eventually(t, func() bool { return pendingInBatch(bc, queued) }, 5*time.Second, time.Millisecond) + + claimDone := make(chan *CommitResult, 1) + go func() { claimDone <- commitClaim(engine, claim) }() + select { + case res := <-claimDone: + require.True(t, res.Success, "claim COMMIT: %s", res.Error) + case <-time.After(2 * time.Second): + t.Fatal("the claim's COMMIT waited on DML queued behind the writer the session holds") + } + require.Equal(t, uint64(510), readClaimBase(t, dm, "testdb", "users")) + + require.NoError(t, session.Rollback()) + for _, done := range []<-chan *CommitResult{flushingDone, queuedDone} { + res := <-done + require.True(t, res.Success, "DML COMMIT: %s", res.Error) + } +} + +// TestDiscardPreparedOfACommittedTxnReportsItCommitted: the log puller +// discards a prepare whose rows are gone and replays the peer's copy; if +// another path committed the transaction in between, DiscardPrepared must +// leave it alone and report ErrTxnAlreadyCommitted, which the puller counts +// as covered (grpc/log_puller.go resolveLocalPending). +func TestDiscardPreparedOfACommittedTxnReportsItCommitted(t *testing.T) { + engine, dm, cleanup := setupTestReplicationEngine(t) + defer cleanup() + mdb := markedTableDB(t, engine, dm, "CREATE TABLE log (id INTEGER PRIMARY KEY, v TEXT)") + const txnID = 9400 + require.True(t, prepareLogRow(engine, txnID, "committed").Success) + res := commitLogRow(engine, txnID) + require.True(t, res.Success, res.Error) + + err := mdb.GetTransactionManager().DiscardPrepared(txnID) + require.ErrorIs(t, err, ErrTxnAlreadyCommitted) + var v string + require.NoError(t, mdb.GetReadDB().QueryRow("SELECT v FROM log WHERE id = 1").Scan(&v)) + require.Equal(t, "committed", v, "the discard must leave a committed transaction's row alone") +} + +// pendingInBatch reports whether txnID is queued in bc, not yet taken by a +// flush. +func pendingInBatch(bc *SQLiteBatchCommitter, txnID uint64) bool { + bc.mu.Lock() + defer bc.mu.Unlock() + _, ok := bc.pending[txnID] + return ok +} diff --git a/db/prepared_recovery_test.go b/db/prepared_recovery_test.go index 8019c601..fba7ee40 100644 --- a/db/prepared_recovery_test.go +++ b/db/prepared_recovery_test.go @@ -189,6 +189,77 @@ func TestRecoveredPreparedTransactionCommitsItsRows(t *testing.T) { } } +// TestRecoveredPreparedTransactionLeavesNoIntentOnceCommitted: a transaction +// recovered at restart and then committed - by the coordinator's COMMIT, or +// by the log pull's local commit - holds no intent on its row afterwards. A +// replay of a later transaction on the row refuses on any intent another +// transaction holds (ErrReplayIntentConflict), so a leftover one keeps every +// later change to that row from ever reaching this node. +func TestRecoveredPreparedTransactionLeavesNoIntentOnceCommitted(t *testing.T) { + for _, mode := range prepareSyncModes { + for _, path := range []string{"commit-rpc", "local-commit"} { + t.Run(mode.name+"/"+path, func(t *testing.T) { + const txnID = 9720 + engine, dm, mdb := restartedLogDB(t, mode.env, txnID, "prepared") + if path == "commit-rpc" { + res := commitLogRow(engine, txnID) + require.True(t, res.Success, res.Error) + } else { + require.NoError(t, dm.CommitLocallyPrepared("testdb", txnID, hlc.Timestamp{WallTime: txnID + 1})) + } + intent, err := mdb.GetMetaStore().GetIntent("log", "log:1") + require.NoError(t, err) + require.Nil(t, intent, "the committed recovered transaction left an intent on its row") + }) + } + } +} + +// TestCommitRepairedAtOpenLeavesNoIntent: a transaction committed to SQLite +// (its applied marker written) whose commit record did not survive a crash +// is recovered at open as prepared - its captured rows are still there - and +// then its commit record is repaired from the marker. The repaired commit +// must free what the recovery restored: a leftover intent refuses the replay +// of every later change to the row (ErrReplayIntentConflict), for good. +// +// Mutation: drop the CleanupAfterCommit of repairAppliedTxnMetadata. The +// intent is still held and "left an intent" fires. +func TestCommitRepairedAtOpenLeavesNoIntent(t *testing.T) { + for _, mode := range prepareSyncModes { + t.Run(mode.name, func(t *testing.T) { + const txnID = 9730 + t.Setenv("MARMOT_CDC_PREPARE_SYNC", mode.env) + engine, dm, cleanup := setupTestReplicationEngine(t) + t.Cleanup(cleanup) + markedTableDB(t, engine, dm, "CREATE TABLE log (id INTEGER PRIMARY KEY, v TEXT)") + prep := prepareLogRow(engine, txnID, "committed-before-crash") + require.True(t, prep.Success, "prepare: %s", prep.Error) + + mdb, err := dm.GetDatabase("testdb") + require.NoError(t, err) + // The SQLite commit landed (its marker with it); the commit record + // is what the crash lost. + tx, err := mdb.GetWriteDB().Begin() + require.NoError(t, err) + require.NoError(t, MarkSQLiteTxnApplied(tx, txnID, hlc.Timestamp{WallTime: txnID + 1, NodeID: 1})) + require.NoError(t, tx.Commit()) + + _, dm = restartNode(t, dm) + mdb, err = dm.GetDatabase("testdb") + require.NoError(t, err) + mdb.GetTransactionManager().StopGarbageCollection() + + rec, err := mdb.GetMetaStore().GetTransaction(txnID) + require.NoError(t, err) + require.NotNil(t, rec) + require.Equal(t, TxnStatusCommitted, rec.Status, "the commit record must be repaired from the marker") + intent, err := mdb.GetMetaStore().GetIntent("log", "log:1") + require.NoError(t, err) + require.Nil(t, intent, "the repaired commit left an intent on its row") + }) + } +} + // TestRecoveredPreparedTransactionEndsOnItsAbort: the coordinator's ABORT // for a transaction recovered at restart finds it pending, and ending it frees // its row at once, without waiting for the stale-transaction GC. diff --git a/db/replication_engine.go b/db/replication_engine.go index f1757ee7..9ddbaa9c 100644 --- a/db/replication_engine.go +++ b/db/replication_engine.go @@ -882,6 +882,18 @@ func (dm *DatabaseManager) CommitLocallyPrepared(database string, txnID uint64, return err } +// DiscardLocallyPrepared aborts this node's local prepare of txnID in +// database (TransactionManager.DiscardPrepared). The log puller calls it when +// CommitLocallyPrepared refused with ErrPreparedRowsMissing, and then +// replays the peer's committed copy. +func (dm *DatabaseManager) DiscardLocallyPrepared(database string, txnID uint64) error { + replicatedDB, err := dm.GetDatabase(database) + if err != nil { + return fmt.Errorf("database %s: %w", database, err) + } + return replicatedDB.GetTransactionManager().DiscardPrepared(txnID) +} + // committedLocally reports whether replicatedDB's local record for txnID is // COMMITTED. func committedLocally(replicatedDB *ReplicatedDatabase, txnID uint64) bool { diff --git a/db/transaction.go b/db/transaction.go index c79c1132..a4ed8cd0 100644 --- a/db/transaction.go +++ b/db/transaction.go @@ -475,16 +475,20 @@ func (tm *TransactionManager) CommitTransactionAfter(txn *Transaction, commitTS return fmt.Errorf("transaction %d: %w", txn.ID, ErrPreparedRowsMissing) } - // Statement/DDL path: flush pending DML first to ensure isolation - if tm.batchCommitEnabled() { - tm.batchCommitter.Flush() - } - intents, err := tm.metaStore.GetIntentsByTxn(txn.ID) if err != nil { return fmt.Errorf("failed to fetch write intents: %w", err) } + // Statement/DDL path: flush pending DML first to ensure isolation. A + // claim-only commit writes nothing to this database + // (applyNonDMLIntents), so it must not wait for queued DML: that DML + // may be waiting for the writer an explicit transaction holds while + // that transaction waits for this very claim. + if tm.batchCommitEnabled() && len(classifyNonDMLIntents(intents)) > 0 { + tm.batchCommitter.Flush() + } + // The captured rows are written before the SQLite commit that writes // the marker: a crash after that commit then leaves rows for // repairAppliedTxnMetadata to finish the commit record from at the @@ -1300,6 +1304,25 @@ func (tm *TransactionManager) runGarbageCollection() { } } +// DiscardPrepared aborts txnID's local prepare under its commit guard +// (acquireCommit), releasing its intents. It is for a prepare that can never +// be committed here - its captured rows are gone (ErrPreparedRowsMissing) - +// of a transaction whose committed copy a peer's log serves instead. A +// transaction that a commit ended while this waited is left alone and +// reported by acquireCommit's error. +func (tm *TransactionManager) DiscardPrepared(txnID uint64) error { + release, err := tm.acquireCommit(txnID) + if err != nil { + return err + } + defer release() + txn := tm.GetTransaction(txnID) + if txn == nil { + return fmt.Errorf("transaction %d is not pending", txnID) + } + return tm.AbortTransaction(txn) +} + // cleanupStaleTransactions aborts transactions that haven't had a heartbeat within the timeout // // Each abort runs under the transaction's commit guard (acquireCommit), so it diff --git a/docs/src/pages/operations.mdx b/docs/src/pages/operations.mdx index a0991bcb..0e03491a 100644 --- a/docs/src/pages/operations.mdx +++ b/docs/src/pages/operations.mdx @@ -191,7 +191,8 @@ A node decommissioned through `/admin/cluster/decommission` while ALIVE also rej Ids of `AUTO_INCREMENT` columns narrower than `BIGINT` come from ranges each node claims through a quorum (see Narrow AUTO_INCREMENT Columns in [Compatibility](/compatibility)). They are unique while the membership is stable. -Every node also raises its claim bases to the highest base every ALIVE peer reports, every 10 seconds, which closes +Every node also raises its claim bases to the highest base every ALIVE peer reports, every +`cluster.autoinc_base_sync_interval_ms` (10 seconds by default), which closes the window a membership change opens. To add or remove nodes, change the membership one node at a time: 1. Add or remove **one** node. @@ -205,6 +206,14 @@ the window a membership change opens. To add or remove nodes, change the members votes, and reached every member. A sync gathers only from the peers a node sees ALIVE, so right after a change it can be incomplete; re-run it until it is complete. +**Rebuilding nodes that lost their data directory:** rebuild them one at a time, the same way. A rebuilt +node holds its claim votes until it has merged claim bases from enough members, which is safe only while at most +one member has lost its claim state. Start one rebuilt node, wait until it is ALIVE, its claim votes are released +(`GET /admin/cluster/autoinc/votes` answers `votes_held: false`) and the sync above reports `complete`, and only then +touch the next one. If a majority lost its data at once, the rebuilt nodes can count each other as the whole +cluster and reissue ids until the periodic base sync raises their bases; such an id is refused with 1062, never +stored twice, but inserts through those nodes fail until the sync has run. + The sync endpoint requires `cluster_secret`; on a cluster without one it answers 403. ## Rolling Upgrades diff --git a/grpc/autoinc_merge.go b/grpc/autoinc_merge.go index f5d2f0a4..b96120e1 100644 --- a/grpc/autoinc_merge.go +++ b/grpc/autoinc_merge.go @@ -12,21 +12,27 @@ import ( "github.com/rs/zerolog/log" ) -// autoIncMergeInterval is how often a node whose claim votes are held retries -// the merge that releases them, and autoIncMergePeerTimeout bounds each -// peer's answer within one round. -// -// autoIncBaseSyncInterval is how often an unheld node raises its claim bases -// to the maximum every alive peer reports (the membership backstop). Claim -// safety is proved for a stable membership; after a membership change a -// claim committed under the old membership may be known only to members a -// new quorum can avoid. Every node pulling every peer's bases bounds that -// window to one interval. -const ( - autoIncMergeInterval = 2 * time.Second - autoIncMergePeerTimeout = 5 * time.Second - autoIncBaseSyncInterval = 10 * time.Second -) +// autoIncMergePeerTimeout bounds each peer's answer within one merge round. +const autoIncMergePeerTimeout = 5 * time.Second + +// AutoIncMergeConfig is how a node runs its claim base merge and sync +// (RunAutoIncBaseMerge). +type AutoIncMergeConfig struct { + // Standalone is the node's cluster.standalone setting: only such a node + // releases while it is its own whole membership, and only while it has + // never seen another member (standaloneRelease). + Standalone bool + // MergeInterval is how often a node whose claim votes are held retries + // the merge that releases them. + MergeInterval time.Duration + // BaseSyncInterval is how often an unheld node raises its claim bases to + // the maximum every alive peer reports (the membership backstop). Claim + // safety is proved for a stable membership; after a membership change a + // claim committed under the old membership may be known only to members + // a new quorum can avoid. Every node pulling every peer's bases bounds + // that window to one interval. + BaseSyncInterval time.Duration +} // GetAutoIncBases serves this node's AUTO_INCREMENT claim bases // (db.AutoIncClaimStore.Bases) to a peer's vote-hold merge or base sync. @@ -196,24 +202,20 @@ func peerAutoIncBases(nodeID uint64, client *Client) fetchAutoIncBases { // RunAutoIncBaseMerge releases this node's held claim votes once it has merged // claim bases from enough of its membership (autoIncMergeSafe), retrying -// every autoIncMergeInterval until then. From then on - at once on a node +// every conf.MergeInterval until then. From then on - at once on a node // whose votes were never held - it runs the membership backstop: one sync -// immediately, then one every autoIncBaseSyncInterval, until ctx ends. -// -// standalone is the node's cluster.standalone setting: only such a node -// releases while it is its own whole membership, and only while it has never -// seen another member (standaloneRelease). -func RunAutoIncBaseMerge(ctx context.Context, nodeID uint64, store autoIncBaseStore, registry *NodeRegistry, client *Client, standalone bool) { - runAutoIncBaseMerge(ctx, nodeID, store, registry, standalone, peerAutoIncBases(nodeID, client)) +// immediately, then one every conf.BaseSyncInterval, until ctx ends. +func RunAutoIncBaseMerge(ctx context.Context, nodeID uint64, store autoIncBaseStore, registry *NodeRegistry, client *Client, conf AutoIncMergeConfig) { + runAutoIncBaseMerge(ctx, nodeID, store, registry, conf, peerAutoIncBases(nodeID, client)) } // runAutoIncBaseMerge is RunAutoIncBaseMerge over an explicit membership and // peer fetch. -func runAutoIncBaseMerge(ctx context.Context, nodeID uint64, store autoIncBaseStore, registry autoIncMembership, standalone bool, fetch fetchAutoIncBases) { - if !holdUntilMerged(ctx, nodeID, store, registry, standalone, fetch) { +func runAutoIncBaseMerge(ctx context.Context, nodeID uint64, store autoIncBaseStore, registry autoIncMembership, conf AutoIncMergeConfig, fetch fetchAutoIncBases) { + if !holdUntilMerged(ctx, nodeID, store, registry, conf, fetch) { return } - ticker := time.NewTicker(autoIncBaseSyncInterval) + ticker := time.NewTicker(conf.BaseSyncInterval) defer ticker.Stop() for { if _, err := syncAutoIncBasesOnce(ctx, nodeID, store, registry, fetch); err != nil { @@ -227,15 +229,15 @@ func runAutoIncBaseMerge(ctx context.Context, nodeID uint64, store autoIncBaseSt } } -// holdUntilMerged runs merge rounds every autoIncMergeInterval until this +// holdUntilMerged runs merge rounds every conf.MergeInterval until this // node's votes are released. It reports false when ctx ended first. -func holdUntilMerged(ctx context.Context, nodeID uint64, store autoIncBaseStore, registry autoIncMembership, standalone bool, fetch fetchAutoIncBases) bool { +func holdUntilMerged(ctx context.Context, nodeID uint64, store autoIncBaseStore, registry autoIncMembership, conf AutoIncMergeConfig, fetch fetchAutoIncBases) bool { if held, err := store.AutoIncVotesHeld(); err == nil && !held { return true } - ticker := time.NewTicker(autoIncMergeInterval) + ticker := time.NewTicker(conf.MergeInterval) defer ticker.Stop() - release := standaloneRelease{configured: standalone} + release := standaloneRelease{configured: conf.Standalone} warned := false for { membership := registry.Count() @@ -249,7 +251,7 @@ func holdUntilMerged(ctx context.Context, nodeID uint64, store autoIncBaseStore, } if !warned && err == nil { warned = true - log.Warn().Int("membership", membership).Bool("standalone", standalone). + log.Warn().Int("membership", membership).Bool("standalone", conf.Standalone). Msg("AUTO_INCREMENT claim votes held until claim bases are merged from enough cluster members; " + "narrow AUTO_INCREMENT inserts may return 1205 until then. A single-node deployment sets " + "cluster.standalone = true; an operator can force a release with " + diff --git a/grpc/autoinc_merge_test.go b/grpc/autoinc_merge_test.go index 79487a2e..4ceb302d 100644 --- a/grpc/autoinc_merge_test.go +++ b/grpc/autoinc_merge_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/maxpert/marmot/cfg" "github.com/maxpert/marmot/db" "github.com/maxpert/marmot/hlc" "github.com/stretchr/testify/require" @@ -283,6 +284,12 @@ func TestSyncAutoIncBasesRaisesFromEveryAlivePeer(t *testing.T) { require.False(t, report.ReachedEveryMember()) } +// testMergeConfig is the production default merge configuration. +var testMergeConfig = AutoIncMergeConfig{ + MergeInterval: cfg.DefaultAutoIncMergeIntervalMS * time.Millisecond, + BaseSyncInterval: cfg.DefaultAutoIncBaseSyncIntervalMS * time.Millisecond, +} + // TestReleasedNodeSyncsAtOnce pins that a node whose votes are released runs // the backstop immediately, not an interval later: a node that just merged // is exactly one whose membership just changed. @@ -299,7 +306,7 @@ func TestReleasedNodeSyncsAtOnce(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { - runAutoIncBaseMerge(ctx, 1, store, fakeMembership{members: []uint64{1, 2, 3}}, false, peerAnswers(answers)) + runAutoIncBaseMerge(ctx, 1, store, fakeMembership{members: []uint64{1, 2, 3}}, testMergeConfig, peerAnswers(answers)) close(done) }() defer func() { @@ -307,7 +314,7 @@ func TestReleasedNodeSyncsAtOnce(t *testing.T) { <-done }() - deadline := time.Now().Add(autoIncBaseSyncInterval / 2) + deadline := time.Now().Add(testMergeConfig.BaseSyncInterval / 2) for store.raiseCount() == 0 { if time.Now().After(deadline) { t.Fatal("a released node did not sync at once") diff --git a/grpc/log_puller.go b/grpc/log_puller.go index 115f1cb9..01a43e3b 100644 --- a/grpc/log_puller.go +++ b/grpc/log_puller.go @@ -432,7 +432,11 @@ func (lp *LogPuller) applyPage(ctx context.Context, st *pullState, entries []*Lo // - no PENDING record: fetch; // - durably prepared: commit it through the local commit path // (db.DatabaseManager.CommitLocallyPrepared) and mark it covered; on -// failure leave it for the next call, never replaying over it; +// failure leave it for the next call, never replaying over it - except +// when its captured rows are gone (db.ErrPreparedRowsMissing, what a +// crash leaves when the rows were not yet synced): that prepare can never +// commit, so discard it (db.DatabaseManager.DiscardLocallyPrepared) and +// fetch the peer's committed copy; // - begun but abandoned (its PREPARE died with an earlier process): it // promised nothing, so discard it and fetch; // - begun and live (a PREPARE executing now): leave it for the next call. @@ -450,6 +454,20 @@ func (lp *LogPuller) resolveLocalPending(st *pullState, e *LogEntry, covered *bo return false, nil } if err := lp.dbMgr.CommitLocallyPrepared(st.database, txnID, HLCToTimestamp(e.CommitTimestamp)); err != nil { + if errors.Is(err, db.ErrPreparedRowsMissing) { + err := lp.dbMgr.DiscardLocallyPrepared(st.database, txnID) + switch { + case errors.Is(err, db.ErrTxnAlreadyCommitted): + // Another path (a pull of another peer's copy) committed + // it in between: it is covered. + *covered = true + lp.clearAttempt(st.database, st.peer.NodeID, txnID) + return false, nil + case err != nil: + return false, fmt.Errorf("discard local prepare of txn %d whose rows are gone: %w", txnID, err) + } + return true, nil + } st.localCommitFailed = true // A busy writer (a pinned session holding it, say) is not this // transaction's fault and must not push it toward STUCK, whose diff --git a/grpc/log_puller_gc_race_test.go b/grpc/log_puller_gc_race_test.go index 82571b39..7d0302dc 100644 --- a/grpc/log_puller_gc_race_test.go +++ b/grpc/log_puller_gc_race_test.go @@ -15,11 +15,12 @@ import ( // deletes a prepared transaction's intents and captured rows, and the log // puller then reaches that transaction in a peer's log. Its local commit must // be refused - committing it would record it COMMITTED with no rows, and the -// cursor would pass it for good - so the cursor stays before it; once the -// transaction is aborted, the next pull replays it from the peer. +// cursor would pass it for good - so the puller discards the local prepare +// and replays the peer's committed copy in the same call. // // Mutation: make TransactionManager.verifyPreparedPayload return nil. The -// first pull passes the cursor over txn 2000 and row 2 is never applied. +// pull commits txn 2000 locally with no rows, passes the cursor over it, and +// row 2 is never applied. func TestLogPuller_PreparedPayloadGoneNeverPassesTheCursor(t *testing.T) { peer := newPullTestNode(t, 2, "app") peer.serve(t, nil) @@ -51,21 +52,12 @@ func TestLogPuller_PreparedPayloadGoneNeverPassesTheCursor(t *testing.T) { peerRef := PeerRef{NodeID: 2, Address: peer.addr} res, err := lp.PullPair(context.Background(), peerRef, "app") require.NoError(t, err) - require.False(t, res.CaughtUp) - - rec, err := ms.GetTransaction(2000) - require.NoError(t, err) - require.NotNil(t, rec) - require.Equal(t, db.TxnStatusPending, rec.Status, "a txn whose prepared payload is gone must not be committed") - cursor, err := ms.GetPullCursor(2) - require.NoError(t, err) - require.Equal(t, uint64(1000), cursor.TxnID, "the cursor must stay before the refused txn") - - require.NoError(t, ms.AbortTransaction(2000)) - res, err = lp.PullPair(context.Background(), peerRef, "app") - require.NoError(t, err) require.True(t, res.CaughtUp) - require.Equal(t, map[int64]string{1: "a", 2: "b", 3: "c"}, local.rows(t, "app")) + require.Zero(t, res.Stuck) + require.Equal(t, map[int64]string{1: "a", 2: "b", 3: "c"}, local.rows(t, "app"), "row 2 must come from the peer's committed copy") + applied, err := mdb.AppliedTxns([]uint64{2000}) + require.NoError(t, err) + require.True(t, applied[2000]) } // fetchHookServer serves FetchTransactions like the real server, running diff --git a/grpc/log_puller_test.go b/grpc/log_puller_test.go index e33a301e..e53ba0a7 100644 --- a/grpc/log_puller_test.go +++ b/grpc/log_puller_test.go @@ -204,6 +204,64 @@ func TestLogPuller_PreparedLocalTxnCommittedLocallyInSamePull(t *testing.T) { require.Equal(t, 1, countLocalLogEntries(t, mdb, 2000), "the late COMMIT must not log the txn twice") } +// TestLogPuller_PreparedLocalTxnWithLostRowsReplayedFromPeer pins the case a +// SIGKILL leaves behind: a txn durably prepared here whose captured rows did +// not survive (they are synced after the prepare record unless +// strict_prepare_sync is set). The peer's log proves it committed, but it +// cannot be committed locally: that would commit nothing where its prepare +// recorded a row. The pull must discard the local prepare and replay the +// peer's committed copy in the same call - never leave it STUCK, which kept +// the node JOINING, and never skip it. +// +// Mutation: leave a failed local commit for the next call (the old +// behaviour). The txn is never covered: "must not be stuck" fires. +func TestLogPuller_PreparedLocalTxnWithLostRowsReplayedFromPeer(t *testing.T) { + peer := newPullTestNode(t, 2, "app") + peer.serve(t, nil) + local := newPullTestNode(t, 1, "app") + + peer.seed(t, "app", 1000, 2, 1000, 1, "a") + peer.seed(t, "app", 2000, 2, 2000, 2, "b") + peer.seed(t, "app", 3000, 2, 3000, 3, "c") + + mdb, err := local.dm.GetDatabase("app") + require.NoError(t, err) + rh := NewReplicationHandler(1, local.dm, hlc.NewClock(1), db.NewSchemaVersionManager(local.dm)) + resp, err := rh.HandleReplicateTransaction(context.Background(), &TransactionRequest{ + TxnId: 2000, SourceNodeId: 2, Database: "app", Phase: TransactionPhase_PREPARE, + Timestamp: &HLC{WallTime: 2000, NodeId: 2}, RequiredSchemaVersion: mdb.SchemaVersion(), + Statements: []*Statement{{Type: pb.StatementType_INSERT, TableName: "t", Database: "app", + Payload: &Statement_RowChange{RowChange: testInsertRowChange("t", []byte("t:2"), map[string][]byte{ + "id": mustMarshalMsgpack(t, int64(2)), "v": mustMarshalMsgpack(t, "b"), + })}}}, + }) + require.NoError(t, err) + require.True(t, resp.Success, resp.ErrorMessage) + // The captured rows are lost; the durable prepare record still claims one. + require.NoError(t, mdb.GetMetaStore().DeleteIntentEntries(2000)) + + lp := NewLogPuller(LogPullerConfig{NodeID: 1, Client: NewClient(1), DBManager: local.dm}) + for round := 0; round <= DefaultMaxReplayAttempts; round++ { + res, err := lp.PullPair(context.Background(), PeerRef{NodeID: 2, Address: peer.addr}, "app") + require.NoError(t, err) + require.Zero(t, res.Stuck, "the txn must not be stuck (round %d)", round) + if res.CaughtUp { + break + } + } + require.Equal(t, map[int64]string{1: "a", 2: "b", 3: "c"}, local.rows(t, "app"), "the peer's committed copy must be replayed") + + applied, err := mdb.AppliedTxns([]uint64{2000}) + require.NoError(t, err) + require.True(t, applied[2000], "the replay must leave the applied marker") + kind, err := mdb.GetMetaStore().ClassifyPending(2000) + require.NoError(t, err) + require.Equal(t, db.PendingNone, kind, "the local prepare must be gone") + cursor, err := mdb.GetMetaStore().GetPullCursor(2) + require.NoError(t, err) + require.Equal(t, uint64(3000), cursor.TxnID) +} + // countLocalLogEntries counts mdb's local log entries for txnID. func countLocalLogEntries(t *testing.T, mdb *db.ReplicatedDatabase, txnID uint64) int { t.Helper() diff --git a/marmot.go b/marmot.go index 8b38683d..43932f13 100644 --- a/marmot.go +++ b/marmot.go @@ -429,7 +429,11 @@ func main() { // interval. autoIncMergeCtx, autoIncMergeCancel := context.WithCancel(context.Background()) defer autoIncMergeCancel() - go marmotgrpc.RunAutoIncBaseMerge(autoIncMergeCtx, cfg.Config.NodeID, dbMgr, grpcServer.GetNodeRegistry(), client, cfg.Config.Cluster.Standalone) + go marmotgrpc.RunAutoIncBaseMerge(autoIncMergeCtx, cfg.Config.NodeID, dbMgr, grpcServer.GetNodeRegistry(), client, marmotgrpc.AutoIncMergeConfig{ + Standalone: cfg.Config.Cluster.Standalone, + MergeInterval: cfg.Config.Cluster.GetAutoIncMergeInterval(), + BaseSyncInterval: cfg.Config.Cluster.GetAutoIncBaseSyncInterval(), + }) // Phase 7: Setup transaction coordinators for full database replication log.Info().Msg("Setting up transaction coordinators") diff --git a/protocol/error_mapper.go b/protocol/error_mapper.go index cbc41586..52e2fbc8 100644 --- a/protocol/error_mapper.go +++ b/protocol/error_mapper.go @@ -1,6 +1,7 @@ package protocol import ( + "database/sql" "errors" "strings" @@ -25,6 +26,15 @@ func ConvertToMySQLError(err error) *MySQLError { return NewMySQLError(coded.Code, sqlStateForCode(coded.Code), coded.Message) } + // The statement's SQLite transaction has already ended: an explicit + // transaction's pinned session that outlived the lock wait was rolled + // back. MySQL answers a rolled-back transaction with 1213, which tells the + // client to run the whole transaction again. + if errors.Is(err, sql.ErrTxDone) { + return NewMySQLError(ErrCodeDeadlock, SQLStateDeadlock, + "Deadlock found when trying to get lock; try restarting transaction ("+err.Error()+")") + } + // Try to extract sqlite3.Error var sqliteErr sqlite3.Error if errors.As(err, &sqliteErr) { diff --git a/protocol/error_mapper_test.go b/protocol/error_mapper_test.go index e8cb25cb..e2734ecd 100644 --- a/protocol/error_mapper_test.go +++ b/protocol/error_mapper_test.go @@ -1,6 +1,7 @@ package protocol import ( + "database/sql" "errors" "fmt" "testing" @@ -286,6 +287,19 @@ func TestConvertToMySQLError_GenericConstraint(t *testing.T) { } } +// TestConvertToMySQLError_EndedTransactionIsRetryableDeadlock: a statement of +// an explicit transaction whose SQLite transaction already ended (its pinned +// session outlived the lock wait and was rolled back) is answered with 1213, +// MySQL's "the transaction was rolled back, run it again", never the +// unknown error 1105 a client would not retry. +func TestConvertToMySQLError_EndedTransactionIsRetryableDeadlock(t *testing.T) { + err := fmt.Errorf("DML execution failed: failed to execute statement: %w", sql.ErrTxDone) + result := ConvertToMySQLError(err) + if result.Code != ErrCodeDeadlock || result.SQLState != SQLStateDeadlock { + t.Fatalf("got %d/%s (%s), want %d/%s", result.Code, result.SQLState, result.Message, ErrCodeDeadlock, SQLStateDeadlock) + } +} + func TestConvertToMySQLError_WrappedError(t *testing.T) { // Test that errors.As works with wrapped errors sqliteErr := sqlite3.Error{ From ca2ca4a9609babf556b449902a2f3af4200d2099 Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Tue, 29 Sep 2026 16:05:21 -0500 Subject: [PATCH 10/17] test(cluster): rewrite the integration suite to run in under two minutes The cluster suite took about 11.5 minutes and had flaky tests. It is rebuilt on one harness: - The binaries are built once per run in TestMain. Each test gets its own ports and directory, and cleanup kills its nodes and prints log tails on failure. - Readiness means MySQL answers, every node sees the others ALIVE, and no node holds its claim votes; the votes are read over gRPC, including from the old-release node in the rolling upgrade. This removes that test's flake. - Fixed sleeps are replaced by polled conditions with deadlines. The only time.Sleep left is inside the poll helper. - Crash tests still use real SIGKILL and SIGTERM, with short intervals. - Assertions check exact row sets, values, ids and MySQL error codes. Retries are typed, use jittered backoff, and never mask the defect a test guards. The 23 tests that took more than 10s are rewritten from scratch. Each test that guards a fix fails when that fix is reverted. 76 tests pass in about 108s, and none takes more than 10s. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- test/autoinc_backstop_cluster_test.go | 141 +- test/autoinc_claim_cluster_test.go | 531 ++---- test/autoinc_claim_inprocess_test.go | 49 +- test/autoinc_lifecycle_cluster_test.go | 378 +--- test/autoinc_membership_cluster_test.go | 550 ++---- test/autoinc_membership_inprocess_test.go | 4 +- test/autoinc_narrow_cluster_test.go | 472 ++--- test/cluster_integration_test.go | 647 +------ test/crash_recovery_test.go | 1591 +---------------- test/ddl_replication_test.go | 185 +- test/dml_operations_test.go | 561 +----- test/gap_detection_test.go | 132 +- test/graceful_restart_cluster_test.go | 96 +- test/harness_test.go | 995 +++++++++++ test/information_schema_cluster_test.go | 36 +- test/main_test.go | 117 ++ test/node_outage_convergence_cluster_test.go | 1087 ++++------- test/partition_test.go | 13 +- test/quorum_of_one_test.go | 190 +- test/rolling_upgrade_cluster_test.go | 258 +-- test/row_version_convergence_cluster_test.go | 174 +- test/schema_version_kill_cluster_test.go | 45 +- test/setup_test.go | 28 - test/verification_test.go | 23 +- test/write_ledger_convergence_cluster_test.go | 358 ++-- 25 files changed, 2789 insertions(+), 5872 deletions(-) create mode 100644 test/harness_test.go create mode 100644 test/main_test.go delete mode 100644 test/setup_test.go diff --git a/test/autoinc_backstop_cluster_test.go b/test/autoinc_backstop_cluster_test.go index 68ad57c2..26e81655 100644 --- a/test/autoinc_backstop_cluster_test.go +++ b/test/autoinc_backstop_cluster_test.go @@ -2,131 +2,88 @@ package test import ( "fmt" - "os" - "strings" + "net/http" "sync" "sync/atomic" "testing" "time" ) -// Log lines a claim COMMIT that could not be applied leaves: the participant's -// refusal (db ReplicationEngine.Commit) and, on the claimant, a local commit -// that failed after the remote quorum committed (coordinator -// commitLocalAfterRemoteQuorum). -const ( - claimCommitRefusedLog = "COMMIT REFUSED: auto-increment claim could not be applied" - localCommitFailedLog = "Local commit failed after remote quorum achieved" -) +// backstopLoad is how long TestNarrowAutoInc_ClaimsCommitUnderConstantBaseSync +// inserts while the sync command runs back to back. +const backstopLoad = 3 * time.Second -// TestNarrowAutoInc_ClaimsCommitUnderConstantBaseSync amplifies the base -// sync without touching the backstop's interval: while every -// node inserts into two SMALLINT tables (a claim every 31 ids), the test runs -// the operator's sync command back to back on every node, so each node pulls -// its peers' bases many times a second. A peer that committed a claim first -// reports exactly its end; a sync landing between this node's PREPARE and -// COMMIT of that claim used to raise the stored base past the claim's newBase -// and refuse the COMMIT, leaving the claim key locked until the -// pending-transaction GC. Every claim COMMIT must now apply, and no id may be -// acknowledged twice. +// TestNarrowAutoInc_ClaimsCommitUnderConstantBaseSync amplifies the base sync: +// while every node inserts into two SMALLINT tables (a claim every 31 ids), +// the operator's sync command runs back to back on every node, so each node +// pulls its peers' bases many times a second. A sync landing between a +// claim's PREPARE and COMMIT used to raise the stored base past the claim's +// newBase and refuse the COMMIT, leaving the claim key locked until the +// pending-transaction GC: that node's inserts then stall on 1205 far past +// claimRetryDeadline. Every insert must succeed, no id may be issued twice, +// and every node must end with exactly the rows acknowledged. // -// Mutation: test the merged floor in applyClaimTx's condition (or write merges -// to the committed floor). "claim COMMITs were refused" fires. +// The interleaving itself is pinned deterministically by +// db/autoinc_floors_test.go TestCommitAppliesAClaimAMergeRaiseOvertook and +// TestCommitAppliesAClaimAMergeRaisePassed. func TestNarrowAutoInc_ClaimsCommitUnderConstantBaseSync(t *testing.T) { tables := []string{"bs1", "bs2"} - harness := startNarrowCluster(t, tables[0], "CREATE TABLE "+tables[0]+" (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - defer harness.Cleanup() - if _, err := execTimed(openTimedNodeDatabase(t, harness, 1, "marmot"), - "CREATE TABLE "+tables[1]+" (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)"); err != nil { - t.Fatalf("CREATE TABLE %s: %v", tables[1], err) - } - if err := harness.WaitForTableExists(tables[1], []int{1, 2, 3}, 30*time.Second); err != nil { - t.Fatal(err) - } - harness.Phase("start") + c := startNarrowCluster(t, tables[0], "CREATE TABLE "+tables[0]+" (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + c.createTable(1, "marmot", tables[1], "CREATE TABLE "+tables[1]+" (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - // 35s (not 45s): amplifies the backstop enough (100+ syncs) while keeping - // the whole test under a 55s budget. - const load = 35 * time.Second - stop := time.Now().Add(load) - var syncs, syncErrs atomic.Int64 + stop := time.Now().Add(backstopLoad) + var syncs atomic.Int64 + var mu sync.Mutex + ledgers := map[string]map[int64]string{tables[0]: {}, tables[1]: {}} + errs := make(chan error, 6) var wg sync.WaitGroup for n := 1; n <= 3; n++ { - wg.Add(1) + wg.Add(2) go func(n int) { defer wg.Done() for time.Now().Before(stop) { - if _, err := runAutoIncSync(harness, n); err != nil { - syncErrs.Add(1) - time.Sleep(100 * time.Millisecond) - continue + var answer autoIncSyncAnswer + if err := c.admin(n, http.MethodPost, "/cluster/autoinc/sync", syncTimeout, &answer); err != nil { + errs <- fmt.Errorf("sync on node %d: %w", n, err) + return } syncs.Add(1) } }(n) - } - - var mu sync.Mutex - ledger := map[string]string{} - var dups []string - inserted := make([]int, 4) - failures := make([]error, 4) - for n := 1; n <= 3; n++ { - wg.Add(1) go func(n int) { defer wg.Done() - conn := openTimedNodeDatabase(t, harness, n, "marmot") for i := 0; time.Now().Before(stop); i++ { - table := tables[i%len(tables)] - who := fmt.Sprintf("node %d #%d", n, i) - id, err := insertNarrowWithin(conn, 30*time.Second, "INSERT INTO "+table+" (v) VALUES (?)", who) - mu.Lock() + table, who := tables[i%len(tables)], fmt.Sprintf("node %d #%d", n, i) + id, err := insertNarrow(c, n, "marmot", "INSERT INTO "+table+" (v) VALUES (?)", who) if err != nil { - failures[n] = err - mu.Unlock() + errs <- fmt.Errorf("%s: %w", who, err) return } - inserted[n]++ - key := fmt.Sprintf("%s/%d", table, id) - if prev, dup := ledger[key]; dup { - dups = append(dups, fmt.Sprintf("%s: %s and %s", key, prev, who)) - } - ledger[key] = who + mu.Lock() + prev, dup := ledgers[table][id] + ledgers[table][id] = who mu.Unlock() + if dup { + errs <- fmt.Errorf("DUPLICATE id %s/%d: %s and %s", table, id, prev, who) + return + } } }(n) } wg.Wait() - harness.Phase("load-stopped") - - t.Logf("syncs=%d sync errors=%d inserted per node=%v ids=%d", syncs.Load(), syncErrs.Load(), inserted[1:], len(ledger)) - if len(dups) > 0 { - t.Fatalf("DUPLICATE ids: %v", dups) - } - var refusals []string - for n := 1; n <= 3; n++ { - logText, err := os.ReadFile(harness.Nodes[n-1].LogFile) - if err != nil { - t.Fatalf("read node %d log: %v", n, err) - } - refused := strings.Count(string(logText), claimCommitRefusedLog) - failed := strings.Count(string(logText), localCommitFailedLog) - t.Logf("node %d: commit-refused=%d local-commit-failed=%d", n, refused, failed) - if refused+failed > 0 { - refusals = append(refusals, fmt.Sprintf("node %d: %d refused, %d local commits failed", n, refused, failed)) - } + close(errs) + for err := range errs { + t.Error(err) } - if len(refusals) > 0 { - harness.dumpNodeLogs("base_sync_refusals") - t.Fatalf("claim COMMITs were refused under constant base syncs: %v", refusals) + if t.Failed() { + t.FailNow() } - for n := 1; n <= 3; n++ { - if failures[n] != nil { - harness.dumpNodeLogs("base_sync_inserts") - t.Fatalf("node %d stopped inserting after %d rows: %v", n, inserted[n], failures[n]) - } + if syncs.Load() < 1000 { + t.Fatalf("only %d syncs ran in %s; the test did not amplify the backstop", syncs.Load(), backstopLoad) } - if syncs.Load() < 100 { - t.Fatalf("only %d syncs ran in %s; the test did not amplify the backstop", syncs.Load(), load) + for _, table := range tables { + ledger := &idLedger{t: t, seen: ledgers[table]} + c.waitRows("marmot", idStats(table, "id"), ledger.stats(), 1, 2, 3) } + t.Logf("%d syncs, %d + %d rows", syncs.Load(), len(ledgers[tables[0]]), len(ledgers[tables[1]])) } diff --git a/test/autoinc_claim_cluster_test.go b/test/autoinc_claim_cluster_test.go index d42c4d7d..20993ee9 100644 --- a/test/autoinc_claim_cluster_test.go +++ b/test/autoinc_claim_cluster_test.go @@ -1,483 +1,198 @@ package test // Cluster tests for the AUTO_INCREMENT feature's DDL-time seeding -// (coordinator.ClaimRange, db.AutoIncClaimStore, db/autoinc_seed.go) and for -// the hidden claim table's invisibility and write-protection, all driven -// over the real MySQL wire protocol against a 3-node ClusterHarness -// (test/crash_recovery_test.go). -// -// The claim protocol's own scenarios (sequential claim safety, concurrent -// boundary claims, restart and crash recovery, meta-store wipe, full-cluster -// snapshot restore, membership growth) are covered in -// test/autoinc_claim_inprocess_test.go and -// test/autoinc_membership_inprocess_test.go, which drive the real coordinator -// and participant engines in-process, replacing only the gRPC transport, so -// they can script which node sees which round. Inserts that reach -// ClaimRange over the MySQL wire protocol are covered in -// test/autoinc_narrow_cluster_test.go. +// (db/autoinc_seed.go) and for the hidden claim table's invisibility and +// write-protection, driven over the MySQL wire protocol against a 3-node +// cluster. The claim protocol's own scenarios are covered in-process by +// autoinc_claim_inprocess_test.go and autoinc_membership_inprocess_test.go; +// inserts that claim over the wire by autoinc_narrow_cluster_test.go. import ( - "database/sql" "errors" "fmt" - "path/filepath" "strings" "testing" - "time" "github.com/go-sql-driver/mysql" - _ "github.com/mattn/go-sqlite3" "github.com/maxpert/marmot/common" "github.com/maxpert/marmot/db" "github.com/maxpert/marmot/protocol/mysqlcode" ) -// systemDBFile returns the on-disk path of a node's system database, the -// file db.SystemDatabaseName resolves to under its data directory -// (db/database_manager.go: filepath.Join(dataDir, SystemDatabaseName+".db")). -// __marmot__autoinc lives inside this file, never inside a user database -// (db/autoinc_claim.go's AutoIncClaimTable doc). -func systemDBFile(dataDir string) string { - return filepath.Join(dataDir, db.SystemDatabaseName+".db") -} - -// readAutoIncBase opens a node's system database file directly, outside the -// MySQL wire protocol, and reads the base for (database, table) - the -// largest of its committed, seed and merged floors - from __marmot__autoinc. This is the only channel available to a cluster -// test: the table is deliberately hidden from client SQL (see -// common.IsInternalTableName and protocol/query/rules/claim_table_guard.go), -// and nothing on the wire path reveals it. found is false, with no error, -// when the row is absent - not yet seeded, or the table itself does not yet -// exist - matching db.AutoIncClaimStore.ReadBase's own "absent row, absent -// table" equivalence. -func readAutoIncBase(dataDir, database, table string) (base uint64, found bool, err error) { - dsn := fmt.Sprintf("file:%s?mode=ro&_busy_timeout=5000", systemDBFile(dataDir)) - conn, openErr := sql.Open("sqlite3", dsn) - if openErr != nil { - return 0, false, fmt.Errorf("open system db under %s: %w", dataDir, openErr) - } - defer conn.Close() - - var got int64 - scanErr := conn.QueryRow( - "SELECT MAX(committed, seed, merged) FROM "+common.AutoIncClaimTableName+" WHERE db = ? AND tbl = ?", - database, table).Scan(&got) - switch { - case scanErr == sql.ErrNoRows: - return 0, false, nil - case scanErr != nil && strings.Contains(scanErr.Error(), "no such table"): - return 0, false, nil - case scanErr != nil: - return 0, false, fmt.Errorf("read %s under %s: %w", common.AutoIncClaimTableName, dataDir, scanErr) - } - return uint64(got), true, nil -} - -// waitForAutoIncBaseAtLeast polls a node's system database until the -// committed base for (database, table) is >= min, or timeout elapses. DDL -// seeding commits the claim row atomically with the DDL's own COMMIT, before -// that COMMIT is ACKed (db/autoinc_seed.go, db/autoinc_claim.go Seed), so by -// the time WaitForTableExists reports a table visible on a node the seed row -// should already be durable there; this still polls rather than reading -// once, so the assertion is not coupled to that timing guarantee holding -// exactly. -func waitForAutoIncBaseAtLeast(t *testing.T, dataDir, database, table string, min uint64, timeout time.Duration) uint64 { +// asMySQLError returns err as a *mysql.MySQLError, failing the test if err +// is nil or another kind of error: a transport error is a different failure +// than the coded refusal these tests assert. +func asMySQLError(t *testing.T, err error, what string) *mysql.MySQLError { t.Helper() - deadline := time.Now().Add(timeout) - var base uint64 - var found bool - var lastErr error - for time.Now().Before(deadline) { - base, found, lastErr = readAutoIncBase(dataDir, database, table) - if lastErr == nil && found && base >= min { - return base - } - time.Sleep(200 * time.Millisecond) - } - switch { - case lastErr != nil: - t.Fatalf("reading auto-increment base for %s.%s under %s: %v", database, table, dataDir, lastErr) - case !found: - t.Fatalf("no auto-increment claim row for %s.%s under %s after %v", database, table, dataDir, timeout) - default: - t.Fatalf("auto-increment base for %s.%s under %s = %d after %v, want >= %d", - database, table, dataDir, base, timeout, min) + var mysqlErr *mysql.MySQLError + if !errors.As(err, &mysqlErr) { + t.Fatalf("%s: err = %v, want a MySQL error", what, err) } - return 0 + return mysqlErr } -// queryStringColumn runs query against db and returns every row's value for -// the named column (matched case-insensitively, since MySQL identifiers are -// case-insensitive), scanning generically so it works regardless of how many -// other columns the result set carries - information_schema.tables returns a -// fixed wide row shape unrelated to the client's SELECT list. -func queryStringColumn(conn *sql.DB, query, colName string, args ...interface{}) ([]string, error) { - rows, err := conn.Query(query, args...) +// columnValues runs query through node id and returns every row's value of +// the named column (matched case-insensitively, as MySQL identifiers are). +func columnValues(c *cluster, id int, query, column string) ([]string, error) { + rs, err := c.db(id, db.DefaultDatabaseName).Query(query) if err != nil { return nil, err } - defer rows.Close() - - cols, err := rows.Columns() + defer rs.Close() + cols, err := rs.Columns() if err != nil { return nil, err } idx := -1 - for i, c := range cols { - if strings.EqualFold(c, colName) { + for i, name := range cols { + if strings.EqualFold(name, column) { idx = i - break } } - if idx == -1 { - return nil, fmt.Errorf("column %q not found in result columns %v", colName, cols) + if idx < 0 { + return nil, fmt.Errorf("column %q not in %v", column, cols) } - var out []string - for rows.Next() { - dest := make([]interface{}, len(cols)) - raw := make([]sql.NullString, len(cols)) - for i := range dest { - dest[i] = &raw[i] + for rs.Next() { + raw := make([]any, len(cols)) + vals := make([][]byte, len(cols)) + for i := range raw { + raw[i] = &vals[i] } - if err := rows.Scan(dest...); err != nil { + if err := rs.Scan(raw...); err != nil { return nil, err } - out = append(out, raw[idx].String) + out = append(out, string(vals[idx])) } - return out, rows.Err() + return out, rs.Err() } -// asMySQLError extracts *mysql.MySQLError from err via errors.As, failing -// the test if err is not one - a raw driver/transport error is a different -// failure than the coded rejection these tests assert on. -func asMySQLError(t *testing.T, err error, context string) *mysql.MySQLError { - t.Helper() - if err == nil { - t.Fatalf("%s: expected an error, got nil", context) - } - var mysqlErr *mysql.MySQLError - if !errors.As(err, &mysqlErr) { - t.Fatalf("%s: error %v is not a *mysql.MySQLError", context, err) - } - return mysqlErr -} - -// =========================================================================== -// Item 6: DDL-time seeding -// =========================================================================== - -// TestAutoIncSeed_CreateTableWithFloorOption exercises the shape DDL-time -// seeding actually supports: ALTER ... MODIFY/CHANGE to retroactively tag an -// existing BIGINT column AUTO_INCREMENT cannot land, because vitess emits -// MySQL ALTER syntax SQLite cannot apply. CREATE TABLE carrying a -// table-level AUTO_INCREMENT=N option is unaffected by that limitation and -// is the shape production DDL actually uses, so it is what this test -// drives: CREATE TABLE ... AUTO_INCREMENT=5000 must seed base >= 4999 (N-1: -// protocol/query/transform/table_utils.go autoIncFloorFromOptions) on every -// node, since each node seeds from its own DDL apply independently -// (db/autoinc_seed.go). +// TestAutoIncSeed_CreateTableWithFloorOption: CREATE TABLE ... +// AUTO_INCREMENT=5000 seeds a claim base of at least 4999 on every node, +// each from its own DDL apply. func TestAutoIncSeed_CreateTableWithFloorOption(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("StartCluster: %v", err) - } - + c := newCluster(t) + c.start() const table = "autoinc_seed_floor" - allNodes := []int{1, 2, 3} - - if _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT) AUTO_INCREMENT=5000", table)); err != nil { - t.Fatalf("CREATE TABLE ... AUTO_INCREMENT=5000: %v", err) - } - if err := harness.WaitForTableExists(table, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("autoinc_seed_floor_ddl") - t.Fatalf("DDL did not replicate: %v", err) - } - - for _, nodeID := range allNodes { - dataDir := harness.Nodes[nodeID-1].DataDir - base := waitForAutoIncBaseAtLeast(t, dataDir, db.DefaultDatabaseName, table, 4999, 15*time.Second) - t.Logf("node %d: seeded base = %d (want >= 4999)", nodeID, base) - } + c.createTable(1, "marmot", table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT) AUTO_INCREMENT=5000") + c.waitAutoIncBaseAtLeast("marmot", table, 4999, 1, 2, 3) } -// TestAutoIncSeed_AlterSeedsFromExistingMax: a table filled with explicit -// ids 1..rowCount holds a base at or above rowCount on every node, both -// before and after an unrelated ALTER TABLE ADD COLUMN re-triggers DDL-time -// seeding. The explicit ids raise the base as they are inserted (the range -// allocator observes every explicit id), and the ALTER's seed from MAX(id) -// can only raise it further. That the seed itself derives from MAX(id) is -// pinned without the allocator in the way by -// db/autoinc_seed_test.go TestSeedAutoIncBasesForDDL_ExistingRowsRaiseTheFloor. +// TestAutoIncSeed_AlterSeedsFromExistingMax: explicit ids 1..1000 raise the +// claim base to at least 1000 as they are inserted, and an unrelated ALTER +// TABLE, which seeds again from MAX(id), leaves it there on every node. func TestAutoIncSeed_AlterSeedsFromExistingMax(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("StartCluster: %v", err) - } - - const table = "autoinc_seed_altermax" - const rowCount = 1000 - const batchSize = 100 - allNodes := []int{1, 2, 3} - - if _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT AUTO_INCREMENT PRIMARY KEY, v VARCHAR(20))", table)); err != nil { - t.Fatalf("CREATE TABLE: %v", err) - } - if err := harness.WaitForTableExists(table, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("autoinc_seed_altermax_ddl") - t.Fatalf("DDL did not replicate: %v", err) - } - - for start := 1; start <= rowCount; start += batchSize { - var values strings.Builder - for i := start; i < start+batchSize; i++ { - if i > start { - values.WriteString(", ") - } - fmt.Fprintf(&values, "(%d, 'row_%d')", i, i) - } - if _, err := harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, v) VALUES %s", table, values.String())); err != nil { - t.Fatalf("explicit-id batch insert starting at %d: %v", start, err) - } - } - if err := harness.WaitForRowCount(table, allNodes, rowCount, 15*time.Second); err != nil { - t.Fatalf("explicit-id rows did not replicate: %v", err) - } - - // Before the ALTER: explicit ids above the base raise it through the - // claim protocol as they are inserted (id.RangeAllocator.Observe), so no - // node can later issue one of them. - preDataDir := harness.Nodes[0].DataDir - preBase, found, err := readAutoIncBase(preDataDir, db.DefaultDatabaseName, table) - if err != nil { - t.Fatalf("read pre-ALTER base: %v", err) - } - if !found { - t.Fatalf("no claim row for %s before ALTER", table) - } - if preBase < rowCount { - t.Fatalf("pre-ALTER base = %d, want >= %d: explicit ids must raise the base as they are inserted", preBase, rowCount) - } - - if _, err := harness.ExecNode(1, fmt.Sprintf("ALTER TABLE %s ADD COLUMN w INT", table)); err != nil { - t.Fatalf("ALTER TABLE ADD COLUMN w: %v", err) - } - - for _, nodeID := range allNodes { - dataDir := harness.Nodes[nodeID-1].DataDir - base := waitForAutoIncBaseAtLeast(t, dataDir, db.DefaultDatabaseName, table, rowCount, 15*time.Second) - t.Logf("node %d: base after ALTER = %d (want >= %d)", nodeID, base, rowCount) - } + c := newCluster(t) + c.start() + const table, rowCount = "autoinc_seed_altermax", 1000 + c.createTable(1, "marmot", table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v VARCHAR(20))") + for start := 1; start <= rowCount; start += 100 { + values, _ := idValueRows(start, start+99, "row") + c.mustExec(1, "marmot", "INSERT INTO "+table+" (id, v) VALUES "+values) + } + c.waitRows("marmot", "SELECT COUNT(*), COUNT(DISTINCT id), MAX(id) FROM "+table, []string{"1000|1000|1000"}, 1, 2, 3) + base, found, err := c.autoIncBase(1, "marmot", table) + if err != nil || !found || base < rowCount { + t.Fatalf("before ALTER: node 1 base %d (found %v, err %v), want >= %d: explicit ids must raise it as they are inserted", base, found, err, rowCount) + } + c.mustExec(1, "marmot", "ALTER TABLE "+table+" ADD COLUMN w INT") + c.waitAutoIncBaseAtLeast("marmot", table, rowCount, 1, 2, 3) } -// TestAutoIncSeed_AlterFloorWithoutAColumnIsRefused asserts the refusal -// AlterTableColumnTypeRule raises (protocol/query/transform/alter_table_column_type.go): -// an ALTER carrying an AUTO_INCREMENT=N option with no narrow AUTO_INCREMENT -// column definition in the same statement has nowhere to store that floor -// (it belongs in the marker of the column it types), so it is refused with -// MySQL error 1235 (ER_NOT_SUPPORTED_YET), SQLSTATE 42000, rather than -// silently dropped. This covers both shapes the rule refuses: the -// option riding alongside an unrelated column add, and the bare -// option-only form. +// TestAutoIncSeed_AlterFloorWithoutAColumnIsRefused: an ALTER carrying an +// AUTO_INCREMENT=N option with no narrow AUTO_INCREMENT column definition has +// nowhere to store the floor, so it is refused with 1235 / SQLSTATE 42000, +// alone or beside a column add, and leaves no column behind. func TestAutoIncSeed_AlterFloorWithoutAColumnIsRefused(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("StartCluster: %v", err) - } - + c := newCluster(t) + c.start() const table = "autoinc_seed_floorrefused" - allNodes := []int{1, 2, 3} - - if _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, v TEXT)", table)); err != nil { - t.Fatalf("CREATE TABLE: %v", err) - } - if err := harness.WaitForTableExists(table, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("autoinc_seed_floorrefused_ddl") - t.Fatalf("DDL did not replicate: %v", err) - } - - statements := map[string]string{ - "ADD COLUMN alongside floor option": fmt.Sprintf( - "ALTER TABLE %s ADD COLUMN w INT, AUTO_INCREMENT=5000", table), - "floor option alone": fmt.Sprintf( - "ALTER TABLE %s AUTO_INCREMENT=5000", table), - } - - for label, stmt := range statements { - _, err := harness.ExecNode(1, stmt) + c.createTable(1, "marmot", table, "CREATE TABLE "+table+" (id INT PRIMARY KEY, v TEXT)") + for label, stmt := range map[string]string{ + "ADD COLUMN alongside floor option": "ALTER TABLE " + table + " ADD COLUMN w INT, AUTO_INCREMENT=5000", + "floor option alone": "ALTER TABLE " + table + " AUTO_INCREMENT=5000", + } { + _, err := c.exec(1, "marmot", stmt) mysqlErr := asMySQLError(t, err, label) - if mysqlErr.Number != mysqlcode.ErrCodeNotSupportedYet { - t.Errorf("%s: error code = %d, want %d (ER_NOT_SUPPORTED_YET)", - label, mysqlErr.Number, mysqlcode.ErrCodeNotSupportedYet) - } - if got := string(mysqlErr.SQLState[:]); got != mysqlcode.SQLStateSyntax { - t.Errorf("%s: SQLSTATE = %q, want %q", label, got, mysqlcode.SQLStateSyntax) + if mysqlErr.Number != mysqlcode.ErrCodeNotSupportedYet || string(mysqlErr.SQLState[:]) != mysqlcode.SQLStateSyntax { + t.Errorf("%s: %d/%s, want %d/%s", label, mysqlErr.Number, mysqlErr.SQLState[:], mysqlcode.ErrCodeNotSupportedYet, mysqlcode.SQLStateSyntax) } } - - // Neither refused statement may have left column w behind. - if _, err := harness.ExecNode(1, fmt.Sprintf("SELECT w FROM %s LIMIT 1", table)); err == nil { + if _, err := c.exec(1, "marmot", "SELECT w FROM "+table+" LIMIT 1"); err == nil { t.Errorf("column w exists on %s after both ALTERs were refused", table) } } -// TestAutoIncSeed_RefusesAnIDBeyondTheWidth: a TINYINT AUTO_INCREMENT column -// (widthMax = 127, intmarker.Attributes.WidthMax) refuses an explicit id -// above its ceiling at INSERT, with MySQL error 1264 -// (ER_WARN_DATA_OUT_OF_RANGE) / SQLSTATE 22003, as MySQL does in strict -// mode. SQLite would store it; letting it in would leave the table holding an -// id its declared width cannot, so it is refused before it reaches SQLite -// (protocol/query/rules/autoincrement_id.go). A DDL that declares a width the -// existing data does not fit is refused with the same code at PREPARE -// (db/ddl_prepare_validate.go checkAutoIncWidthCeilings), pinned by -// db/ddl_prepare_validate_autoinc_code_test.go and -// grpc/autoinc_claim_handler_test.go. +// TestAutoIncSeed_RefusesAnIDBeyondTheWidth: an explicit id above a TINYINT +// AUTO_INCREMENT column's ceiling is refused at INSERT with 1264 / SQLSTATE +// 22003, as MySQL does in strict mode, and nothing is written. func TestAutoIncSeed_RefusesAnIDBeyondTheWidth(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("StartCluster: %v", err) - } - + c := newCluster(t) + c.start() const table = "autoinc_width_ceiling" - allNodes := []int{1, 2, 3} - - if _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id TINYINT AUTO_INCREMENT PRIMARY KEY, v TEXT)", table)); err != nil { - t.Fatalf("CREATE TABLE: %v", err) - } - if err := harness.WaitForTableExists(table, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("autoinc_width_ceiling_ddl") - t.Fatalf("DDL did not replicate: %v", err) - } - - _, err := harness.ExecNode(1, fmt.Sprintf("INSERT INTO %s (id, v) VALUES (200, 'beyond_tinyint')", table)) - mysqlErr := asMySQLError(t, err, "explicit-id INSERT beyond widthMax") - if mysqlErr.Number != mysqlcode.ErrCodeDataOutOfRange { - t.Errorf("error code = %d, want %d (ER_WARN_DATA_OUT_OF_RANGE)", mysqlErr.Number, mysqlcode.ErrCodeDataOutOfRange) + c.createTable(1, "marmot", table, "CREATE TABLE "+table+" (id TINYINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + _, err := c.exec(1, "marmot", "INSERT INTO "+table+" (id, v) VALUES (200, 'beyond_tinyint')") + mysqlErr := asMySQLError(t, err, "explicit id beyond the width") + if mysqlErr.Number != mysqlcode.ErrCodeDataOutOfRange || string(mysqlErr.SQLState[:]) != mysqlcode.SQLStateDataOutOfRange { + t.Errorf("%d/%s, want %d/%s", mysqlErr.Number, mysqlErr.SQLState[:], mysqlcode.ErrCodeDataOutOfRange, mysqlcode.SQLStateDataOutOfRange) } - if got := string(mysqlErr.SQLState[:]); got != mysqlcode.SQLStateDataOutOfRange { - t.Errorf("SQLSTATE = %q, want %q", got, mysqlcode.SQLStateDataOutOfRange) - } - if n := harness.getRowCount(1, table); n != 0 { - t.Errorf("the refused row reached the table: %d rows", n) - } - t.Logf("refused as required: %v", mysqlErr) + c.waitRows("marmot", "SELECT COUNT(*) FROM "+table, []string{"0"}, 1) } -// =========================================================================== -// Item 7: the hidden table is invisible and write-protected -// =========================================================================== - -// TestAutoIncHiddenTable_AbsentFromListings asserts __marmot__autoinc never -// appears in SHOW TABLES or information_schema.tables on any node, for two -// independent reasons that both hold: it lives in the system database, which -// DatabaseManager never lists as a user database at all -// (db/autoinc_claim.go's placement doc), and the listing queries themselves -// filter the "__marmot__" prefix -// (protocol/handlers/metadata.go, protocol/handlers/information_schema.go). +// TestAutoIncHiddenTable_AbsentFromListings: the claim table never appears in +// SHOW TABLES or information_schema.tables on any node, even once it holds a +// row for a seeded table. func TestAutoIncHiddenTable_AbsentFromListings(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("StartCluster: %v", err) - } - + c := newCluster(t) + c.start() const table = "autoinc_hidden_listing" - allNodes := []int{1, 2, 3} - - // A table that DOES exist and must be seeded, so the system database - // definitely holds a __marmot__autoinc row by the time listings are - // checked - a table with no row at all would make this assertion trivial. - if _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)", table)); err != nil { - t.Fatalf("CREATE TABLE: %v", err) - } - if err := harness.WaitForTableExists(table, allNodes, 15*time.Second); err != nil { - t.Fatalf("DDL did not replicate: %v", err) - } - for _, nodeID := range allNodes { - dataDir := harness.Nodes[nodeID-1].DataDir - waitForAutoIncBaseAtLeast(t, dataDir, db.DefaultDatabaseName, table, 0, 15*time.Second) - } - - for _, nodeID := range allNodes { - if harness.tableExistsOnNode(nodeID, common.AutoIncClaimTableName) { - t.Errorf("node %d: SHOW TABLES lists %s", nodeID, common.AutoIncClaimTableName) + c.createTable(1, "marmot", table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + c.waitAutoIncBaseAtLeast("marmot", table, 0, 1, 2, 3) + for id := 1; id <= 3; id++ { + if listed, err := c.hasTable(id, "marmot", common.AutoIncClaimTableName); err != nil || listed { + t.Errorf("node %d: SHOW TABLES lists %s (err %v)", id, common.AutoIncClaimTableName, err) } - - conn, err := harness.ConnectToNode(nodeID) - if err != nil { - t.Fatalf("connect to node %d: %v", nodeID, err) - } - // The schema name is formatted into the statement rather than bound as - // a parameter on purpose. A bound parameter makes the client use a - // prepared statement, and the prepared path answers COM_STMT_EXECUTE - // by re-running the TRANSPILED text (protocol/server.go:1411), which - // no longer parses as an INFORMATION_SCHEMA query - so the query lands - // on SQLite as "no such table: tables". That is a pre-existing gap in - // prepared-statement routing for every statement Marmot answers above - // the database, not a property of the claim table, and it is recorded - // as such rather than worked around silently. The value is a constant - // in this test, so formatting it injects nothing. - names, err := queryStringColumn(conn, fmt.Sprintf( - "SELECT table_name FROM information_schema.tables WHERE table_schema = '%s'", + // Formatted rather than bound: a bound parameter takes the prepared + // statement path, which does not route INFORMATION_SCHEMA queries. + names, err := columnValues(c, id, fmt.Sprintf("SELECT table_name FROM information_schema.tables WHERE table_schema = '%s'", db.DefaultDatabaseName), "table_name") if err != nil { - t.Fatalf("node %d: information_schema.tables: %v", nodeID, err) + t.Fatalf("node %d: information_schema.tables: %v", id, err) } - for _, name := range names { - if strings.EqualFold(name, common.AutoIncClaimTableName) { - t.Errorf("node %d: information_schema.tables lists %s", nodeID, common.AutoIncClaimTableName) - } + if !containsFold(names, table) { + t.Errorf("node %d: information_schema.tables omits %s: %v", id, table, names) + } + if containsFold(names, common.AutoIncClaimTableName) { + t.Errorf("node %d: information_schema.tables lists %s", id, common.AutoIncClaimTableName) } } } -// TestAutoIncHiddenTable_RejectsClientWrites asserts INSERT, UPDATE, DELETE -// and DROP TABLE against __marmot__autoinc are all refused with MySQL error -// 1142 (ER_TABLEACCESS_DENIED_ERROR), the code -// protocol/query/rules/claim_table_guard.go's ClaimTableGuardRule uses. The -// table need not exist in the user database for the rule to fire - it is a -// structural match on the parsed statement's target table name, evaluated -// before the statement ever reaches SQLite (ClaimTableGuardRule.Priority() -// == 0, runs before every other transform rule). -func TestAutoIncHiddenTable_RejectsClientWrites(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("StartCluster: %v", err) +// containsFold reports whether names holds name, ignoring case. +func containsFold(names []string, name string) bool { + for _, n := range names { + if strings.EqualFold(n, name) { + return true + } } + return false +} +// TestAutoIncHiddenTable_RejectsClientWrites: INSERT, UPDATE, DELETE and DROP +// TABLE against the claim table are refused with 1142. +func TestAutoIncHiddenTable_RejectsClientWrites(t *testing.T) { + c := newCluster(t) + c.start() table := common.AutoIncClaimTableName - statements := map[string]string{ - "INSERT": fmt.Sprintf( - "INSERT INTO %s (db, tbl, committed, owner, granted_at) VALUES ('marmot', 'x', 1, 1, 1)", table), - "UPDATE": fmt.Sprintf("UPDATE %s SET committed = 999 WHERE tbl = 'x'", table), - "DELETE": fmt.Sprintf("DELETE FROM %s WHERE tbl = 'x'", table), - "DROP": fmt.Sprintf("DROP TABLE %s", table), - } - - for label, stmt := range statements { - _, err := harness.ExecNode(1, stmt) - mysqlErr := asMySQLError(t, err, label+" against "+table) - if mysqlErr.Number != mysqlcode.ErrCodeTableAccessDenied { - t.Errorf("%s: error code = %d, want %d (ER_TABLEACCESS_DENIED_ERROR)", - label, mysqlErr.Number, mysqlcode.ErrCodeTableAccessDenied) + for label, stmt := range map[string]string{ + "INSERT": "INSERT INTO " + table + " (db, tbl, committed, owner, granted_at) VALUES ('marmot', 'x', 1, 1, 1)", + "UPDATE": "UPDATE " + table + " SET committed = 999 WHERE tbl = 'x'", + "DELETE": "DELETE FROM " + table + " WHERE tbl = 'x'", + "DROP": "DROP TABLE " + table, + } { + _, err := c.exec(1, "marmot", stmt) + if code := asMySQLError(t, err, label).Number; code != mysqlcode.ErrCodeTableAccessDenied { + t.Errorf("%s: error %d, want %d", label, code, mysqlcode.ErrCodeTableAccessDenied) } - t.Logf("%s correctly refused: %v", label, mysqlErr) } } diff --git a/test/autoinc_claim_inprocess_test.go b/test/autoinc_claim_inprocess_test.go index 863b9488..9e8a60af 100644 --- a/test/autoinc_claim_inprocess_test.go +++ b/test/autoinc_claim_inprocess_test.go @@ -195,7 +195,7 @@ type inprocCluster struct { const inprocCoordinatorTimeout = 30 * time.Second -// newInprocCluster builds nodeIDs participants, each with its own t.TempDir() +// newInprocCluster builds nodeIDs participants, each with its own freshDir // data directory (db/replication_engine_test.go:64-78's own-tempdir // pattern). It registers a cleanup that closes whatever DatabaseManager is // current for each node at test end, so a test that reopens a node (restart, @@ -209,7 +209,7 @@ func newInprocCluster(t *testing.T, nodeIDs []uint64) *inprocCluster { nodes: make(map[uint64]*inprocNode), } for _, id := range nodeIDs { - c.nodes[id] = c.buildNode(id, t.TempDir()) + c.nodes[id] = c.buildNode(id, freshDir(t)) } c.releaseNewCluster() t.Cleanup(func() { @@ -285,7 +285,7 @@ func (c *inprocCluster) reopenNode(id uint64) { // newInprocClusterFromDirs rebuilds an inprocCluster from data directories a // prior process already populated - one per node ID - instead of creating -// fresh t.TempDir() storage the way newInprocCluster does. This is what a +// fresh freshDir storage the way newInprocCluster does. This is what a // node process reopening its own on-disk state, potentially from a separate // os.Process that no longer exists, looks like from the test's side; it is // how TestClaimRange_KillNineRecoversCommittedBase inspects a cluster after @@ -350,21 +350,10 @@ func driveDDL(t *testing.T, node *inprocNode, database, table, ddl string) { func waitForBase(t *testing.T, node *inprocNode, database, table string, want uint64) { t.Helper() store := node.claimStore() - deadline := time.Now().Add(5 * time.Second) - var last uint64 - var lastErr error - for { - last, lastErr = store.ReadBase(database, table) - if lastErr == nil && last == want { - return - } - if time.Now().After(deadline) { - break - } - time.Sleep(10 * time.Millisecond) - } - require.NoError(t, lastErr, "node %d: final ReadBase(%s.%s)", node.id, database, table) - require.Equal(t, want, last, "node %d: base for %s.%s did not converge in time", node.id, database, table) + waitFor(t, fmt.Sprintf("node %d base for %s.%s = %d", node.id, database, table, want), 5*time.Second, func() (bool, string) { + last, err := store.ReadBase(database, table) + return err == nil && last == want, fmt.Sprintf("base %d, err %v", last, err) + }) } // waitForQuorumBase polls until at least minNodes of nodes read `want` for @@ -395,25 +384,15 @@ func waitForBase(t *testing.T, node *inprocNode, database, table string, want ui // coordinator itself required (RequiredQuorum for 3 nodes = 2, coordinator/cluster.go:100). func waitForQuorumBase(t *testing.T, nodes []*inprocNode, database, table string, want uint64, minNodes int) { t.Helper() - deadline := time.Now().Add(5 * time.Second) - var lastCount int - for { - lastCount = 0 + waitFor(t, fmt.Sprintf("%d of %d nodes base for %s.%s = %d", minNodes, len(nodes), database, table, want), 5*time.Second, func() (bool, string) { + count := 0 for _, n := range nodes { if b, err := n.claimStore().ReadBase(database, table); err == nil && b == want { - lastCount++ + count++ } } - if lastCount >= minNodes { - return - } - if time.Now().After(deadline) { - break - } - time.Sleep(10 * time.Millisecond) - } - require.GreaterOrEqual(t, lastCount, minNodes, - "only %d/%d nodes converged to %d for %s.%s within the deadline", lastCount, len(nodes), want, database, table) + return count >= minNodes, fmt.Sprintf("%d nodes converged", count) + }) } // rangesDisjoint reports whether two claimed ranges (newBase, newBase+size] @@ -842,7 +821,7 @@ func killNineNodeDir(root string, id uint64) string { // three nodes from the same directories and asserts the committed base and // the claim protocol's staleness rejection both survived the kill. func TestClaimRange_KillNineRecoversCommittedBase(t *testing.T) { - root := t.TempDir() + root := freshDir(t) dataDirs := make(map[uint64]string, 3) for _, id := range []uint64{1, 2, 3} { dir := killNineNodeDir(root, id) @@ -1075,7 +1054,7 @@ func TestClaimRange_SurvivesFullClusterSnapshotRestore(t *testing.T) { // Snapshot every node BEFORE any of them stop. for _, id := range []uint64{1, 2, 3} { - snapDir := t.TempDir() + snapDir := freshDir(t) snapshots, _, err := c.nodes[id].dm.TakeSnapshotToDir(snapDir) require.NoError(t, err) require.NotEmpty(t, snapshots) diff --git a/test/autoinc_lifecycle_cluster_test.go b/test/autoinc_lifecycle_cluster_test.go index 5e93de7a..dd0d2e06 100644 --- a/test/autoinc_lifecycle_cluster_test.go +++ b/test/autoinc_lifecycle_cluster_test.go @@ -2,343 +2,163 @@ package test // Cluster tests for the lifecycle of narrow AUTO_INCREMENT ranges across DDL, // rebuilt nodes and the client shapes that reach the allocator only through -// bound values or a qualified table, over the real MySQL wire protocol -// against a 3-node ClusterHarness (test/crash_recovery_test.go). +// bound values or a qualified table. import ( - "database/sql" "fmt" "os" "path/filepath" - "strings" "testing" - "time" ) -// openNodeDatabase connects to database on nodeID. -func openNodeDatabase(t *testing.T, harness *ClusterHarness, nodeID int, database string) *sql.DB { - t.Helper() - conn, err := sql.Open("mysql", fmt.Sprintf("root:@tcp(localhost:%d)/%s", harness.Nodes[nodeID-1].MySQLPort, database)) - if err != nil { - t.Fatalf("open %s on node %d: %v", database, nodeID, err) - } - t.Cleanup(func() { conn.Close() }) - return conn +// idLedger records every generated id a test saw issued, failing the test on +// an id issued twice or outside a 32-bit column. +type idLedger struct { + t *testing.T + seen map[int64]string } -// waitForDatabase waits until database accepts a connection on every node. -func waitForDatabase(t *testing.T, harness *ClusterHarness, database string, timeout time.Duration) { - t.Helper() - deadline := time.Now().Add(timeout) - for nodeID := 1; nodeID <= numNodes; nodeID++ { - for { - conn, err := sql.Open("mysql", fmt.Sprintf("root:@tcp(localhost:%d)/%s", harness.Nodes[nodeID-1].MySQLPort, database)) - if err == nil { - err = conn.Ping() - conn.Close() - } - if err == nil { - break - } - if time.Now().After(deadline) { - harness.dumpNodeLogs("wait_database_" + database) - t.Fatalf("database %s not on node %d: %v", database, nodeID, err) - } - time.Sleep(250 * time.Millisecond) - } - } -} +func newIDLedger(t *testing.T) *idLedger { return &idLedger{t: t, seen: map[int64]string{}} } -// waitForTableIn waits until table exists in database on every node. -func waitForTableIn(t *testing.T, harness *ClusterHarness, database, table string, timeout time.Duration) { - t.Helper() - deadline := time.Now().Add(timeout) - for nodeID := 1; nodeID <= numNodes; nodeID++ { - conn := openNodeDatabase(t, harness, nodeID, database) - for { - var n int - err := conn.QueryRow("SELECT COUNT(*) FROM " + table).Scan(&n) - if err == nil { - break - } - if time.Now().After(deadline) { - harness.dumpNodeLogs("wait_table_" + table) - t.Fatalf("%s.%s not on node %d: %v", database, table, nodeID, err) - } - time.Sleep(250 * time.Millisecond) - } +// record adds id, issued for who, to the ledger. +func (l *idLedger) record(who string, id int64, err error) { + l.t.Helper() + if err != nil { + l.t.Fatalf("%s: %v", who, err) + } + if prev, dup := l.seen[id]; dup || id <= 0 || id > narrowInt32Max { + l.t.Fatalf("%s: id %d is not a new 32-bit id (already issued to %q)", who, id, prev) } + l.seen[id] = who } -// insertGenerated inserts one row with a generated id through conn, retrying -// the retryable lock-wait code a claim returns while the cluster settles. -func insertGenerated(conn *sql.DB, query string, args ...interface{}) (int64, error) { - var lastErr error - for attempt := 0; attempt < 40; attempt++ { - res, err := conn.Exec(query, args...) - if err == nil { - return res.LastInsertId() - } - lastErr = err - if !strings.Contains(err.Error(), "1205") { - return 0, err - } - time.Sleep(250 * time.Millisecond) +// stats is the idStats row every node must hold once the ledger's rows +// replicated: every id present once. +func (l *idLedger) stats() []string { + var highest int64 + for id := range l.seen { + highest = max(highest, id) } - return 0, lastErr + return []string{fmt.Sprintf("%d|%d|%d", len(l.seen), len(l.seen), highest)} } -// TestNarrowAutoInc_DropDatabaseRecreateNeverReissues is reviewer A's F1: -// node 1 issues ids from a range, the database is dropped and recreated with -// the same table, and every node inserts again. Before, node 1 kept minting -// from its dead range while the others were granted the same ids afresh, and -// generated inserts failed with 1062 (or, unreplicated, overwrote). Now the -// claim row survives the drop and every node forgets its ranges of the -// dropped incarnation: no generated insert fails and no id repeats. +// TestNarrowAutoInc_DropDatabaseRecreateNeverReissues: node 1 issues ids from +// a range, the database is dropped and recreated with the same table, and +// every node inserts again. The claim row survives the drop and every node +// forgets its ranges of the dropped incarnation: no generated insert fails +// and no id repeats (before the fix node 1 kept minting from its dead range +// while the others were granted the same ids afresh). func TestNarrowAutoInc_DropDatabaseRecreateNeverReissues(t *testing.T) { const database, table = "dropdb", "t" - harness := NewClusterHarness(t) - defer harness.Cleanup() - if err := harness.StartCluster(); err != nil { - t.Fatalf("StartCluster: %v", err) - } - + c := newCluster(t) + c.start() create := func() { - t.Helper() - if _, err := harness.ExecNode(1, "CREATE DATABASE "+database); err != nil { - t.Fatalf("CREATE DATABASE: %v", err) - } - waitForDatabase(t, harness, database, 30*time.Second) - if _, err := openNodeDatabase(t, harness, 1, database).Exec("CREATE TABLE " + table + " (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)"); err != nil { - t.Fatalf("CREATE TABLE: %v", err) - } - waitForTableIn(t, harness, database, table, 30*time.Second) + c.createDatabase(1, database) + c.createTable(1, database, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") } - create() - node1 := openNodeDatabase(t, harness, 1, database) for i := 0; i < 3; i++ { - if _, err := insertGenerated(node1, "INSERT INTO "+table+" (v) VALUES ('before')"); err != nil { + if _, err := insertNarrow(c, 1, database, "INSERT INTO "+table+" (v) VALUES ('before')"); err != nil { t.Fatalf("insert before the drop: %v", err) } } - - if _, err := harness.ExecNode(1, "DROP DATABASE "+database); err != nil { - t.Fatalf("DROP DATABASE: %v", err) - } - time.Sleep(2 * time.Second) + c.mustExec(1, "marmot", "DROP DATABASE "+database) + c.waitDatabase(database, false, 1, 2, 3) create() // Enough rounds that a node granted the dropped incarnation's range // afresh would reach the ids node 1 kept issuing from its old cursor. - seen := make(map[int64]int) + ledger := newIDLedger(t) for round := 0; round < 10; round++ { - for nodeID := 1; nodeID <= numNodes; nodeID++ { - id, err := insertGenerated(openNodeDatabase(t, harness, nodeID, database), - "INSERT INTO "+table+" (v) VALUES (?)", fmt.Sprintf("n%d-%d", nodeID, round)) - if err != nil { - harness.dumpNodeLogs("dropdb_after") - t.Fatalf("node %d: generated insert after the recreate failed: %v", nodeID, err) - } - if other, dup := seen[id]; dup { - t.Fatalf("id %d issued by node %d and node %d after the recreate", id, other, nodeID) - } - seen[id] = nodeID - } - } - if err := waitForRowCountIn(harness, database, table, len(seen), 30*time.Second); err != nil { - harness.dumpNodeLogs("dropdb_rows") - t.Fatal(err) - } -} - -// waitForRowCountIn waits until database.table holds want rows with want -// distinct ids on every node. -func waitForRowCountIn(harness *ClusterHarness, database, table string, want int, timeout time.Duration) error { - deadline := time.Now().Add(timeout) - for nodeID := 1; nodeID <= numNodes; nodeID++ { - for { - conn, err := sql.Open("mysql", fmt.Sprintf("root:@tcp(localhost:%d)/%s", harness.Nodes[nodeID-1].MySQLPort, database)) - var rows, distinct int - if err == nil { - err = conn.QueryRow("SELECT COUNT(*), COUNT(DISTINCT id) FROM "+table).Scan(&rows, &distinct) - conn.Close() - } - if err == nil && rows == want && distinct == want { - break - } - if time.Now().After(deadline) { - return fmt.Errorf("node %d: %s.%s has %d rows, %d distinct ids, want %d (%v)", nodeID, database, table, rows, distinct, want, err) - } - time.Sleep(250 * time.Millisecond) + for node := 1; node <= 3; node++ { + who := fmt.Sprintf("node %d round %d", node, round) + id, err := insertNarrow(c, node, database, "INSERT INTO "+table+" (v) VALUES (?)", who) + ledger.record(who, id, err) } } - return nil + c.waitRows(database, idStats(table, "id"), ledger.stats(), 1, 2, 3) } -// TestNarrowAutoInc_BoundNullAndQualifiedInsertsAreGenerated is reviewer A's -// F3 and F4 over the wire: a prepared INSERT binding NULL to the id, and an -// INSERT naming its table as db.t from a session in another database, text -// and prepared. Before, SQLite assigned such a row's id itself (MAX+1), and -// whenever that fell inside another node's unissued range the admission -// refused it with 1235: about half the bound NULLs and every qualified INSERT -// in reviewer A's runs. -// -// Each round node 1 first issues a generated id from its range and the row -// reaches the others; nodes 2 and 3 then write every shape. A SQLite rowid -// would land inside another node's unissued range and be refused, so every -// write must get a generated, distinct 32-bit id. +// TestNarrowAutoInc_BoundNullAndQualifiedInsertsAreGenerated: a prepared +// INSERT binding NULL to the id, and an INSERT naming its table as db.t from +// a session in another database, text and prepared, all get generated ids. +// Before the fix SQLite assigned such a row's id itself (MAX+1), inside +// another node's unissued range, and the admission refused it with 1235. +// Each round node 1 first issues an id from its range and the row reaches +// the others; nodes 2 and 3 then write every shape, each shape first on its +// node in round one. func TestNarrowAutoInc_BoundNullAndQualifiedInsertsAreGenerated(t *testing.T) { const database, table = "qual", "t" - harness := NewClusterHarness(t) - defer harness.Cleanup() - if err := harness.StartCluster(); err != nil { - t.Fatalf("StartCluster: %v", err) - } - if _, err := harness.ExecNode(1, "CREATE DATABASE "+database); err != nil { - t.Fatalf("CREATE DATABASE: %v", err) - } - waitForDatabase(t, harness, database, 30*time.Second) - if _, err := openNodeDatabase(t, harness, 1, database).Exec("CREATE TABLE " + table + " (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)"); err != nil { - t.Fatalf("CREATE TABLE: %v", err) - } - waitForTableIn(t, harness, database, table, 30*time.Second) - - seen := make(map[int64]bool) - record := func(nodeID int, shape string, id int64, err error) { - t.Helper() - if err != nil { - harness.dumpNodeLogs("qual_" + shape) - t.Fatalf("node %d %s: %v", nodeID, shape, err) - } - if id <= 0 || id > narrowInt32Max || seen[id] { - t.Fatalf("node %d %s: id %d is not a new 32-bit id", nodeID, shape, id) - } - seen[id] = true + c := newCluster(t) + c.start() + c.createDatabase(1, database) + c.createTable(1, database, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + + ledger := newIDLedger(t) + boundNull := func(node int) { + id, err := insertNarrow(c, node, database, "INSERT INTO "+table+" (id, v) VALUES (?, ?)", nil, "bound-null") + ledger.record(fmt.Sprintf("node %d bound NULL", node), id, err) + } + qualified := func(node int) { + id, err := insertNarrow(c, node, "marmot", "INSERT INTO "+database+"."+table+" (v) VALUES ('qualified')") + ledger.record(fmt.Sprintf("node %d qualified text", node), id, err) + id, err = insertNarrow(c, node, "marmot", "INSERT INTO "+database+"."+table+" (v) VALUES (?)", "qualified-bound") + ledger.record(fmt.Sprintf("node %d qualified bound", node), id, err) } - node1 := openNodeDatabase(t, harness, 1, database) - const rounds = 10 - for round := 0; round < rounds; round++ { - id, err := insertGenerated(node1, "INSERT INTO "+table+" (v) VALUES ('node1')") - record(1, "generated", id, err) - if err := waitForRowCountIn(harness, database, table, len(seen), 30*time.Second); err != nil { - t.Fatal(err) - } - // Node 2 writes the qualified shapes first and node 3 the bound NULL - // first, so that in the first round each shape is the first write - // on its node: a rowid it got from SQLite would be MAX+1, inside - // another node's unissued range. - boundNull := func(nodeID int) { - id, err := insertGenerated(openNodeDatabase(t, harness, nodeID, database), - "INSERT INTO "+table+" (id, v) VALUES (?, ?)", nil, "bound-null") - record(nodeID, "bound NULL", id, err) - } - qualified := func(nodeID int) { - inDefault := openNodeDatabase(t, harness, nodeID, "marmot") - id, err := insertGenerated(inDefault, "INSERT INTO "+database+"."+table+" (v) VALUES ('qualified')") - record(nodeID, "qualified text", id, err) - id, err = insertGenerated(inDefault, "INSERT INTO "+database+"."+table+" (v) VALUES (?)", "qualified-bound") - record(nodeID, "qualified bound", id, err) - } + for round := 0; round < 10; round++ { + id, err := insertNarrow(c, 1, database, "INSERT INTO "+table+" (v) VALUES ('node1')") + ledger.record("node 1 generated", id, err) + c.waitRows(database, idStats(table, "id"), ledger.stats(), 1, 2, 3) qualified(2) boundNull(2) boundNull(3) qualified(3) } - if err := waitForRowCountIn(harness, database, table, len(seen), 30*time.Second); err != nil { - harness.dumpNodeLogs("qual_rows") - t.Fatal(err) - } + c.waitRows(database, idStats(table, "id"), ledger.stats(), 1, 2, 3) } -// wipeNode stops nodeID and deletes everything in its data directory but its -// config, as a lost disk would. -func wipeNode(t *testing.T, harness *ClusterHarness, nodeID int) { - t.Helper() - if err := harness.StopNode(nodeID); err != nil { - t.Fatalf("StopNode(%d): %v", nodeID, err) - } - node := harness.Nodes[nodeID-1] - entries, err := os.ReadDir(node.DataDir) +// wipe kills node id and deletes everything in its directory but its config +// and log, as a lost disk would. +func (c *cluster) wipe(id int) { + c.t.Helper() + c.kill(id) + n := c.node(id) + entries, err := os.ReadDir(n.dir) if err != nil { - t.Fatalf("read node %d data dir: %v", nodeID, err) + c.t.Fatal(err) } for _, e := range entries { - if e.Name() == filepath.Base(node.ConfigPath) { + path := filepath.Join(n.dir, e.Name()) + if path == n.config || path == n.log { continue } - if err := os.RemoveAll(filepath.Join(node.DataDir, e.Name())); err != nil { - t.Fatalf("wipe node %d: %v", nodeID, err) + if err := os.RemoveAll(path); err != nil { + c.t.Fatalf("wipe node %d: %v", id, err) } } } -// nodeLogContains reports whether nodeID's current log holds text. -func nodeLogContains(t *testing.T, harness *ClusterHarness, nodeID int, text string) bool { - t.Helper() - log, err := os.ReadFile(harness.Nodes[nodeID-1].LogFile) - if err != nil { - t.Fatalf("read node %d log: %v", nodeID, err) - } - return strings.Contains(string(log), text) -} - -const autoIncReleasedLog = "AUTO_INCREMENT claim votes released" - -// TestNarrowAutoInc_RebuiltNodesHoldTheirVotes is reviewer B's C2 end to end: -// a node that loses its data directory restarts with its votes held, merges -// from the other members and releases, and inserts on every node then -// succeed with no repeated id. The new cluster itself released without help: -// every node of a new cluster starts held. -// -// The seedless seed node (C2 (a)) is covered in process -// (TestClaimRange_RebuiltNodeHoldsVotesUntilMerged) and its membership-of-one -// belt by grpc.TestAutoIncMergeSafe: in this harness a wiped node 1 comes back -// knowing only itself and stays that way, so the only observable outcome here -// would be that it never releases. +// TestNarrowAutoInc_RebuiltNodesHoldTheirVotes: a node that loses its data +// directory restarts with its votes held, merges from the other members and +// releases (restart waits for that), restores the table, and inserts on +// every node then succeed with no repeated id. The new cluster itself +// released without help: every node of a new cluster starts held. func TestNarrowAutoInc_RebuiltNodesHoldTheirVotes(t *testing.T) { const table = "rebuilt" - harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - defer harness.Cleanup() - - seen := make(map[int64]bool) - insertOn := func(phase string, nodes ...int) { - t.Helper() - for _, nodeID := range nodes { + c := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + ledger := newIDLedger(t) + insertOn := func(phase string) { + for node := 1; node <= 3; node++ { for i := 0; i < 5; i++ { - id, err := insertNarrow(harness, nodeID, "INSERT INTO "+table+" (v) VALUES (?)", phase) - if err != nil { - harness.dumpNodeLogs("rebuilt_" + phase) - t.Fatalf("%s: node %d insert: %v", phase, nodeID, err) - } - if seen[id] || id > narrowInt32Max { - t.Fatalf("%s: node %d issued %d again or beyond INT", phase, nodeID, id) - } - seen[id] = true + id, err := insertNarrow(c, node, "marmot", "INSERT INTO "+table+" (v) VALUES (?)", phase) + ledger.record(fmt.Sprintf("%s node %d", phase, node), id, err) } } } - insertOn("before", 1, 2, 3) - - wipeNode(t, harness, 3) - if err := harness.StartNode(3); err != nil { - t.Fatalf("StartNode(3): %v", err) - } - if err := harness.WaitForAlive(3, 60*time.Second); err != nil { - harness.dumpNodeLogs("rebuilt_node3") - t.Fatalf("node 3 did not come back: %v", err) - } - deadline := time.Now().Add(60 * time.Second) - for !nodeLogContains(t, harness, 3, autoIncReleasedLog) { - if time.Now().After(deadline) { - harness.dumpNodeLogs("rebuilt_node3_release") - t.Fatal("rebuilt node 3 never released its claim votes") - } - time.Sleep(500 * time.Millisecond) - } - if err := harness.WaitForTableExists(table, []int{3}, 60*time.Second); err != nil { - harness.dumpNodeLogs("rebuilt_node3_table") - t.Fatalf("node 3 did not restore %s: %v", table, err) - } - insertOn("after-node3", 1, 2, 3) + insertOn("before") + c.wipe(3) + c.restart(3) + c.waitTable("marmot", table, 3) + insertOn("after-node3") + c.waitRows("marmot", idStats(table, "id"), ledger.stats(), 1, 2, 3) } diff --git a/test/autoinc_membership_cluster_test.go b/test/autoinc_membership_cluster_test.go index 584b845f..e4759029 100644 --- a/test/autoinc_membership_cluster_test.go +++ b/test/autoinc_membership_cluster_test.go @@ -2,85 +2,32 @@ package test // Cluster tests for how a node that lost its claim state, or joined, comes // back into the AUTO_INCREMENT claim protocol: the vote hold on every path -// that loses claim state, the release once the node can reach enough members, -// and the operator's sync command for a membership change. Every client call -// here carries its own timeout, so a hang fails the test instead of blocking -// it. +// that loses claim state, the release once the node can reach enough +// members, and the operator's sync command for a membership change. import ( - "context" - "database/sql" - "encoding/json" - "errors" "fmt" - "io" + "maps" "net/http" + "slices" "testing" "time" - "github.com/go-sql-driver/mysql" - "github.com/maxpert/marmot/cfg" "github.com/maxpert/marmot/protocol/mysqlcode" ) -// clusterQueryTimeout bounds every single client query in these tests. -const clusterQueryTimeout = 20 * time.Second - -// openTimedNodeDatabase connects to database on nodeID with I/O timeouts on -// the connection itself, on top of each query's own context. -func openTimedNodeDatabase(t *testing.T, harness *ClusterHarness, nodeID int, database string) *sql.DB { - t.Helper() - dsn := fmt.Sprintf("root:@tcp(localhost:%d)/%s?timeout=5s&readTimeout=%s&writeTimeout=%s", - harness.Nodes[nodeID-1].MySQLPort, database, clusterQueryTimeout, clusterQueryTimeout) - conn, err := sql.Open("mysql", dsn) - if err != nil { - t.Fatalf("open %s on node %d: %v", database, nodeID, err) - } - t.Cleanup(func() { conn.Close() }) - return conn -} - -// execTimed runs one statement under clusterQueryTimeout. -func execTimed(conn *sql.DB, query string, args ...interface{}) (sql.Result, error) { - ctx, cancel := context.WithTimeout(context.Background(), clusterQueryTimeout) - defer cancel() - defer noteClientCall() - return conn.ExecContext(ctx, query, args...) -} - -// isLockTimeout reports whether err is the retryable 1205 a held or -// quorum-less claim returns. -func isLockTimeout(err error) bool { - var mysqlErr *mysql.MySQLError - return errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlcode.ErrCodeLockTimeout -} - -// insertNarrowWithin inserts one row with a generated id, retrying 1205 until -// budget is spent. -func insertNarrowWithin(conn *sql.DB, budget time.Duration, query string, args ...interface{}) (int64, error) { - deadline := time.Now().Add(budget) - for { - res, err := execTimed(conn, query, args...) - if err == nil { - return res.LastInsertId() - } - if !isLockTimeout(err) || time.Now().After(deadline) { - return 0, err - } - time.Sleep(250 * time.Millisecond) - } -} +// syncTimeout bounds one run of the sync command: it asks every member, each +// within the merge's own per-peer timeout. +const syncTimeout = 6 * time.Second // autoIncSyncNode is one member's entry in the sync command's answer. type autoIncSyncNode struct { NodeID uint64 `json:"node_id"` Answered bool `json:"answered"` - Complete bool `json:"complete"` VotesHeld bool `json:"votes_held"` Members []uint64 `json:"members"` Reached []uint64 `json:"reached"` ReachedEveryMember bool `json:"reached_every_member"` - Error string `json:"error"` } // autoIncSyncAnswer is the sync command's answer. @@ -99,339 +46,224 @@ func (a autoIncSyncAnswer) node(id uint64) (autoIncSyncNode, bool) { return autoIncSyncNode{}, false } -// runAutoIncSync runs POST /admin/cluster/autoinc/sync on nodeID with the -// harness's cluster secret. -func runAutoIncSync(harness *ClusterHarness, nodeID int) (autoIncSyncAnswer, error) { - url := fmt.Sprintf("http://localhost:%d/admin/cluster/autoinc/sync", harness.Nodes[nodeID-1].GRPCPort) - req, err := http.NewRequest(http.MethodPost, url, nil) - if err != nil { - return autoIncSyncAnswer{}, err - } - req.Header.Set("X-Marmot-Secret", "test-secret") - resp, err := (&http.Client{Timeout: 90 * time.Second}).Do(req) - noteClientCall() - if err != nil { - return autoIncSyncAnswer{}, err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return autoIncSyncAnswer{}, err - } - if resp.StatusCode != http.StatusOK { - return autoIncSyncAnswer{}, fmt.Errorf("sync on node %d: %d %s", nodeID, resp.StatusCode, body) - } - var wrapped struct { - Data autoIncSyncAnswer `json:"data"` - } - if err := json.Unmarshal(body, &wrapped); err != nil { - return autoIncSyncAnswer{}, fmt.Errorf("sync on node %d: %w: %s", nodeID, err, body) - } - return wrapped.Data, nil -} - -// waitForRelease waits until nodeID's current log says its held votes were -// released. The log starts afresh with each start, so the line proves this -// run of the node was held. -func waitForRelease(t *testing.T, harness *ClusterHarness, nodeID int, timeout time.Duration, why string) { - t.Helper() - deadline := time.Now().Add(timeout) - for !nodeLogContains(t, harness, nodeID, autoIncReleasedLog) { - if time.Now().After(deadline) { - harness.dumpNodeLogs(fmt.Sprintf("release_node%d", nodeID)) - t.Fatalf("node %d: %s", nodeID, why) - } - time.Sleep(500 * time.Millisecond) - } -} - -// idLedger records every acknowledged generated id and fails on a repeat. -type idLedger map[int64]string - -func (l idLedger) record(t *testing.T, id int64, who string) { - t.Helper() - if id <= 0 || id > narrowInt32Max { - t.Fatalf("%s: id %d is not a 32-bit id", who, id) - } - if prev, dup := l[id]; dup { - t.Fatalf("DUPLICATE id %d: acknowledged to %s and to %s", id, prev, who) - } - l[id] = who -} - -// quorumProbeTable is a wide table every test here creates, so -// waitForQuorumWrites can tell when each node can commit writes again. -const quorumProbeTable = "quorum_probe" - -// createQuorumProbe creates quorumProbeTable on every node. -func createQuorumProbe(t *testing.T, harness *ClusterHarness) { - t.Helper() - if _, err := execTimed(openTimedNodeDatabase(t, harness, 1, "marmot"), - "CREATE TABLE "+quorumProbeTable+" (k INT PRIMARY KEY, v TEXT)"); err != nil { - t.Fatalf("CREATE TABLE %s: %v", quorumProbeTable, err) - } - if err := harness.WaitForTableExists(quorumProbeTable, []int{1, 2, 3}, 30*time.Second); err != nil { - t.Fatal(err) +// autoIncSync runs the sync command (POST /admin/cluster/autoinc/sync) on +// node id. +func (c *cluster) autoIncSync(id int) autoIncSyncAnswer { + c.t.Helper() + var answer autoIncSyncAnswer + if err := c.admin(id, http.MethodPost, "/cluster/autoinc/sync", syncTimeout, &answer); err != nil { + c.t.Fatal(err) } + return answer } -// waitForQuorumWrites waits until every node in nodes commits an ordinary -// replicated write. A node's connections to peers that restarted come back -// on gRPC's reconnect backoff, and until they do its writes cannot reach a -// quorum; that is not what these tests are about. -func waitForQuorumWrites(t *testing.T, harness *ClusterHarness, timeout time.Duration, nodes ...int) { - t.Helper() - deadline := time.Now().Add(timeout) - for _, nodeID := range nodes { - conn := openTimedNodeDatabase(t, harness, nodeID, "marmot") - for { - _, err := execTimed(conn, "REPLACE INTO "+quorumProbeTable+" (k, v) VALUES (?, 'probe')", nodeID) - if err == nil { - break +// waitCompleteSync re-runs the sync command on node id until it reports +// complete, as the membership-change procedure does. A sync gathers only +// from the peers a node sees ALIVE, so a report before that may miss only +// restarted, whose promotion has not reached every peer yet; any other +// unanswered, held or unreached member fails the test, and the complete +// report must show every member reached by every node. +func (c *cluster) waitCompleteSync(id int, restarted ...uint64) autoIncSyncAnswer { + c.t.Helper() + var answer autoIncSyncAnswer + waitFor(c.t, fmt.Sprintf("sync on node %d complete", id), replicationDeadline, func() (bool, string) { + answer = c.autoIncSync(id) + for _, n := range answer.Nodes { + if !answer.Complete && slices.Contains(restarted, n.NodeID) { + continue } - if time.Now().After(deadline) { - harness.dumpNodeLogs("quorum_writes") - t.Fatalf("node %d cannot commit a replicated write: %v", nodeID, err) + if !n.Answered || n.VotesHeld { + c.t.Fatalf("node %d did not answer the sync or is held: %+v", n.NodeID, answer) } - time.Sleep(time.Second) - } - } -} - -// insertOnEvery inserts k rows on each node and records their ids. -func insertOnEvery(t *testing.T, harness *ClusterHarness, ledger idLedger, table, phase string, k int, nodes ...int) { - t.Helper() - for _, nodeID := range nodes { - conn := openTimedNodeDatabase(t, harness, nodeID, "marmot") - for i := 0; i < k; i++ { - who := fmt.Sprintf("%s node %d #%d", phase, nodeID, i) - id, err := insertNarrowWithin(conn, 60*time.Second, "INSERT INTO "+table+" (v) VALUES (?)", who) - if err != nil { - harness.dumpNodeLogs(phase) - t.Fatalf("%s: %v", who, err) + for _, member := range n.Members { + if !slices.Contains(n.Reached, member) && (answer.Complete || !slices.Contains(restarted, member)) { + c.t.Fatalf("node %d did not reach member %d in a sync that reported complete=%v: %+v", n.NodeID, member, answer.Complete, answer) + } } - ledger.record(t, id, who) } - } + return answer.Complete, fmt.Sprintf("%+v", answer) + }) + return answer } // TestNarrowAutoInc_HeldNodeReleasesAfterItsSeedReturns: a wiped node whose -// only seed is down, so its join fails and its catch-up strategy cannot use -// the seed, is held while it cannot reach enough members, and releases once -// its seed is back. That needs it to connect to the returned seed, which it -// learned of as ALIVE from another member; a node that never connected -// stayed held. +// only seed is down - so its join fails and its catch-up cannot use the seed +// - is held while it cannot reach enough members: the sync reports it held +// and incomplete, and a narrow insert through it returns 1205. It releases +// once its seed is back, which needs it to connect to the returned seed it +// learned of as ALIVE from another member. // // Mutations: never hold (both hold sites) - "was not held while its seed was -// down" fires; drop the ALIVE callback on discovery (node_registry.go) - "did -// not release after its seed returned" fires. +// down" fires; drop the ALIVE callback on discovery (node_registry.go) - the +// cluster never becomes ready after the seed returns. func TestNarrowAutoInc_HeldNodeReleasesAfterItsSeedReturns(t *testing.T) { const table = "seedback" - harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - defer harness.Cleanup() - createQuorumProbe(t, harness) - ledger := idLedger{} - insertOnEvery(t, harness, ledger, table, "before", 3, 1, 2, 3) - - wipeNode(t, harness, 3) - if err := harness.StopNode(1); err != nil { - t.Fatalf("StopNode(1): %v", err) - } - if err := harness.StartNode(3); err != nil { - t.Fatalf("StartNode(3): %v", err) - } - if err := harness.WaitForAlive(3, 90*time.Second); err != nil { - t.Fatalf("node 3 did not come back: %v", err) - } - if err := harness.WaitForTableExists(table, []int{3}, 60*time.Second); err != nil { - harness.dumpNodeLogs("seedback_table") - t.Fatalf("node 3 did not restore %s: %v", table, err) - } - - // With node 1 down node 3 hears one unheld member of three: it must stay - // held, and its narrow inserts wait. - answer, err := runAutoIncSync(harness, 3) - if err != nil { - t.Fatal(err) - } - self, ok := answer.node(3) - if !ok || !self.VotesHeld { - harness.dumpNodeLogs("seedback_held") - t.Fatalf("node 3 was not held while its seed was down: %+v", answer) - } - if answer.Complete { - t.Fatalf("the sync reported complete with node 1 down: %+v", answer) - } - if _, err := execTimed(openTimedNodeDatabase(t, harness, 3, "marmot"), "INSERT INTO "+table+" (v) VALUES ('held')"); !isLockTimeout(err) { + c := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + ledger := newIDLedger(t) + insertOnEvery(c, ledger, table, "before", 3, 1, 2, 3) + c.wipe(3) + c.kill(1) + c.startNode(3) + waitFor(t, "node 3 answers MySQL", readyDeadline, func() (bool, string) { + err := c.ping(3) + return err == nil, fmt.Sprint(err) + }) + c.waitTable("marmot", table, 3) + c.phase("node-3-rebuilt-seed-down") + + answer := c.autoIncSync(3) + if self, ok := answer.node(3); !ok || !self.VotesHeld || answer.Complete { + t.Fatalf("node 3 was not held while its seed was down, or the sync reported complete: %+v", answer) + } + if _, err := c.exec(3, "marmot", "INSERT INTO "+table+" (v) VALUES ('held')"); mysqlCode(err) != mysqlcode.ErrCodeLockTimeout { t.Fatalf("a narrow insert on held node 3 returned %v, want 1205", err) } - if err := harness.StartNode(1); err != nil { - t.Fatalf("StartNode(1): %v", err) - } - if err := harness.WaitForAlive(1, 90*time.Second); err != nil { - t.Fatalf("node 1 did not come back: %v", err) - } - waitForRelease(t, harness, 3, 90*time.Second, "did not release after its seed returned") - - waitForQuorumWrites(t, harness, 120*time.Second, 1, 2, 3) - insertOnEvery(t, harness, ledger, table, "after", 3, 1, 2, 3) - waitForCompleteSync(t, harness, 2, 3) -} - -// syncCompletePromotions is how many JOINING-to-ALIVE promotion windows a -// complete sync may take after a restart: a sync gathers only from the peers -// a node sees ALIVE, and a restarted node is ALIVE to its peers once it has -// been promoted and gossip has carried that. -const syncCompletePromotions = 6 - -// syncRetryInterval spaces the sync command's re-runs. -const syncRetryInterval = time.Second - -// syncCompleteDeadline bounds waitForCompleteSync: syncCompletePromotions -// promotion windows (check interval plus minimum healthy time) of the -// configuration the harness's nodes run with. -func syncCompleteDeadline() time.Duration { - p := cfg.Config.Cluster.Promotion - return syncCompletePromotions * time.Duration(p.CheckIntervalSeconds+p.MinHealthyDurationSec) * time.Second + c.restart(1) + c.phase("seed-returned") + insertOnEvery(c, ledger, table, "after", 3, 1, 2, 3) + c.waitCompleteSync(2, 1) + c.waitRows("marmot", idStats(table, "id"), ledger.stats(), 1, 2, 3) } -// waitForCompleteSync runs the sync command on nodeID until it reports -// complete, as the membership-change procedure does: re-run until the report -// is complete. Every report before that may miss only restarted, the member -// whose promotion to ALIVE has not reached every peer yet; any other -// unanswered, held or unreached member fails the test, and a complete report -// must show every member reached by every node. -func waitForCompleteSync(t *testing.T, harness *ClusterHarness, nodeID int, restarted uint64) { - t.Helper() - deadline := time.Now().Add(syncCompleteDeadline()) - for { - answer, err := runAutoIncSync(harness, nodeID) - if err != nil { - t.Fatal(err) - } - for _, n := range answer.Nodes { - if !n.Answered || n.VotesHeld { - t.Fatalf("node %d did not answer the sync or is held: %+v", n.NodeID, answer) - } - reached := make(map[uint64]bool, len(n.Reached)) - for _, id := range n.Reached { - reached[id] = true - } - for _, member := range n.Members { - if reached[member] { - continue - } - if answer.Complete || member != restarted { - t.Fatalf("node %d did not reach member %d in a sync that reported complete=%v: %+v", - n.NodeID, member, answer.Complete, answer) - } - } - } - if answer.Complete { - return - } - if time.Now().After(deadline) { - harness.dumpNodeLogs("sync_incomplete") - t.Fatalf("the sync with every member back is not complete after %s: %+v", syncCompleteDeadline(), answer) - } - time.Sleep(syncRetryInterval) +// highestID is the largest id l recorded. +func (l *idLedger) highestID() int64 { + var highest int64 + for id := range l.seen { + highest = max(highest, id) } + return highest } -// TestNarrowAutoInc_WipedSeedAndItsJoinerHold: the seed node, which has no seeds of its own, loses its data -// directory, and so does a node whose only seed it is, which therefore -// catches up from an empty seed. Both restart held and release only by -// merging claim bases from the member that kept them, and the sync command -// then finds every member unheld and reachable. -// -// It stops there: with two of three nodes rebuilt, a new write needs one of -// them, and a rebuilt node refuses writes until anti-entropy restores its -// schema version (the pre-existing catch-up gap, a separate step). +// TestNarrowAutoInc_WipedSeedHoldsUntilMerged: the seed node, which has no +// seeds of its own, loses its data directory - one lost member, inside the +// failure model autoIncMergeSafe assumes. It comes back knowing only itself +// and reports its votes held (over gRPC) while no other member answers it; +// once the members that kept their bases return, it merges and releases, the +// first id it then issues lies above every id committed before the wipe, and +// it catches up every row from their commit logs. The other two stop +// gracefully (SIGTERM): a SIGKILLed node may lose the unsynced tail of its +// commit log, which peers then cannot pull from it, and three failures at +// once are outside the model this test is about. // -// Mutation: never hold (both hold sites). "node 1 was never held" fires. -func TestNarrowAutoInc_WipedSeedAndItsJoinerHold(t *testing.T) { +// Mutation: never hold (both hold sites). "not held" fires. +func TestNarrowAutoInc_WipedSeedHoldsUntilMerged(t *testing.T) { const table = "wipedseed" - harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - defer harness.Cleanup() - insertOnEvery(t, harness, idLedger{}, table, "before", 5, 1, 2, 3) + c := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + ledger := newIDLedger(t) + insertOnEvery(c, ledger, table, "before", 5, 1, 2, 3) + c.stop(2) + c.stop(3) + c.wipe(1) + c.startNode(1) + waitFor(t, "wiped node 1 answers", readyDeadline, func() (bool, string) { + _, err := c.votesHeld(1) + return err == nil, fmt.Sprint(err) + }) + if held, err := c.votesHeld(1); err != nil || !held { + t.Fatalf("wiped seed node 1 is not held while no other member answers it (held %v, err %v)", held, err) + } + c.phase("wiped-seed-held") + + c.startNode(2) + c.startNode(3) + c.waitReady() + c.waitTable("marmot", table, 1) + id, err := insertNarrow(c, 1, "marmot", "INSERT INTO "+table+" (v) VALUES ('after')") + ledger.record("rebuilt node 1", id, err) + if highest := ledger.highestID(); id != highest { + t.Fatalf("rebuilt node 1 issued %d, not above the ids committed before the wipe", id) + } + c.waitRows("marmot", idStats(table, "id"), ledger.stats(), 1, 2, 3) +} - wipeNode(t, harness, 1) - wipeNode(t, harness, 3) - for _, nodeID := range []int{1, 3} { - if err := harness.StartNode(nodeID); err != nil { - t.Fatalf("StartNode(%d): %v", nodeID, err) - } - if err := harness.WaitForAlive(nodeID, 90*time.Second); err != nil { - t.Fatalf("node %d did not come back: %v", nodeID, err) - } +// TestNarrowAutoInc_TwoWipedNodesNeverDuplicate: the seed node and a joiner +// whose only seed it is both lose their data directories. Two of three +// members lost claim state, outside the failure model autoIncMergeSafe +// assumes: the two may count each other as the whole membership and release +// before they hear from the member that kept the bases. What is owed there +// is the backstop: an id they reissue is refused with 1062 (the row it would +// collide with is already in the table), never committed twice, and once +// the base sync has raised their bases, inserts through them succeed. +func TestNarrowAutoInc_TwoWipedNodesNeverDuplicate(t *testing.T) { + const table = "twowiped" + c := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + ledger := newIDLedger(t) + insertOnEvery(c, ledger, table, "before", 5, 1, 2, 3) + c.wipe(1) + c.wipe(3) + c.startNode(1) + c.startNode(3) + c.waitReady() + c.waitTable("marmot", table, 1, 3) + c.phase("rebuilt") + + const wanted = 3 + for _, node := range []int{1, 3} { + refused, done := 0, 0 + waitFor(t, fmt.Sprintf("%d inserts through rebuilt node %d", wanted, node), replicationDeadline, func() (bool, string) { + who := fmt.Sprintf("rebuilt node %d #%d", node, done) + id, err := insertNarrow(c, node, "marmot", "INSERT INTO "+table+" (v) VALUES (?)", who) + if code := mysqlCode(err); err != nil && code != mysqlcode.ErrCodeDupEntry { + t.Fatalf("%s: %v, want success or 1062", who, err) + } + if err != nil { + refused++ + return false, fmt.Sprintf("%d refused with 1062", refused) + } + ledger.record(who, id, nil) + done++ + return done == wanted, fmt.Sprintf("%d inserted, %d refused with 1062", done, refused) + }) + t.Logf("rebuilt node %d: %d inserts refused with 1062 before the base sync", node, refused) } - waitForRelease(t, harness, 1, 90*time.Second, "node 1 was never held, or never released") - waitForRelease(t, harness, 3, 90*time.Second, "node 3 was never held, or never released") - - deadline := time.Now().Add(60 * time.Second) - for { - answer, err := runAutoIncSync(harness, 2) - if err != nil { - t.Fatal(err) - } - if answer.Complete { - break - } - if time.Now().After(deadline) { - harness.dumpNodeLogs("wipedseed_sync") - t.Fatalf("the sync never found every member unheld and reachable: %+v", answer) - } - time.Sleep(2 * time.Second) + var want []string + for _, id := range slices.Sorted(maps.Keys(ledger.seen)) { + want = append(want, fmt.Sprintf("%d|%s", id, ledger.seen[id])) } + c.waitRows("marmot", "SELECT id, v FROM "+table+" ORDER BY id", want, 1, 2, 3) } -// TestNarrowAutoInc_SyncAfterAddingAMember is the membership-change procedure -// on a growing cluster: a fourth node joins, becomes ALIVE and -// releases, the operator runs the sync command, and it reports every member -// answered, unheld and reaching every member. Ids stay unique across all four. +// TestNarrowAutoInc_SyncAfterAddingAMember is the membership-change +// procedure on a growing cluster: a fourth node joins, becomes ALIVE and +// releases, the operator runs the sync command, and it reports all four +// members answered, unheld and reaching every member. Ids stay unique across +// all four. // -// Mutation: syncAutoIncBasesEverywhere asks no other member. "the sync did -// not reach every member" fires. +// Mutation: syncAutoIncBasesEverywhere asks no other member. The sync never +// reports four members reaching every member. func TestNarrowAutoInc_SyncAfterAddingAMember(t *testing.T) { const table = "grow" - harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - defer harness.Cleanup() - ledger := idLedger{} - insertOnEvery(t, harness, ledger, table, "three", 3, 1, 2, 3) - - harness.Nodes = append(harness.Nodes, harness.createNode(4)) - if err := harness.StartNode(4); err != nil { - t.Fatalf("StartNode(4): %v", err) - } - if err := harness.WaitForAlive(4, 90*time.Second); err != nil { - t.Fatalf("node 4 did not come up: %v", err) - } - waitForRelease(t, harness, 4, 90*time.Second, "the new member never released its votes") - if err := harness.WaitForTableExists(table, []int{4}, 90*time.Second); err != nil { - harness.dumpNodeLogs("grow_table") - t.Fatalf("node 4 did not restore %s: %v", table, err) + c := newCluster(t, func(cfg *clusterConfig) { cfg.size = 4 }) + c.startNodes(1, 2, 3) + c.createTable(1, "marmot", table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + ledger := newIDLedger(t) + insertOnEvery(c, ledger, table, "three", 3, 1, 2, 3) + + c.restart(4) + c.waitTable("marmot", table, 4) + answer := c.waitCompleteSync(1, 4) + if len(answer.Nodes) != 4 { + t.Fatalf("the complete sync covers %d members, want 4: %+v", len(answer.Nodes), answer) + } + for _, n := range answer.Nodes { + if len(n.Members) != 4 || !n.ReachedEveryMember { + t.Fatalf("node %d: complete, yet reached %v of %v", n.NodeID, n.Reached, n.Members) + } } + insertOnEvery(c, ledger, table, "four", 3, 1, 2, 3, 4) + c.waitRows("marmot", idStats(table, "id"), ledger.stats(), 1, 2, 3, 4) +} - deadline := time.Now().Add(60 * time.Second) - for { - answer, err := runAutoIncSync(harness, 1) - if err != nil { - t.Fatal(err) - } - if answer.Complete && len(answer.Nodes) == 4 { - for _, n := range answer.Nodes { - if len(n.Members) != 4 || !n.ReachedEveryMember { - t.Fatalf("node %d: complete, yet reached %v of %v", n.NodeID, n.Reached, n.Members) - } - } - break - } - if time.Now().After(deadline) { - harness.dumpNodeLogs("grow_sync") - t.Fatalf("the sync did not reach every member: %+v", answer) +// insertOnEvery inserts k generated rows through each of nodes and records +// their ids in ledger. +func insertOnEvery(c *cluster, ledger *idLedger, table, phase string, k int, nodes ...int) { + c.t.Helper() + for _, node := range nodes { + for i := 0; i < k; i++ { + who := fmt.Sprintf("%s node %d #%d", phase, node, i) + id, err := insertNarrow(c, node, "marmot", "INSERT INTO "+table+" (v) VALUES (?)", who) + ledger.record(who, id, err) } - time.Sleep(2 * time.Second) } - insertOnEvery(t, harness, ledger, table, "four", 3, 1, 2, 3, 4) } diff --git a/test/autoinc_membership_inprocess_test.go b/test/autoinc_membership_inprocess_test.go index 46223603..93abb9a0 100644 --- a/test/autoinc_membership_inprocess_test.go +++ b/test/autoinc_membership_inprocess_test.go @@ -40,7 +40,7 @@ func TestClaimRange_JoinersHoldVotesUntilMerged(t *testing.T) { require.Equal(t, uint64(0), base3, "n3 must have missed C for the sequence to mean anything") for _, id := range []uint64{4, 5} { - joiner := c.buildNode(id, t.TempDir()) + joiner := c.buildNode(id, freshDir(t)) c.nodes[id] = joiner require.NoError(t, joiner.dm.CreateDatabase(database)) mdb, err := joiner.dm.GetDatabase(database) @@ -89,7 +89,7 @@ func TestClaimRange_RebuiltNodeHoldsVotesUntilMerged(t *testing.T) { c.fanout.setUnreachable(3, false) c.closeNode(1) - rebuilt := c.buildNode(1, t.TempDir()) + rebuilt := c.buildNode(1, freshDir(t)) c.nodes[1] = rebuilt require.NoError(t, rebuilt.dm.CreateDatabase(database)) mdb, err := rebuilt.dm.GetDatabase(database) diff --git a/test/autoinc_narrow_cluster_test.go b/test/autoinc_narrow_cluster_test.go index 8fe4adf5..6a8c7f4f 100644 --- a/test/autoinc_narrow_cluster_test.go +++ b/test/autoinc_narrow_cluster_test.go @@ -2,182 +2,132 @@ package test // Cluster tests for narrow AUTO_INCREMENT columns on the insert path: every // INSERT into an INT/TINYINT AUTO_INCREMENT table takes its id from a range -// claimed cluster-wide (id.RangeAllocator over coordinator.ClaimRange), -// driven over the real MySQL wire protocol against a 3-node ClusterHarness -// (test/crash_recovery_test.go). +// claimed cluster-wide (id.RangeAllocator over coordinator.ClaimRange). import ( "database/sql" - "errors" "fmt" + "maps" "path/filepath" - "strings" + "slices" "sync" "testing" "time" - "github.com/go-sql-driver/mysql" + _ "github.com/mattn/go-sqlite3" "github.com/maxpert/marmot/db" "github.com/maxpert/marmot/protocol/mysqlcode" ) -const narrowInt32Max = 2147483647 +const ( + narrowInt32Max = 2147483647 + // claimRetryDeadline bounds insertNarrow's retries of 1205, the + // retryable answer to a claim that lost a race for the claim key. + claimRetryDeadline = 3 * time.Second +) -// startNarrowCluster starts a 3-node cluster and creates table on it. opts -// configure the harness (e.g. GCIntervalSeconds) before any node config is -// written. -func startNarrowCluster(t *testing.T, table, ddl string, opts ...func(*ClusterHarness)) *ClusterHarness { +// startNarrowCluster starts a cluster and creates table with ddl on it. +func startNarrowCluster(t *testing.T, table, ddl string) *cluster { t.Helper() - harness := NewClusterHarness(t, opts...) - if err := harness.StartCluster(); err != nil { - harness.Cleanup() - t.Fatalf("StartCluster: %v", err) - } - if _, err := harness.ExecNode(1, ddl); err != nil { - harness.Cleanup() - t.Fatalf("CREATE TABLE %s: %v", table, err) - } - if err := harness.WaitForTableExists(table, []int{1, 2, 3}, 15*time.Second); err != nil { - harness.dumpNodeLogs("narrow_" + table) - harness.Cleanup() - t.Fatalf("DDL did not replicate: %v", err) - } - return harness + c := newCluster(t) + c.start() + c.createTable(1, "marmot", table, ddl) + return c } -// insertNarrow inserts one row on nodeID, retrying the retryable lock-wait -// timeout a claim returns while the cluster settles, and returns the id. -func insertNarrow(harness *ClusterHarness, nodeID int, query string, args ...interface{}) (int64, error) { - var lastErr error - for attempt := 0; attempt < 20; attempt++ { - res, err := harness.ExecNode(nodeID, query, args...) - if err == nil { - return res.LastInsertId() - } - lastErr = err - var mysqlErr *mysql.MySQLError - if !errors.As(err, &mysqlErr) || mysqlErr.Number != mysqlcode.ErrCodeLockTimeout { - return 0, err - } - time.Sleep(250 * time.Millisecond) +// insertNarrow runs a single-row INSERT on database through node id and +// returns its generated id, retrying 1205 (a claim that lost a race for the +// claim key) for up to claimRetryDeadline. It never fails the test itself, +// so writer goroutines can call it. +func insertNarrow(c *cluster, id int, database, q string, args ...any) (int64, error) { + var res sql.Result + var err error + state, ok := poll(claimRetryDeadline, func() (bool, string) { + res, err = c.exec(id, database, q, args...) + return err == nil || mysqlCode(err) != mysqlcode.ErrCodeLockTimeout, fmt.Sprint(err) + }) + if !ok { + return 0, fmt.Errorf("node %d: %s: still 1205 after %s: %s", id, q, claimRetryDeadline, state) } - return 0, lastErr -} - -// narrowIDStats returns COUNT(*), COUNT(DISTINCT id) and MAX(id) on nodeID. -func narrowIDStats(harness *ClusterHarness, nodeID int, table, col string) (count, distinct, maxID int64, err error) { - rows, err := harness.QueryNode(nodeID, fmt.Sprintf("SELECT COUNT(*), COUNT(DISTINCT %s), MAX(%s) FROM %s", col, col, table)) if err != nil { - return 0, 0, 0, err - } - defer rows.Close() - if !rows.Next() { - return 0, 0, 0, fmt.Errorf("no row from %s", table) + return 0, err } - err = rows.Scan(&count, &distinct, &maxID) - return count, distinct, maxID, err + return res.LastInsertId() } // TestNarrowAutoInc_ConcurrentInsertsFitAndNeverCollide: three nodes insert -// concurrently into one INT AUTO_INCREMENT table. Every id must fit 32 bits -// and no id may be issued twice - the two properties the 53-bit ids of -// GitHub issue #170 broke and node-local rowids would break. +// concurrently into one INT AUTO_INCREMENT table. Every id fits 32 bits, no +// id is issued twice, and every node ends with exactly the ids issued - the +// properties the 53-bit ids of GitHub issue #170 broke and node-local rowids +// would break. func TestNarrowAutoInc_ConcurrentInsertsFitAndNeverCollide(t *testing.T) { - const table = "narrow_concurrent" - harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (group_id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)") - defer harness.Cleanup() + const table, perNode = "narrow_concurrent", 40 + c := startNarrowCluster(t, table, "CREATE TABLE "+table+" (group_id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)") - const perNode = 40 + var mu sync.Mutex + issued := map[int64]string{} var wg sync.WaitGroup errs := make(chan error, 3*perNode) - for nodeID := 1; nodeID <= 3; nodeID++ { + for node := 1; node <= 3; node++ { wg.Add(1) - go func(nodeID int) { + go func(node int) { defer wg.Done() for i := 0; i < perNode; i++ { - id, err := insertNarrow(harness, nodeID, "INSERT INTO "+table+" (name) VALUES (?)", fmt.Sprintf("n%d-%d", nodeID, i)) + name := fmt.Sprintf("n%d-%d", node, i) + id, err := insertNarrow(c, node, "marmot", "INSERT INTO "+table+" (name) VALUES (?)", name) if err != nil { - errs <- fmt.Errorf("node %d insert %d: %w", nodeID, i, err) + errs <- err return } - if id <= 0 || id > narrowInt32Max { - errs <- fmt.Errorf("node %d issued id %d, outside [1, %d]", nodeID, id, narrowInt32Max) + mu.Lock() + if prev, dup := issued[id]; dup || id <= 0 || id > narrowInt32Max { + errs <- fmt.Errorf("node %d issued id %d for %s (already issued for %q, or outside [1, %d])", node, id, name, prev, narrowInt32Max) } + issued[id] = name + mu.Unlock() } - }(nodeID) + }(node) } wg.Wait() close(errs) for err := range errs { - t.Error(err) - } - if t.Failed() { - harness.dumpNodeLogs("narrow_concurrent") - t.FailNow() - } - - if err := harness.WaitForRowCount(table, []int{1, 2, 3}, 3*perNode, 30*time.Second); err != nil { - harness.dumpNodeLogs("narrow_concurrent_rows") - t.Fatalf("rows did not replicate: %v", err) + t.Fatal(err) } - for nodeID := 1; nodeID <= 3; nodeID++ { - count, distinct, maxID, err := narrowIDStats(harness, nodeID, table, "group_id") - if err != nil { - t.Fatalf("node %d: %v", nodeID, err) - } - if count != 3*perNode || distinct != count { - t.Fatalf("node %d: COUNT(*)=%d COUNT(DISTINCT group_id)=%d, want both %d", nodeID, count, distinct, 3*perNode) - } - if maxID > narrowInt32Max { - t.Fatalf("node %d: MAX(group_id)=%d does not fit INT", nodeID, maxID) - } + var want []string + for _, id := range slices.Sorted(maps.Keys(issued)) { + want = append(want, fmt.Sprintf("%d|%s", id, issued[id])) } + c.waitRows("marmot", "SELECT group_id, name FROM "+table+" ORDER BY group_id", want, 1, 2, 3) } // TestNarrowAutoInc_KillNineNeverReissues: a node killed with SIGKILL in the -// middle of a range loses its unissued tail as a gap; after restart it -// claims a fresh range and never issues an id it issued before. +// middle of a range loses its unissued tail as a gap; after it restarts it +// claims a fresh range and never issues an id at or below one it issued +// before. func TestNarrowAutoInc_KillNineNeverReissues(t *testing.T) { const table = "narrow_kill9" - harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - defer harness.Cleanup() - - issued := make(map[int64]bool) + c := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") var highest int64 for i := 0; i < 10; i++ { - id, err := insertNarrow(harness, 2, "INSERT INTO "+table+" (v) VALUES ('before')") + id, err := insertNarrow(c, 2, "marmot", "INSERT INTO "+table+" (v) VALUES ('before')") if err != nil { t.Fatalf("insert before kill: %v", err) } - issued[id] = true highest = max(highest, id) } - - if err := harness.KillNode(2); err != nil { - t.Fatalf("KillNode: %v", err) - } - if err := harness.StartNode(2); err != nil { - t.Fatalf("StartNode: %v", err) - } - if err := harness.WaitForAlive(2, 60*time.Second); err != nil { - harness.dumpNodeLogs("narrow_kill9_restart") - t.Fatalf("node 2 did not come back: %v", err) - } - + c.kill(2) + c.restart(2) for i := 0; i < 10; i++ { - id, err := insertNarrow(harness, 2, "INSERT INTO "+table+" (v) VALUES ('after')") + id, err := insertNarrow(c, 2, "marmot", "INSERT INTO "+table+" (v) VALUES ('after')") if err != nil { - harness.dumpNodeLogs("narrow_kill9_after") t.Fatalf("insert after restart: %v", err) } - if issued[id] || id <= highest { + if id <= highest || id > narrowInt32Max { t.Fatalf("restarted node issued %d; ids up to %d were issued before the kill", id, highest) } - if id > narrowInt32Max { - t.Fatalf("id %d does not fit INT", id) - } - issued[id] = true } + c.waitRows("marmot", idStats(table, "id")+" WHERE v = 'before'", []string{fmt.Sprintf("10|10|%d", highest)}, 1, 2, 3) + c.waitRows("marmot", "SELECT COUNT(*), COUNT(DISTINCT id) FROM "+table, []string{"20|20"}, 1, 2, 3) } // TestNarrowAutoInc_TinyIntExhaustionIsDupEntry: a TINYINT AUTO_INCREMENT @@ -185,83 +135,57 @@ func TestNarrowAutoInc_KillNineNeverReissues(t *testing.T) { // error MySQL reports, never the retryable 1205. func TestNarrowAutoInc_TinyIntExhaustionIsDupEntry(t *testing.T) { const table = "narrow_tiny" - harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id TINYINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - defer harness.Cleanup() - + c := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id TINYINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") for i := 1; i <= 127; i++ { - if _, err := insertNarrow(harness, 1, "INSERT INTO "+table+" (v) VALUES ('x')"); err != nil { - harness.dumpNodeLogs("narrow_tiny_fill") + if _, err := insertNarrow(c, 1, "marmot", "INSERT INTO "+table+" (v) VALUES ('x')"); err != nil { t.Fatalf("insert %d of 127: %v", i, err) } } - - _, err := harness.ExecNode(1, "INSERT INTO "+table+" (v) VALUES ('overflow')") - var mysqlErr *mysql.MySQLError - if !errors.As(err, &mysqlErr) || mysqlErr.Number != mysqlcode.ErrCodeDupEntry { + _, err := c.exec(1, "marmot", "INSERT INTO "+table+" (v) VALUES ('overflow')") + if code := mysqlCode(err); code != mysqlcode.ErrCodeDupEntry { t.Fatalf("insert past 127 into TINYINT: err = %v, want MySQL error %d", err, mysqlcode.ErrCodeDupEntry) } + c.waitRows("marmot", idStats(table, "id"), []string{"127|127|127"}, 1, 2, 3) } // TestNarrowAutoInc_LLDAPTransactionShape: LLDAP boots with one transaction -// inserting into two narrow tables that both need their first claim. It must -// complete, and each INSERT's LAST_INSERT_ID must be its own first id. +// inserting into two narrow tables that both need their first claim. It +// commits, and each INSERT's LAST_INSERT_ID is its own first id. func TestNarrowAutoInc_LLDAPTransactionShape(t *testing.T) { - harness := startNarrowCluster(t, "users", "CREATE TABLE users (user_id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)") - defer harness.Cleanup() - if _, err := harness.ExecNode(1, "CREATE TABLE `groups` (group_id INT AUTO_INCREMENT PRIMARY KEY, display_name TEXT)"); err != nil { - t.Fatalf("CREATE TABLE groups: %v", err) - } - if err := harness.WaitForTableExists("groups", []int{1, 2, 3}, 15*time.Second); err != nil { - t.Fatalf("groups did not replicate: %v", err) - } + c := startNarrowCluster(t, "users", "CREATE TABLE users (user_id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)") + c.createTable(1, "marmot", "groups", "CREATE TABLE `groups` (group_id INT AUTO_INCREMENT PRIMARY KEY, display_name TEXT)") - conn, err := harness.ConnectToNode(2) + tx, err := c.db(2, "marmot").Begin() if err != nil { - t.Fatalf("ConnectToNode: %v", err) - } - tx, err := conn.Begin() - if err != nil { - t.Fatalf("BEGIN: %v", err) + t.Fatal(err) } usersRes, err := tx.Exec("INSERT INTO users (name) VALUES (?)", "admin") if err != nil { - _ = tx.Rollback() t.Fatalf("INSERT users: %v", err) } groupsRes, err := tx.Exec("INSERT INTO `groups` (display_name) VALUES (?), (?)", "lldap_admin", "lldap_password_manager") if err != nil { - _ = tx.Rollback() t.Fatalf("INSERT groups: %v", err) } if err := tx.Commit(); err != nil { t.Fatalf("COMMIT: %v", err) } - userID, _ := usersRes.LastInsertId() groupID, _ := groupsRes.LastInsertId() - var storedUser, firstGroup, groupCount int64 - if err := conn.QueryRow("SELECT user_id FROM users WHERE name = 'admin'").Scan(&storedUser); err != nil { - t.Fatalf("read users: %v", err) - } - if err := conn.QueryRow("SELECT MIN(group_id), COUNT(*) FROM `groups`").Scan(&firstGroup, &groupCount); err != nil { - t.Fatalf("read groups: %v", err) - } - if userID != storedUser || groupID != firstGroup || groupCount != 2 { - t.Fatalf("LAST_INSERT_ID users=%d (stored %d), groups=%d (first stored %d, %d rows)", userID, storedUser, groupID, firstGroup, groupCount) - } - if storedUser > narrowInt32Max || firstGroup+1 > narrowInt32Max { - t.Fatalf("ids do not fit INT: user %d, groups from %d", storedUser, firstGroup) + if userID <= 0 || groupID <= 0 || groupID+1 > narrowInt32Max || userID > narrowInt32Max { + t.Fatalf("LAST_INSERT_ID users=%d groups=%d, want ids in [1, %d]", userID, groupID, narrowInt32Max) } + c.waitRows("marmot", "SELECT user_id, name FROM users ORDER BY user_id", []string{fmt.Sprintf("%d|admin", userID)}, 1, 2, 3) + c.waitRows("marmot", "SELECT group_id, display_name FROM `groups` ORDER BY group_id", + []string{fmt.Sprintf("%d|lldap_admin", groupID), fmt.Sprintf("%d|lldap_password_manager", groupID+1)}, 1, 2, 3) } // TestNarrowAutoInc_UnmarkedTableKeepsWideIDs: a BIGINT AUTO_INCREMENT table -// carries no width marker and keeps the 53-bit ids it always had. +// carries no width marker and keeps its 53-bit generated ids. func TestNarrowAutoInc_UnmarkedTableKeepsWideIDs(t *testing.T) { const table = "wide_bigint" - harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id BIGINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - defer harness.Cleanup() - - id, err := insertNarrow(harness, 1, "INSERT INTO "+table+" (v) VALUES ('x')") + c := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id BIGINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + id, err := insertNarrow(c, 1, "marmot", "INSERT INTO "+table+" (v) VALUES ('x')") if err != nil { t.Fatalf("insert: %v", err) } @@ -271,189 +195,117 @@ func TestNarrowAutoInc_UnmarkedTableKeepsWideIDs(t *testing.T) { } // TestNarrowAutoInc_CopiedIDsRaiseTheBaseOnEveryNode: ids copied into an INT -// AUTO_INCREMENT table with INSERT ... SELECT - MySQL's usual way to -// re-declare a column's width, whose ids are known only once it has run - are -// admitted from the transaction's rows before it replicates, so the first -// generated id on EVERY node lands above the copied ones. +// AUTO_INCREMENT table with INSERT ... SELECT are admitted from the +// transaction's rows before it replicates, so the first generated id on +// every node lands above the copied ones. // -// Mutation: skip the admission of the transaction's rows -// (coordinator.admitNarrowIDs). The first generated id then reuses a copied -// one and the insert fails or "landed at or below the copied MAX" fires. +// Mutation: skip coordinator.admitNarrowIDs. The first generated id then +// reuses a copied one: the insert fails or lands at or below the copied MAX. func TestNarrowAutoInc_CopiedIDsRaiseTheBaseOnEveryNode(t *testing.T) { const source, table, copied = "copy_source", "copy_target", 300 - harness := startNarrowCluster(t, source, "CREATE TABLE "+source+" (id BIGINT PRIMARY KEY, v TEXT)") - defer harness.Cleanup() + c := startNarrowCluster(t, source, "CREATE TABLE "+source+" (id BIGINT PRIMARY KEY, v TEXT)") + fillExplicitIDs(c, source, copied) + c.createTable(1, "marmot", table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + c.mustExec(1, "marmot", "INSERT INTO "+table+" (id, v) SELECT id, v FROM "+source) + c.waitRows("marmot", idStats(table, "id"), []string{fmt.Sprintf("%d|%d|%d", copied, copied, copied)}, 1, 2, 3) + assertFirstGeneratedAbove(c, table, copied) +} - fillExplicitIDs(t, harness, source, copied) - if _, err := harness.ExecNode(1, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)"); err != nil { - t.Fatalf("CREATE TABLE %s: %v", table, err) - } - if err := harness.WaitForTableExists(table, []int{1, 2, 3}, 15*time.Second); err != nil { - t.Fatalf("%s did not replicate: %v", table, err) - } - if _, err := harness.ExecNode(1, "INSERT INTO "+table+" (id, v) SELECT id, v FROM "+source); err != nil { - t.Fatalf("INSERT ... SELECT: %v", err) +// fillExplicitIDs inserts rows 1..n into table through node 1. +func fillExplicitIDs(c *cluster, table string, n int) { + c.t.Helper() + for start := 1; start <= n; start += 100 { + values, _ := idValueRows(start, min(start+99, n), "row") + c.mustExec(1, "marmot", "INSERT INTO "+table+" (id, v) VALUES "+values) } - if err := harness.WaitForRowCount(table, []int{1, 2, 3}, copied, 30*time.Second); err != nil { - t.Fatalf("copied rows did not replicate: %v", err) +} + +// assertFirstGeneratedAbove inserts one generated row through every node and +// requires each id to lie above highest. +func assertFirstGeneratedAbove(c *cluster, table string, highest int64) { + c.t.Helper() + for node := 1; node <= 3; node++ { + id, err := insertNarrow(c, node, "marmot", "INSERT INTO "+table+" (v) VALUES ('generated')") + if err != nil { + c.t.Fatalf("node %d: first generated insert: %v", node, err) + } + if id <= highest { + c.t.Fatalf("node %d: generated id %d landed at or below the copied MAX %d", node, id, highest) + } } - assertFirstGeneratedAbove(t, harness, table, copied) } // TestNarrowAutoInc_SeedFromLegacyRowsOnEveryNode: rows that reached a narrow -// table without ever passing the allocator - written by a binary that -// predates it - are covered by the DDL-time seed. The next DDL on the table -// seeds every node's base from that node's own MAX(id), so the first -// generated id on EVERY node lands above them. The legacy rows are written -// straight into each stopped node's SQLite file, the only way such rows can -// arise now that every client write is admitted. +// table without passing the allocator - written by a binary that predates +// it, here straight into each stopped node's SQLite file - are covered by the +// DDL-time seed: the next DDL seeds every node's base from its own MAX(id), +// so the first generated id on every node lands above them. // -// Mutation: make autoIncSeedFloor ignore MAX(id) and seed only the declared -// floor. The first generated id then reuses a legacy one, and the insert -// fails or "landed at or below the copied MAX" fires. +// Mutation: make autoIncSeedFloor ignore MAX(id). The first generated id +// then reuses a legacy one: the insert fails or lands at or below the MAX. func TestNarrowAutoInc_SeedFromLegacyRowsOnEveryNode(t *testing.T) { const table, legacy = "legacy_rows", 300 - harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - defer harness.Cleanup() - - harness.StopCluster() - for nodeID := 1; nodeID <= 3; nodeID++ { - path := filepath.Join(harness.Nodes[nodeID-1].DataDir, "databases", db.DefaultDatabaseName+".db") + c := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + values, _ := idValueRows(1, legacy, "legacy") + for node := 1; node <= 3; node++ { + c.kill(node) + path := filepath.Join(c.node(node).dir, "databases", db.DefaultDatabaseName+".db") conn, err := sql.Open("sqlite3", path) if err != nil { - t.Fatalf("open node %d database: %v", nodeID, err) - } - for id := 1; id <= legacy; id++ { - if _, err := conn.Exec("INSERT INTO "+table+" (id, v) VALUES (?, 'legacy')", id); err != nil { - conn.Close() - t.Fatalf("write legacy row %d on node %d: %v", id, nodeID, err) - } + t.Fatalf("open node %d database: %v", node, err) } + _, err = conn.Exec("INSERT INTO " + table + " (id, v) VALUES " + values) conn.Close() - } - for nodeID := 1; nodeID <= 3; nodeID++ { - if err := harness.StartNode(nodeID); err != nil { - t.Fatalf("StartNode %d: %v", nodeID, err) - } - } - for nodeID := 1; nodeID <= 3; nodeID++ { - if err := harness.WaitForAlive(nodeID, 60*time.Second); err != nil { - harness.dumpNodeLogs("narrow_legacy_restart") - t.Fatalf("node %d did not come back: %v", nodeID, err) + if err != nil { + t.Fatalf("write legacy rows on node %d: %v", node, err) } } - - if _, err := harness.ExecNode(1, "ALTER TABLE "+table+" ADD COLUMN note TEXT"); err != nil { - t.Fatalf("ALTER TABLE ADD COLUMN: %v", err) - } - for nodeID := 1; nodeID <= 3; nodeID++ { - waitForAutoIncBaseAtLeast(t, harness.Nodes[nodeID-1].DataDir, db.DefaultDatabaseName, table, legacy, 15*time.Second) - } - assertFirstGeneratedAbove(t, harness, table, legacy) + c.start() + c.mustExec(1, "marmot", "ALTER TABLE "+table+" ADD COLUMN note TEXT") + c.waitAutoIncBaseAtLeast("marmot", table, legacy, 1, 2, 3) + assertFirstGeneratedAbove(c, table, legacy) } -// TestNarrowAutoInc_IDInAnotherNodesRangeIsRefused: node 1 claims a range -// and issues its first id. An explicit id inside node 1's unissued range, -// written through node 2 - as a literal, as a bound parameter, or copied -// with INSERT ... SELECT - is refused with 1235 before anything is written; -// were it accepted, node 1 would later issue the same id and the replicated -// row would replace the client's on every node. Node 1 then issues ids past -// it and the table never holds the client's row. +// TestNarrowAutoInc_IDInAnotherNodesRangeIsRefused: node 1 claims a range and +// issues its first id. An explicit id inside node 1's unissued range written +// through node 2 - as a literal, a bound parameter, or copied with INSERT ... +// SELECT - is refused with 1235 before anything is written; were it +// accepted, node 1 would later issue the same id and replace the client's +// row on every node. Node 1 then issues ids past it and no node ever holds a +// client row. // -// Mutation: admit any id at or below the base. Node 2's writes succeed and -// "was admitted" fires. +// Mutation: admit any id at or below the base. Node 2's writes succeed. func TestNarrowAutoInc_IDInAnotherNodesRangeIsRefused(t *testing.T) { const table, source = "foreign_range", "foreign_source" - harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - defer harness.Cleanup() - if _, err := harness.ExecNode(1, "CREATE TABLE "+source+" (id BIGINT PRIMARY KEY, v TEXT)"); err != nil { - t.Fatalf("CREATE TABLE %s: %v", source, err) - } - if err := harness.WaitForTableExists(source, []int{1, 2, 3}, 15*time.Second); err != nil { - t.Fatalf("%s did not replicate: %v", source, err) - } - - first, err := insertNarrow(harness, 1, "INSERT INTO "+table+" (v) VALUES ('node1')") + c := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + c.createTable(1, "marmot", source, "CREATE TABLE "+source+" (id BIGINT PRIMARY KEY, v TEXT)") + first, err := insertNarrow(c, 1, "marmot", "INSERT INTO "+table+" (v) VALUES ('node1')") if err != nil { t.Fatalf("node 1 insert: %v", err) } - if _, err := harness.ExecNode(2, "INSERT INTO "+source+" (id, v) VALUES (?, 'copied')", first+3); err != nil { - t.Fatalf("fill %s: %v", source, err) - } - if err := harness.WaitForRowCount(source, []int{2}, 1, 15*time.Second); err != nil { - t.Fatalf("%s row did not reach node 2: %v", source, err) - } - - for name, write := range map[string]func() error{ - "literal": func() error { - _, err := harness.ExecNode(2, fmt.Sprintf("INSERT INTO %s (id, v) VALUES (%d, 'client')", table, first+1)) - return err - }, - "bound parameter": func() error { - _, err := harness.ExecNode(2, "INSERT INTO "+table+" (id, v) VALUES (?, 'client')", first+2) - return err - }, - "insert select": func() error { - _, err := harness.ExecNode(2, "INSERT INTO "+table+" (id, v) SELECT id, v FROM "+source) - return err - }, + c.mustExec(2, "marmot", "INSERT INTO "+source+" (id, v) VALUES (?, 'copied')", first+3) + c.waitRows("marmot", "SELECT id FROM "+source, []string{fmt.Sprint(first + 3)}, 2) + for name, w := range map[string]struct { + q string + args []any + }{ + "literal": {q: fmt.Sprintf("INSERT INTO %s (id, v) VALUES (%d, 'client')", table, first+1)}, + "bound parameter": {q: "INSERT INTO " + table + " (id, v) VALUES (?, 'client')", args: []any{first + 2}}, + "insert select": {q: "INSERT INTO " + table + " (id, v) SELECT id, v FROM " + source}, } { - err := write() - var mysqlErr *mysql.MySQLError - if !errors.As(err, &mysqlErr) || mysqlErr.Number != mysqlcode.ErrCodeNotSupportedYet { + if _, err := c.exec(2, "marmot", w.q, w.args...); mysqlCode(err) != mysqlcode.ErrCodeNotSupportedYet { t.Fatalf("%s: an id inside node 1's unissued range was admitted through node 2: err = %v", name, err) } } - for i := 0; i < 5; i++ { - if _, err := insertNarrow(harness, 1, "INSERT INTO "+table+" (v) VALUES ('node1')"); err != nil { + if _, err := insertNarrow(c, 1, "marmot", "INSERT INTO "+table+" (v) VALUES ('node1')"); err != nil { t.Fatalf("node 1 insert %d: %v", i, err) } } - if err := harness.WaitForRowCount(table, []int{1, 2, 3}, 6, 15*time.Second); err != nil { - t.Fatalf("rows did not replicate: %v", err) - } - for nodeID := 1; nodeID <= 3; nodeID++ { - rows, err := harness.QueryNode(nodeID, "SELECT COUNT(*) FROM "+table+" WHERE v <> 'node1'") - if err != nil { - t.Fatalf("node %d: %v", nodeID, err) - } - var foreign int - rows.Next() - _ = rows.Scan(&foreign) - rows.Close() - if foreign != 0 { - t.Fatalf("node %d holds %d rows written through node 2 in node 1's range", nodeID, foreign) - } - } + c.waitRows("marmot", "SELECT COUNT(*), COUNT(DISTINCT id), SUM(v = 'node1') FROM "+table, []string{"6|6|6"}, 1, 2, 3) } -// fillExplicitIDs inserts rows 1..n into table through node 1. -func fillExplicitIDs(t *testing.T, harness *ClusterHarness, table string, n int) { - t.Helper() - for start := 1; start <= n; start += 100 { - var values []string - for id := start; id < start+100 && id <= n; id++ { - values = append(values, fmt.Sprintf("(%d, 'row_%d')", id, id)) - } - if _, err := harness.ExecNode(1, "INSERT INTO "+table+" (id, v) VALUES "+strings.Join(values, ", ")); err != nil { - t.Fatalf("fill %s: %v", table, err) - } - } -} - -// assertFirstGeneratedAbove inserts one generated row through every node and -// requires each id to lie above highest. -func assertFirstGeneratedAbove(t *testing.T, harness *ClusterHarness, table string, highest int64) { - t.Helper() - for nodeID := 1; nodeID <= 3; nodeID++ { - id, err := insertNarrow(harness, nodeID, "INSERT INTO "+table+" (v) VALUES ('generated')") - if err != nil { - harness.dumpNodeLogs("narrow_generated_above") - t.Fatalf("node %d: first generated insert: %v", nodeID, err) - } - if id <= highest { - t.Fatalf("node %d: generated id %d landed at or below the copied MAX %d", nodeID, id, highest) - } - } +// idStats is "COUNT(*)|COUNT(DISTINCT col)|MAX(col)" of table. +func idStats(table, col string) string { + return fmt.Sprintf("SELECT COUNT(*), COUNT(DISTINCT %s), MAX(%s) FROM %s", col, col, table) } diff --git a/test/cluster_integration_test.go b/test/cluster_integration_test.go index 03e3af33..82642429 100644 --- a/test/cluster_integration_test.go +++ b/test/cluster_integration_test.go @@ -1,619 +1,104 @@ package test import ( - "context" "database/sql" "fmt" "os" "path/filepath" - "strings" "testing" - "time" - - _ "github.com/go-sql-driver/mysql" - "github.com/maxpert/marmot/coordinator" - "github.com/maxpert/marmot/db" - marmotgrpc "github.com/maxpert/marmot/grpc" - "github.com/maxpert/marmot/hlc" - "github.com/maxpert/marmot/protocol" - "github.com/stretchr/testify/require" -) - -const ( - propagationTimeout = 10 * time.Second - startupDelay = 500 * time.Millisecond ) -type testNode struct { - nodeID uint64 - dataDir string - grpcPort int - mysqlPort int - dbMgr *db.DatabaseManager - grpcServer *marmotgrpc.Server - mysqlServer *protocol.MySQLServer - client *marmotgrpc.Client - gossip *marmotgrpc.GossipProtocol - clock *hlc.Clock - mysqlConn *sql.DB - antiEntropy *marmotgrpc.AntiEntropyService - writeCoord *coordinator.WriteCoordinator - readCoord *coordinator.ReadCoordinator - catchUpClient *marmotgrpc.CatchUpClient -} - -func (n *testNode) cleanup() { - if n.mysqlConn != nil { - n.mysqlConn.Close() - } - if n.antiEntropy != nil { - n.antiEntropy.Stop() - } - if n.gossip != nil { - n.gossip.Stop() - } - if n.client != nil { - _ = n.client.Close() - } - if n.mysqlServer != nil { - n.mysqlServer.Stop() - } - if n.grpcServer != nil { - n.grpcServer.Stop() - } - if n.dbMgr != nil { - n.dbMgr.Close() - } - if n.dataDir != "" { - os.RemoveAll(n.dataDir) - } -} - +// TestClusterReplication: CREATE TABLE, INSERTs, UPDATEs and DELETEs, each +// through a different node, leave exactly the same rows on every node. func TestClusterReplication(t *testing.T) { - // Skip if short mode (this is an integration test) - if testing.Short() { - t.Skip("Skipping integration test in short mode") - } + c := newCluster(t) + c.start() + c.createTable(1, "marmot", "users", "CREATE TABLE users (id INT PRIMARY KEY, name TEXT, balance INT)") + const q = "SELECT id, name, balance FROM users ORDER BY id" - // Create 3 nodes - nodes := make([]*testNode, 3) - - // Setup and start nodes - for i := 0; i < 3; i++ { - nodeID := uint64(i + 1) - node := &testNode{ - nodeID: nodeID, - dataDir: filepath.Join(testDataRoot, fmt.Sprintf("marmot-test-node-%d-%d", nodeID, time.Now().UnixNano())), - grpcPort: 18081 + i, - mysqlPort: 13307 + i, - } - nodes[i] = node - - // Ensure cleanup - t.Cleanup(node.cleanup) - - // Create data directory - require.NoError(t, os.MkdirAll(node.dataDir, 0755)) + for i := 1; i <= 10; i++ { + c.mustExec(1, "marmot", "INSERT INTO users (id, name, balance) VALUES (?, ?, ?)", i, fmt.Sprintf("user%d", i), i*100) } - - // Start nodes sequentially (node 1 first as seed) - for i, node := range nodes { - var seedNodes []string - if i > 0 { - // Non-seed nodes point to node 1 - seedNodes = []string{fmt.Sprintf("localhost:%d", nodes[0].grpcPort)} + rows := func(balances map[int]int, ids ...int) []string { + var want []string + for _, id := range ids { + want = append(want, fmt.Sprintf("%d|user%d|%d", id, id, balances[id])) } - - startNode(t, node, seedNodes) - - // Give node time to start up - time.Sleep(startupDelay) - } - - // Wait for cluster to stabilize - t.Log("Waiting for cluster to stabilize...") - time.Sleep(2 * time.Second) - - // Verify all nodes see each other - for _, node := range nodes { - aliveNodes := node.gossip.GetNodeRegistry().GetAlive() - require.GreaterOrEqual(t, len(aliveNodes), 3, "Node %d should see at least 3 nodes", node.nodeID) - } - - // Test 1: CREATE TABLE on node 1, verify on all nodes - t.Log("Test 1: CREATE TABLE replication") - _, err := nodes[0].mysqlConn.Exec("CREATE TABLE users (id INT PRIMARY KEY, name TEXT, balance INT)") - require.NoError(t, err, "Failed to create table on node 1") - - // Wait for replication - waitForReplication(t, "Table creation") - - // Verify table exists on all nodes - for _, node := range nodes { - verifyTableExists(t, node, "users") + return want } - - // Test 2: INSERT 10 rows on node 1, verify on all nodes - t.Log("Test 2: INSERT replication (10 rows)") + balances := map[int]int{} for i := 1; i <= 10; i++ { - _, err := nodes[0].mysqlConn.Exec( - "INSERT INTO users (id, name, balance) VALUES (?, ?, ?)", - i, fmt.Sprintf("user%d", i), i*100, - ) - require.NoError(t, err, "Failed to insert row %d on node 1", i) - } - - // Wait for replication - waitForReplication(t, "10 INSERTs") - - // Verify all 10 rows on all nodes - for _, node := range nodes { - verifyRowCount(t, node, "users", 10) - verifyRowData(t, node, "users", 1, "user1", 100) - verifyRowData(t, node, "users", 5, "user5", 500) - verifyRowData(t, node, "users", 10, "user10", 1000) + balances[i] = i * 100 } + c.waitRows("marmot", q, rows(balances, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), 1, 2, 3) - // Test 3: UPDATE 5 rows on node 2, verify on all nodes - t.Log("Test 3: UPDATE replication (5 rows)") for i := 1; i <= 5; i++ { - _, err := nodes[1].mysqlConn.Exec( - "UPDATE users SET balance = ? WHERE id = ?", - i*200, i, - ) - require.NoError(t, err, "Failed to update row %d on node 2", i) + c.mustExec(2, "marmot", "UPDATE users SET balance = ? WHERE id = ?", i*200, i) + balances[i] = i * 200 } + c.waitRows("marmot", q, rows(balances, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), 1, 2, 3) - // Wait for replication - waitForReplication(t, "5 UPDATEs") - - // Verify updates on all nodes - for _, node := range nodes { - verifyRowCount(t, node, "users", 10) - verifyRowData(t, node, "users", 1, "user1", 200) - verifyRowData(t, node, "users", 3, "user3", 600) - verifyRowData(t, node, "users", 5, "user5", 1000) - // Unchanged rows - verifyRowData(t, node, "users", 6, "user6", 600) - verifyRowData(t, node, "users", 10, "user10", 1000) - } - - // Test 4: DELETE 3 rows on node 3, verify on all nodes - t.Log("Test 4: DELETE replication (3 rows)") for i := 8; i <= 10; i++ { - _, err := nodes[2].mysqlConn.Exec("DELETE FROM users WHERE id = ?", i) - require.NoError(t, err, "Failed to delete row %d on node 3", i) - } - - // Wait for replication - waitForReplication(t, "3 DELETEs") - - // Verify deletes on all nodes - for _, node := range nodes { - verifyRowCount(t, node, "users", 7) - // Verify deleted rows are gone - verifyRowNotExists(t, node, "users", 8) - verifyRowNotExists(t, node, "users", 9) - verifyRowNotExists(t, node, "users", 10) - // Verify remaining rows - verifyRowData(t, node, "users", 1, "user1", 200) - verifyRowData(t, node, "users", 7, "user7", 700) + c.mustExec(3, "marmot", "DELETE FROM users WHERE id = ?", i) } - - // Test 5: Final verification - exact data match across all nodes - t.Log("Test 5: Final verification - data consistency") - for _, node := range nodes { - rows, err := node.mysqlConn.Query("SELECT id, name, balance FROM users ORDER BY id") - require.NoError(t, err, "Failed to query users on node %d", node.nodeID) - - var results []struct { - id int - name string - balance int - } - for rows.Next() { - var id, balance int - var name string - require.NoError(t, rows.Scan(&id, &name, &balance)) - results = append(results, struct { - id int - name string - balance int - }{id, name, balance}) - } - rows.Close() - - require.Equal(t, 7, len(results), "Node %d should have exactly 7 rows", node.nodeID) - - // Verify exact data - expected := []struct { - id int - name string - balance int - }{ - {1, "user1", 200}, - {2, "user2", 400}, - {3, "user3", 600}, - {4, "user4", 800}, - {5, "user5", 1000}, - {6, "user6", 600}, - {7, "user7", 700}, - } - - for i, exp := range expected { - require.Equal(t, exp.id, results[i].id, "Node %d row %d: ID mismatch", node.nodeID, i) - require.Equal(t, exp.name, results[i].name, "Node %d row %d: Name mismatch", node.nodeID, i) - require.Equal(t, exp.balance, results[i].balance, "Node %d row %d: Balance mismatch", node.nodeID, i) - } - - t.Logf("Node %d: All data verified ✓", node.nodeID) - } - - t.Log("All cluster replication tests passed ✓") + c.waitRows("marmot", q, rows(balances, 1, 2, 3, 4, 5, 6, 7), 1, 2, 3) } +// TestClusterLoadDataLocalReplication: rows a LOAD DATA LOCAL INFILE loads +// through one node reach every node exactly. func TestClusterLoadDataLocalReplication(t *testing.T) { - // Skip if short mode (this is an integration test) - if testing.Short() { - t.Skip("Skipping integration test in short mode") - } + c := newCluster(t) + c.start() + c.createTable(1, "marmot", "bulk_users", "CREATE TABLE bulk_users (id INT PRIMARY KEY, name TEXT, score INT)") - // Create 3 nodes with dedicated ports to avoid collisions with other integration tests - nodes := make([]*testNode, 3) - for i := 0; i < 3; i++ { - nodeID := uint64(i + 1) - node := &testNode{ - nodeID: nodeID, - dataDir: filepath.Join(testDataRoot, fmt.Sprintf("marmot-load-test-node-%d-%d", nodeID, time.Now().UnixNano())), - grpcPort: 18181 + i, - mysqlPort: 13407 + i, - } - nodes[i] = node - t.Cleanup(node.cleanup) - require.NoError(t, os.MkdirAll(node.dataDir, 0755)) + csvPath := filepath.Join(testDir(t), "bulk_users.csv") + csv := "id,name,score\n1,alice,100\n2,bob,200\n3,charlie,300\n4,dana,400\n5,eve,500\n" + if err := os.WriteFile(csvPath, []byte(csv), 0o644); err != nil { + t.Fatal(err) } - - // Start nodes sequentially (node 1 as seed) - for i, node := range nodes { - var seedNodes []string - if i > 0 { - seedNodes = []string{fmt.Sprintf("localhost:%d", nodes[0].grpcPort)} - } - startNode(t, node, seedNodes) - time.Sleep(startupDelay) - } - - // Wait for membership convergence - time.Sleep(2 * time.Second) - for _, node := range nodes { - aliveNodes := node.gossip.GetNodeRegistry().GetAlive() - require.GreaterOrEqual(t, len(aliveNodes), 3, "Node %d should see at least 3 nodes", node.nodeID) + loadConn, err := sql.Open("mysql", c.dsn(1, "marmot")+"&allowAllFiles=true") + if err != nil { + t.Fatal(err) } - - _, err := nodes[0].mysqlConn.Exec("CREATE TABLE bulk_users (id INT PRIMARY KEY, name TEXT, score INT)") - require.NoError(t, err, "Failed to create bulk_users table") - waitForReplication(t, "bulk_users table creation") - for _, node := range nodes { - verifyTableExists(t, node, "bulk_users") - } - - // Create LOCAL INFILE payload on disk (header + rows) - csvPath := filepath.Join(t.TempDir(), "bulk_users.csv") - csvBody := strings.Join([]string{ - "id,name,score", - "1,alice,100", - "2,bob,200", - "3,charlie,300", - "4,dana,400", - "5,eve,500", - "", - }, "\n") - require.NoError(t, os.WriteFile(csvPath, []byte(csvBody), 0644)) - - // Use a dedicated connection that allows LOCAL INFILE file reads. - loadDSN := fmt.Sprintf("root:@tcp(127.0.0.1:%d)/marmot?allowAllFiles=true", nodes[0].mysqlPort) - loadConn, err := sql.Open("mysql", loadDSN) - require.NoError(t, err) defer loadConn.Close() - require.NoError(t, loadConn.Ping()) - - loadSQL := fmt.Sprintf( - "LOAD DATA LOCAL INFILE '%s' INTO TABLE bulk_users FIELDS TERMINATED BY ',' LINES TERMINATED BY '\\n' IGNORE 1 LINES (id,name,score)", - csvPath, - ) - res, err := loadConn.Exec(loadSQL) - require.NoError(t, err, "LOAD DATA LOCAL INFILE should succeed") - - rowsAffected, err := res.RowsAffected() - require.NoError(t, err) - require.Greater(t, rowsAffected, int64(0), "LOAD DATA should report some inserted rows") - - waitForReplication(t, "LOAD DATA LOCAL replication") - - // Verify convergence using existing replication metadata (txn_id + seq_num). - // This gives a fast cluster-level barrier before row-by-row assertions. - targetTxnID, err := nodes[0].dbMgr.GetMaxTxnID("marmot") - require.NoError(t, err, "Failed to get source node max txn_id") - targetSeqNum, err := nodes[0].dbMgr.GetMaxSeqNum("marmot") - require.NoError(t, err, "Failed to get source node max seq_num") - waitForClusterWatermark(t, nodes, "marmot", targetTxnID, targetSeqNum, propagationTimeout) - - for _, node := range nodes { - verifyRowCount(t, node, "bulk_users", 5) + res, err := loadConn.Exec(fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE bulk_users FIELDS TERMINATED BY ',' "+ + "LINES TERMINATED BY '\\n' IGNORE 1 LINES (id,name,score)", csvPath)) + if err != nil { + t.Fatalf("LOAD DATA LOCAL INFILE: %v", err) } - - // Final strict equality check: all nodes must have identical ordered rows. - baseline := getOrderedRows(t, nodes[0], "SELECT id, name, score FROM bulk_users ORDER BY id") - require.Len(t, baseline, 5) - require.Equal(t, []string{ - "1|alice|100", - "2|bob|200", - "3|charlie|300", - "4|dana|400", - "5|eve|500", - }, baseline, "Source node should contain expected LOAD DATA rows") - for _, node := range nodes[1:] { - rows := getOrderedRows(t, node, "SELECT id, name, score FROM bulk_users ORDER BY id") - require.Equal(t, baseline, rows, "Node %d should exactly match node %d", node.nodeID, nodes[0].nodeID) + if n, err := res.RowsAffected(); err != nil || n <= 0 { + t.Fatalf("LOAD DATA reported %d rows affected (err=%v), want some", n, err) } + want := []string{"1|alice|100", "2|bob|200", "3|charlie|300", "4|dana|400", "5|eve|500"} + c.waitRows("marmot", "SELECT id, name, score FROM bulk_users ORDER BY id", want, 1, 2, 3) } -func startNode(t *testing.T, node *testNode, seedNodes []string) { - // Initialize HLC clock - node.clock = hlc.NewClock(node.nodeID) - - // Initialize gRPC server - grpcConfig := marmotgrpc.ServerConfig{ - NodeID: node.nodeID, - Address: "0.0.0.0", - Port: node.grpcPort, - AdvertiseAddress: fmt.Sprintf("localhost:%d", node.grpcPort), - } - - grpcServer, err := marmotgrpc.NewServer(grpcConfig) - require.NoError(t, err, "Failed to create gRPC server for node %d", node.nodeID) - node.grpcServer = grpcServer - - require.NoError(t, grpcServer.Start(), "Failed to start gRPC server for node %d", node.nodeID) - - // Initialize gossip - gossipConfig := marmotgrpc.DefaultGossipConfig() - gossip := grpcServer.GetGossipProtocol() - node.gossip = gossip - - client := marmotgrpc.NewClient(node.nodeID) - node.client = client - gossip.SetClient(client) - - // Join cluster if seed nodes provided - if len(seedNodes) > 0 { - err := gossip.JoinCluster(seedNodes, grpcConfig.AdvertiseAddress) - require.NoError(t, err, "Failed to join cluster for node %d", node.nodeID) - } - - gossip.Start(gossipConfig) - - // Initialize database manager - dbMgr, err := db.NewDatabaseManager(node.dataDir, node.nodeID, node.clock) - require.NoError(t, err, "Failed to create database manager for node %d", node.nodeID) - node.dbMgr = dbMgr - - // Get system database for schema versioning - _, err = dbMgr.GetDatabase(db.SystemDatabaseName) - require.NoError(t, err, "Failed to get system database for node %d", node.nodeID) - schemaVersionMgr := db.NewSchemaVersionManager(dbMgr) - - // Wire up replication handlers - replicationHandler := marmotgrpc.NewReplicationHandler( - node.nodeID, - dbMgr, - node.clock, - schemaVersionMgr, - ) - replicationHandler.SetClient(client) - grpcServer.SetReplicationHandler(replicationHandler) - grpcServer.SetDatabaseManager(dbMgr) - - // Setup anti-entropy - node.catchUpClient = marmotgrpc.NewCatchUpClient( - node.nodeID, - node.dataDir, - grpcServer.GetNodeRegistry(), - seedNodes, - ) - node.catchUpClient.SetDatabaseManager(dbMgr) - - logPuller := marmotgrpc.NewLogPuller(marmotgrpc.LogPullerConfig{ - NodeID: node.nodeID, - Client: client, - DBManager: dbMgr, - }) - - snapshotFunc := func(ctx context.Context, peerNodeID uint64, peerAddr string, database string) error { - return node.catchUpClient.CatchUpFromPeer(ctx, peerNodeID, peerAddr, database) - } - - node.antiEntropy = marmotgrpc.NewAntiEntropyServiceFromConfig( - node.nodeID, - grpcServer.GetNodeRegistry(), - client, - dbMgr, - logPuller, - snapshotFunc, - ) - node.antiEntropy.Start() - - // Setup coordinators - nodeProvider := marmotgrpc.NewGossipNodeProvider(gossip.GetNodeRegistry(), false) - // As marmot.go does: a claim participant counts the cluster exactly as a - // claimant's quorum does. - dbMgr.SetClusterMembership(nodeProvider.GetTotalMembershipSize) - replicator := marmotgrpc.NewGRPCReplicator(client) - - localReplicator := db.NewLocalReplicator(node.nodeID, dbMgr, node.clock) - node.writeCoord = coordinator.NewWriteCoordinator( - node.nodeID, - nodeProvider, - replicator, - localReplicator, - 5*time.Second, - node.clock, - ) - - localReader := db.NewLocalReader(dbMgr) - node.readCoord = coordinator.NewReadCoordinator( - node.nodeID, - nodeProvider, - localReader, - 2*time.Second, - ) - - // Setup MySQL server - ddlLockMgr := coordinator.NewDDLLockManager(30 * time.Second) - registryAdapter := marmotgrpc.NewNodeRegistryAdapter(gossip.GetNodeRegistry()) - - handler := coordinator.NewCoordinatorHandler( - node.nodeID, - node.writeCoord, - node.readCoord, - node.clock, - dbMgr, - ddlLockMgr, - schemaVersionMgr, - registryAdapter, - ) - - mysqlServer := protocol.NewMySQLServer( - fmt.Sprintf("127.0.0.1:%d", node.mysqlPort), - "", - 0, - handler, - ) - require.NoError(t, mysqlServer.Start(), "Failed to start MySQL server for node %d", node.nodeID) - node.mysqlServer = mysqlServer - - // Mark node as ALIVE (seed node behavior) - if len(seedNodes) == 0 { - grpcServer.GetNodeRegistry().MarkAlive(node.nodeID) +// TestClusterLoadDataLocalNarrowReportsEveryRow: a LOAD DATA LOCAL INFILE into +// a narrow AUTO_INCREMENT table runs as INSERTs through the coordinator +// (7a66596), which reports exactly the rows loaded, and every node ends with +// them under distinct generated ids. +func TestClusterLoadDataLocalNarrowReportsEveryRow(t *testing.T) { + c := newCluster(t) + c.start() + c.createTable(1, "marmot", "bulk_narrow", "CREATE TABLE bulk_narrow (id INT AUTO_INCREMENT PRIMARY KEY, name TEXT, score INT)") + csvPath := filepath.Join(testDir(t), "bulk_narrow.csv") + if err := os.WriteFile(csvPath, []byte("name,score\nalice,100\nbob,200\ncharlie,300\ndana,400\neve,500\n"), 0o644); err != nil { + t.Fatal(err) + } + loadConn, err := sql.Open("mysql", c.dsn(1, "marmot")+"&allowAllFiles=true") + if err != nil { + t.Fatal(err) } - - // Connect MySQL client - time.Sleep(100 * time.Millisecond) // Let server start - - dsn := fmt.Sprintf("root:@tcp(127.0.0.1:%d)/marmot", node.mysqlPort) - mysqlConn, err := sql.Open("mysql", dsn) - require.NoError(t, err, "Failed to open MySQL connection for node %d", node.nodeID) - require.NoError(t, mysqlConn.Ping(), "Failed to ping MySQL server for node %d", node.nodeID) - node.mysqlConn = mysqlConn - - t.Logf("Node %d started (gRPC: %d, MySQL: %d)", node.nodeID, node.grpcPort, node.mysqlPort) -} - -func waitForReplication(t *testing.T, description string) { - t.Logf("Waiting for replication: %s", description) - time.Sleep(2 * time.Second) // Give time for replication + anti-entropy -} - -func verifyTableExists(t *testing.T, node *testNode, tableName string) { - rows, err := node.mysqlConn.Query("SHOW TABLES") - require.NoError(t, err, "Failed to list tables on node %d", node.nodeID) - defer rows.Close() - - found := false - for rows.Next() { - var name string - require.NoError(t, rows.Scan(&name), "Failed scanning table name on node %d", node.nodeID) - if name == tableName { - found = true - break - } - } - require.NoError(t, rows.Err(), "Error iterating tables on node %d", node.nodeID) - require.True(t, found, "Table %s should exist on node %d", tableName, node.nodeID) -} - -func verifyRowCount(t *testing.T, node *testNode, tableName string, expectedCount int) { - var count int - err := node.mysqlConn.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName)).Scan(&count) - require.NoError(t, err, "Failed to count rows in %s on node %d", tableName, node.nodeID) - require.Equal(t, expectedCount, count, "Node %d should have %d rows in %s", node.nodeID, expectedCount, tableName) -} - -func verifyRowData(t *testing.T, node *testNode, tableName string, id int, expectedName string, expectedBalance int) { - var name string - var balance int - err := node.mysqlConn.QueryRow( - fmt.Sprintf("SELECT name, balance FROM %s WHERE id = ?", tableName), - id, - ).Scan(&name, &balance) - require.NoError(t, err, "Failed to query row %d from %s on node %d", id, tableName, node.nodeID) - require.Equal(t, expectedName, name, "Node %d row %d: name mismatch", node.nodeID, id) - require.Equal(t, expectedBalance, balance, "Node %d row %d: balance mismatch", node.nodeID, id) -} - -func verifyRowNotExists(t *testing.T, node *testNode, tableName string, id int) { - var count int - err := node.mysqlConn.QueryRow( - fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE id = ?", tableName), - id, - ).Scan(&count) - require.NoError(t, err, "Failed to check row %d in %s on node %d", id, tableName, node.nodeID) - require.Equal(t, 0, count, "Node %d should not have row %d in %s", node.nodeID, id, tableName) -} - -func getOrderedRows(t *testing.T, node *testNode, query string) []string { - rows, err := node.mysqlConn.Query(query) - require.NoError(t, err, "Failed ordered query on node %d", node.nodeID) - defer rows.Close() - - result := make([]string, 0) - for rows.Next() { - var id int - var name string - var score int - require.NoError(t, rows.Scan(&id, &name, &score)) - result = append(result, fmt.Sprintf("%d|%s|%d", id, name, score)) + defer loadConn.Close() + res, err := loadConn.Exec(fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE bulk_narrow FIELDS TERMINATED BY ',' "+ + "LINES TERMINATED BY '\\n' IGNORE 1 LINES (name,score)", csvPath)) + if err != nil { + t.Fatalf("LOAD DATA LOCAL INFILE: %v", err) } - require.NoError(t, rows.Err()) - return result -} - -func waitForClusterWatermark(t *testing.T, nodes []*testNode, database string, targetTxnID, targetSeqNum uint64, timeout time.Duration) { - t.Helper() - deadline := time.Now().Add(timeout) - - var lastStatus []string - for time.Now().Before(deadline) { - allReached := true - lastStatus = lastStatus[:0] - - for _, node := range nodes { - maxTxnID, txnErr := node.dbMgr.GetMaxTxnID(database) - require.NoError(t, txnErr, "Failed to get max txn_id on node %d", node.nodeID) - maxSeqNum, seqErr := node.dbMgr.GetMaxSeqNum(database) - require.NoError(t, seqErr, "Failed to get max seq_num on node %d", node.nodeID) - - lastStatus = append(lastStatus, fmt.Sprintf("node=%d txn=%d seq=%d", node.nodeID, maxTxnID, maxSeqNum)) - if maxTxnID < targetTxnID || maxSeqNum < targetSeqNum { - allReached = false - } - } - - if allReached { - t.Logf("Cluster watermark reached: target_txn=%d target_seq=%d", targetTxnID, targetSeqNum) - return - } - - time.Sleep(100 * time.Millisecond) + if n, err := res.RowsAffected(); err != nil || n != 5 { + t.Fatalf("LOAD DATA reported %d rows affected (err=%v), want 5", n, err) } - - require.FailNowf( - t, - "cluster watermark timeout", - "database=%s target_txn=%d target_seq=%d status=%s", - database, - targetTxnID, - targetSeqNum, - strings.Join(lastStatus, "; "), - ) + want := []string{"alice|100", "bob|200", "charlie|300", "dana|400", "eve|500"} + c.waitRows("marmot", "SELECT name, score FROM bulk_narrow ORDER BY name", want, 1, 2, 3) + c.waitRows("marmot", "SELECT COUNT(DISTINCT id) FROM bulk_narrow", []string{"5"}, 1, 2, 3) } diff --git a/test/crash_recovery_test.go b/test/crash_recovery_test.go index d5edfcd9..107f2e80 100644 --- a/test/crash_recovery_test.go +++ b/test/crash_recovery_test.go @@ -1,1535 +1,136 @@ package test import ( - "context" - "database/sql" - "encoding/json" "fmt" - "io" - "net/http" - "os" - "os/exec" - "path/filepath" - "runtime" - "strconv" - "strings" - "sync" - "sync/atomic" - "syscall" + "slices" "testing" - "time" - - _ "github.com/go-sql-driver/mysql" - - "github.com/maxpert/marmot/test/stallwatch" -) - -const ( - baseGRPCPort = 8080 - baseMySQLPort = 3306 - numNodes = 3 - - // clusterStallTimeout bounds how long a cluster may go with no progress: - // no file written under any node's directory (logs, SQLite, meta store), - // no client call returned (noteClientCall) and no progress reported by - // the test (ClusterHarness.Progress). - clusterStallTimeout = 20 * time.Second - // clusterStallPoll is how often the stall watchdog looks. - clusterStallPoll = 250 * time.Millisecond - // clusterReadyDeadline bounds StartCluster's wait for each node to be - // ready for writes (WaitForReady). - clusterReadyDeadline = 20 * time.Second - // clusterConvergencePoll is how often WaitForClusterConvergence asks a - // node for its membership. - clusterConvergencePoll = 250 * time.Millisecond - // stallLogTailLines is how much of each node's log a stall report shows. - stallLogTailLines = 30 - // stallStackWait bounds the wait for a node to write its goroutine dump. - stallStackWait = 3 * time.Second - - // clusterTestBudget is the hard wall-clock budget for any test built on - // ClusterHarness (user rule: every cluster test finishes in <60s). It is - // enforced independently of the stall watchdog: a test can keep making - // progress (writes, restarts) and still blow well past a sane budget. - clusterTestBudget = 60 * time.Second - - // defaultGCIntervalSeconds is the harness's [replication] gc_interval_seconds. - defaultGCIntervalSeconds = 30 - // defaultHeartbeatTimeoutSeconds is the harness's [transaction] - // heartbeat_timeout_seconds. - defaultHeartbeatTimeoutSeconds = 10 - // unsetOverride marks a ClusterHarness int override field as "use the - // harness default" (0 is a real value for some of these fields, so it - // cannot double as the sentinel). - unsetOverride = -1 ) -var ( - crashHarnessBuildOnce sync.Once - crashHarnessBinPath string - crashHarnessBuildErr error -) - -// cleanupStalePorts kills any processes using ports that will be used by the test cluster -func cleanupStalePorts() { - selfPID := os.Getpid() - ports := []int{ - baseGRPCPort + 1, baseGRPCPort + 2, baseGRPCPort + 3, - baseMySQLPort + 1, baseMySQLPort + 2, baseMySQLPort + 3, - } - for _, port := range ports { - cmd := exec.Command("lsof", "-ti", fmt.Sprintf(":%d", port)) - output, err := cmd.Output() - if err == nil && len(output) > 0 { - pids := strings.Fields(strings.TrimSpace(string(output))) - for _, pid := range pids { - parsedPID, parseErr := strconv.Atoi(pid) - if parseErr != nil { - continue - } - if parsedPID == selfPID { - continue - } - if !isMarmotProcess(parsedPID) { - continue - } - killCmd := exec.Command("kill", "-9", pid) - killCmd.Run() - } - } - } - time.Sleep(100 * time.Millisecond) -} - -func isMarmotProcess(pid int) bool { - cmd := exec.Command("ps", "-p", strconv.Itoa(pid), "-o", "command=") - output, err := cmd.Output() - if err != nil { - return false - } - commandLine := strings.ToLower(strings.TrimSpace(string(output))) - if commandLine == "" { - return false - } - if strings.Contains(commandLine, "go test") || strings.Contains(commandLine, "___go_build") { - return false - } - return strings.Contains(commandLine, "marmot") -} - -// ClusterNode represents a running Marmot node -type ClusterNode struct { - NodeID int - GRPCPort int - MySQLPort int - DataDir string - ConfigPath string - PIDFile string - LogFile string - Cmd *exec.Cmd - DB *sql.DB - isRunning bool - mu sync.Mutex -} - -// ClusterHarness manages a test cluster -type ClusterHarness struct { - Nodes []*ClusterNode - BaseDir string - MarmotBin string - t *testing.T - cleanupOnce sync.Once - watchdog *stallwatch.Watchdog - - // NodeBin overrides MarmotBin for one node id, so a test can run a mix of - // binaries (e.g. a rolling-upgrade test). Set it before StartNode. - NodeBin map[int]string - - // GCIntervalSeconds overrides [replication] gc_interval_seconds on every - // node's config. unsetOverride (the default) keeps defaultGCIntervalSeconds. - GCIntervalSeconds int - // GCMinRetentionHours overrides [replication] gc_min_retention_hours. - // unsetOverride keeps the config's own default (2h). - GCMinRetentionHours int - // DeltaSyncThresholdSeconds overrides [replication] delta_sync_threshold_seconds. - // unsetOverride keeps the config's own default (3600s). A live-GC test - // needs this lowered together with GCMinRetentionHours: cfg.Validate - // requires gc_min_retention_hours >= delta_sync_threshold_seconds in hours. - DeltaSyncThresholdSeconds int - // HeartbeatTimeoutSeconds overrides [transaction] heartbeat_timeout_seconds, - // after which the stale-transaction GC aborts a PENDING transaction. - // unsetOverride keeps defaultHeartbeatTimeoutSeconds. - HeartbeatTimeoutSeconds int - - // budgetStart is when the harness (and so the test using it) began. - budgetStart time.Time - budgetDone chan struct{} - budgetOnce sync.Once - - phaseMu sync.Mutex - lastPhase time.Time - phases []phaseRecord -} - -// phaseRecord is one named phase of a cluster test and how long it took -// since the previous phase (or since the harness was created). -type phaseRecord struct { - name string - elapsed time.Duration -} - -// NewClusterHarness creates a new cluster harness for testing. Options run -// before any node config is written, so they can set GCIntervalSeconds and -// friends. -func NewClusterHarness(t *testing.T, opts ...func(*ClusterHarness)) *ClusterHarness { - cleanupStalePorts() - - baseDir := filepath.Join(testDataRoot, fmt.Sprintf("marmot_crash_test_%d", time.Now().UnixNano())) - marmotBin, err := getCrashHarnessBinary(t) - if err != nil { - t.Fatalf("Failed to build Marmot: %v", err) - } - - now := time.Now() - harness := &ClusterHarness{ - Nodes: make([]*ClusterNode, numNodes), - BaseDir: baseDir, - MarmotBin: marmotBin, - t: t, - GCIntervalSeconds: unsetOverride, - HeartbeatTimeoutSeconds: unsetOverride, - GCMinRetentionHours: unsetOverride, - DeltaSyncThresholdSeconds: unsetOverride, - budgetStart: now, - budgetDone: make(chan struct{}), - lastPhase: now, - } - for _, opt := range opts { - opt(harness) - } - - for i := 0; i < numNodes; i++ { - node := harness.createNode(i + 1) - harness.Nodes[i] = node - } - harness.watchdog = stallwatch.Start(clusterStallTimeout, clusterStallPoll, harness.activity, harness.reportStall) - go harness.watchBudget() - - return harness -} - -// Progress tells the stall watchdog the test saw the cluster make progress, -// such as an ACKed write. -func (h *ClusterHarness) Progress() { - h.watchdog.Progress() -} - -// Phase logs how long this phase of the test took (since the previous phase, -// or since the harness was created) and records it for the budget report. -// Call it at each phase of a cluster test: start, load, kill/restart, -// converge, verify, ... -func (h *ClusterHarness) Phase(name string) { - h.phaseMu.Lock() - now := time.Now() - elapsed := now.Sub(h.lastPhase) - h.lastPhase = now - h.phases = append(h.phases, phaseRecord{name: name, elapsed: elapsed}) - h.phaseMu.Unlock() - h.t.Logf("phase=%s elapsed=%s", name, elapsed.Round(time.Millisecond)) -} - -// watchBudget fails the test once the harness has lived past clusterTestBudget, -// independently of the stall watchdog: a test can keep making progress and -// still blow well past the budget the user rule sets. -func (h *ClusterHarness) watchBudget() { - timer := time.NewTimer(clusterTestBudget) - defer timer.Stop() - select { - case <-timer.C: - h.reportBudgetExceeded() - case <-h.budgetDone: - } -} - -// reportBudgetExceeded fails the test with every recorded phase's elapsed -// time and every node's log tail, then stops the nodes so the test's own -// calls fail at once. A goroutine cannot call t.Fatalf, so this mirrors -// reportStall's pattern (t.Errorf + StopCluster). -func (h *ClusterHarness) reportBudgetExceeded() { - h.t.Errorf("cluster test exceeded its %s budget", clusterTestBudget) - h.phaseMu.Lock() - for _, p := range h.phases { - h.t.Logf("phase=%s elapsed=%s", p.name, p.elapsed.Round(time.Millisecond)) - } - h.phaseMu.Unlock() - for i := range h.Nodes { - nodeID := i + 1 - h.t.Logf("=== node %d: last %d log lines ===\n%s", nodeID, stallLogTailLines, h.getNodeLogTail(nodeID, stallLogTailLines)) - } - h.StopCluster() -} - -// clientCalls counts client calls the tests' shared helpers completed -// (noteClientCall); the stall watchdog counts each as progress. -var clientCalls atomic.Int64 - -// noteClientCall records that one client call to a node returned, with or -// without an error: a cluster answering its clients is not stalled. -func noteClientCall() { - clientCalls.Add(1) -} - -// activity digests everything the stall watchdog counts as progress: every -// file write under the harness's directory (size and modification time, so -// a write into a preallocated file counts too) and every returned client -// call. -func (h *ClusterHarness) activity() int64 { - total := clientCalls.Load() - _ = filepath.WalkDir(h.BaseDir, func(_ string, d os.DirEntry, err error) error { - if err != nil || d.IsDir() { - return nil - } - if info, err := d.Info(); err == nil { - total += info.Size() + info.ModTime().UnixNano() - } - return nil - }) - return total -} - -// reportStall fails the test for a cluster that made no progress for -// clusterStallTimeout: it shows each node's log tail, has every running node -// dump its goroutines (SIGQUIT, into its log) and shows them, and stops the -// nodes so the test's own calls fail at once. -func (h *ClusterHarness) reportStall(idle time.Duration) { - h.t.Errorf("cluster stalled: no node wrote anything, no client call returned and the test reported no progress for %s", idle.Round(time.Second)) - for i, node := range h.Nodes { - nodeID := i + 1 - h.t.Logf("=== node %d: last %d log lines ===\n%s", nodeID, stallLogTailLines, h.getNodeLogTail(nodeID, stallLogTailLines)) - node.mu.Lock() - running, cmd := node.isRunning, node.Cmd - node.mu.Unlock() - if !running || cmd == nil || cmd.Process == nil { - continue - } - before, err := os.Stat(node.LogFile) - if err != nil { - continue - } - if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil { - h.t.Logf("node %d: SIGQUIT: %v", nodeID, err) - continue - } - deadline := time.Now().Add(stallStackWait) - for time.Now().Before(deadline) { - if after, err := os.Stat(node.LogFile); err == nil && after.Size() > before.Size() { - break - } - time.Sleep(100 * time.Millisecond) - } - time.Sleep(100 * time.Millisecond) - data, err := os.ReadFile(node.LogFile) - if err != nil || int64(len(data)) <= before.Size() { - h.t.Logf("node %d wrote no goroutine dump", nodeID) - continue - } - stackPath := filepath.Join(node.DataDir, "stall-stacks.txt") - if err := os.WriteFile(stackPath, data[before.Size():], 0o644); err == nil { - h.t.Logf("node %d goroutine dump: %s", nodeID, stackPath) - } - } - h.StopCluster() -} - -// testDataRoot holds the integration tests' node data and binaries: Marmot's -// runtime files live under /tmp/marmot. -const testDataRoot = "/tmp/marmot/test" - -// testDataDir creates a fresh directory under testDataRoot. -func testDataDir(pattern string) (string, error) { - if err := os.MkdirAll(testDataRoot, 0o755); err != nil { - return "", err - } - return os.MkdirTemp(testDataRoot, pattern) -} - -// repoRoot is the module root: the directory above this test package, taken -// from this file's own path so the harness builds the tree it is compiled from. -func repoRoot() string { - _, file, _, _ := runtime.Caller(0) - return filepath.Dir(filepath.Dir(file)) -} - -func getCrashHarnessBinary(t *testing.T) (string, error) { - t.Helper() - - crashHarnessBuildOnce.Do(func() { - t.Logf("Building shared Marmot binary for crash-recovery tests...") - - buildDir, err := testDataDir("marmot_crash_bin_") - if err != nil { - crashHarnessBuildErr = fmt.Errorf("failed to create build dir: %w", err) - return - } - - binPath := filepath.Join(buildDir, "marmot") - cmd := exec.Command("go", "build", "-tags", "sqlite_preupdate_hook sqlite_fts5 sqlite_json sqlite_math_functions sqlite_foreign_keys sqlite_stat4 sqlite_vacuum_incr", "-o", binPath, ".") - cmd.Dir = repoRoot() - output, err := cmd.CombinedOutput() - if err != nil { - crashHarnessBuildErr = fmt.Errorf("build failed: %v\n%s", err, output) - return - } - - crashHarnessBinPath = binPath - t.Logf("Shared Marmot binary built at %s", crashHarnessBinPath) - }) - - return crashHarnessBinPath, crashHarnessBuildErr -} - -func (h *ClusterHarness) createNode(nodeID int) *ClusterNode { - dataDir := filepath.Join(h.BaseDir, fmt.Sprintf("node%d", nodeID)) - os.MkdirAll(dataDir, 0755) - - node := &ClusterNode{ - NodeID: nodeID, - GRPCPort: baseGRPCPort + nodeID, - MySQLPort: baseMySQLPort + nodeID, - DataDir: dataDir, - ConfigPath: filepath.Join(dataDir, "config.toml"), - PIDFile: filepath.Join(dataDir, "marmot.pid"), - LogFile: filepath.Join(dataDir, "marmot.log"), - isRunning: false, - } - - h.createNodeConfig(node) - return node -} - -func (h *ClusterHarness) createNodeConfig(node *ClusterNode) { - seedNodes := []string{} - if node.NodeID != 1 { - seedNodes = append(seedNodes, fmt.Sprintf("\"localhost:%d\"", baseGRPCPort+1)) - } - - gcInterval := defaultGCIntervalSeconds - if h.GCIntervalSeconds != unsetOverride { - gcInterval = h.GCIntervalSeconds - } - heartbeatTimeout := defaultHeartbeatTimeoutSeconds - if h.HeartbeatTimeoutSeconds != unsetOverride { - heartbeatTimeout = h.HeartbeatTimeoutSeconds - } - gcMinRetentionLine, deltaSyncThresholdLine := "", "" - if h.GCMinRetentionHours != unsetOverride { - gcMinRetentionLine = fmt.Sprintf("gc_min_retention_hours = %d\n", h.GCMinRetentionHours) - } - if h.DeltaSyncThresholdSeconds != unsetOverride { - deltaSyncThresholdLine = fmt.Sprintf("delta_sync_threshold_seconds = %d\n", h.DeltaSyncThresholdSeconds) - } - - config := fmt.Sprintf(`# Marmot v2.9.16-beta Test Node %d -node_id = %d -data_dir = "%s" - -[transaction] -heartbeat_timeout_seconds = %d -conflict_window_seconds = 10 - -[connection_pool] -pool_size = 4 -max_idle_time_seconds = 10 -max_lifetime_seconds = 300 - -[grpc_client] -keepalive_time_seconds = 10 -keepalive_timeout_seconds = 3 -max_retries = 3 -retry_backoff_ms = 100 - -[coordinator] -prepare_timeout_ms = 5000 -commit_timeout_ms = 5000 -abort_timeout_ms = 3000 -intent_ttl_ms = 60000 -max_guard_rows = 65536 - -[cluster] -grpc_bind_address = "0.0.0.0" -grpc_advertise_address = "localhost:%d" -grpc_port = %d -seed_nodes = [%s] -gossip_interval_ms = 500 -gossip_fanout = 3 -suspect_timeout_ms = 10000 -dead_timeout_ms = 20000 -cluster_secret = "test-secret" - -[replication] -replication_factor = 3 -virtual_nodes = 150 -default_write_consistency = "QUORUM" -default_read_consistency = "LOCAL_ONE" -write_timeout_ms = 10000 -read_timeout_ms = 5000 -enable_anti_entropy = true -anti_entropy_interval_seconds = 5 -gc_interval_seconds = %d -%s%s -[mysql] -enabled = true -bind_address = "0.0.0.0" -port = %d -max_connections = 100 - -[logging] -verbose = false -format = "console" - -[prometheus] -enabled = false -`, - node.NodeID, - node.NodeID, - node.DataDir, - heartbeatTimeout, - node.GRPCPort, - node.GRPCPort, - strings.Join(seedNodes, ", "), - gcInterval, - gcMinRetentionLine, - deltaSyncThresholdLine, - node.MySQLPort, - ) - - if err := os.WriteFile(node.ConfigPath, []byte(config), 0644); err != nil { - h.t.Fatalf("Failed to write config for node %d: %v", node.NodeID, err) - } - - h.t.Logf("Created config for node %d (gRPC: %d, MySQL: %d)", node.NodeID, node.GRPCPort, node.MySQLPort) -} - -func (h *ClusterHarness) StartNode(nodeID int) error { - node := h.Nodes[nodeID-1] - node.mu.Lock() - defer node.mu.Unlock() - - if node.isRunning { - return fmt.Errorf("node %d is already running", nodeID) - } - - h.t.Logf("Starting node %d...", nodeID) - - // Append across restarts: a rolling-restart or kill/restart test needs - // every incarnation's log, not just the last one's. - logFile, err := os.OpenFile(node.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) - if err != nil { - return fmt.Errorf("failed to create log file: %v", err) - } - fmt.Fprintf(logFile, "=== node %d start %s ===\n", nodeID, time.Now().Format(time.RFC3339Nano)) - - bin := h.MarmotBin - if b, ok := h.NodeBin[nodeID]; ok { - bin = b - } - cmd := exec.Command(bin, "--config", node.ConfigPath) - cmd.Stdout = logFile - cmd.Stderr = logFile - - if err := cmd.Start(); err != nil { - logFile.Close() - return fmt.Errorf("failed to start node %d: %v", nodeID, err) - } - - node.Cmd = cmd - node.isRunning = true - - os.WriteFile(node.PIDFile, []byte(fmt.Sprintf("%d", cmd.Process.Pid)), 0644) - - h.t.Logf("Node %d started (PID: %d)", nodeID, cmd.Process.Pid) - return nil -} - -func (h *ClusterHarness) StopNode(nodeID int) error { - node := h.Nodes[nodeID-1] - node.mu.Lock() - defer node.mu.Unlock() - - if !node.isRunning { - return fmt.Errorf("node %d is not running", nodeID) - } - - pid := node.Cmd.Process.Pid - h.t.Logf("Stopping node %d (PID: %d)...", nodeID, pid) - - if node.DB != nil { - node.DB.Close() - node.DB = nil - } - - if err := node.Cmd.Process.Kill(); err != nil { - h.t.Logf("Warning: failed to kill node %d: %v", nodeID, err) - } - - node.Cmd.Wait() - node.isRunning = false - node.Cmd = nil - - os.Remove(node.PIDFile) - - h.t.Logf("Node %d stopped", nodeID) - return nil -} - -// GracefulStopNode sends SIGTERM and waits for the node's own shutdown - the -// path that announces LEAVING to its peers - unlike StopNode, which kills it. -func (h *ClusterHarness) GracefulStopNode(nodeID int, timeout time.Duration) error { - node := h.Nodes[nodeID-1] - node.mu.Lock() - defer node.mu.Unlock() - - if !node.isRunning { - return fmt.Errorf("node %d is not running", nodeID) - } - if node.DB != nil { - node.DB.Close() - node.DB = nil - } - if err := node.Cmd.Process.Signal(syscall.SIGTERM); err != nil { - return fmt.Errorf("SIGTERM node %d: %w", nodeID, err) - } - - exited := make(chan error, 1) - go func() { exited <- node.Cmd.Wait() }() - var err error - select { - case waitErr := <-exited: - if waitErr != nil { - err = fmt.Errorf("node %d exited with an error after SIGTERM: %w", nodeID, waitErr) - } else { - h.t.Logf("Node %d stopped gracefully", nodeID) - } - case <-time.After(timeout): - err = fmt.Errorf("node %d did not shut down within %v of SIGTERM", nodeID, timeout) - if killErr := node.Cmd.Process.Kill(); killErr != nil { - err = fmt.Errorf("%w; kill: %v", err, killErr) - } - <-exited - } - node.isRunning = false - node.Cmd = nil - os.Remove(node.PIDFile) - return err -} - -func (h *ClusterHarness) KillNode(nodeID int) error { - node := h.Nodes[nodeID-1] - node.mu.Lock() - defer node.mu.Unlock() - - if !node.isRunning { - return fmt.Errorf("node %d is not running", nodeID) - } - - h.t.Logf("Killing node %d (simulating crash)...", nodeID) - - if node.DB != nil { - node.DB.Close() - node.DB = nil - } - - if err := node.Cmd.Process.Kill(); err != nil { - return fmt.Errorf("failed to kill node %d: %v", nodeID, err) - } - - node.Cmd.Wait() - node.isRunning = false - node.Cmd = nil - - h.t.Logf("Node %d killed (crash simulated)", nodeID) - return nil -} - -func (h *ClusterHarness) WaitForAlive(nodeID int, timeout time.Duration) error { - h.t.Logf("Waiting for node %d to become ALIVE...", nodeID) - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - ticker := time.NewTicker(500 * time.Millisecond) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return fmt.Errorf("timeout waiting for node %d to become ALIVE; log tail:\n%s", nodeID, h.getNodeLogTail(nodeID, 40)) - case <-ticker.C: - db, err := h.ConnectToNode(nodeID) - if err != nil { - continue - } - - var result int - err = db.QueryRow("SELECT 1").Scan(&result) - if err == nil { - h.t.Logf("Node %d is ALIVE", nodeID) - return nil - } - } - } -} - -func (h *ClusterHarness) getNodeLogTail(nodeID int, lines int) string { - if nodeID < 1 || nodeID > len(h.Nodes) { - return "invalid node id" - } - data, err := os.ReadFile(h.Nodes[nodeID-1].LogFile) - if err != nil { - return fmt.Sprintf("failed to read log: %v", err) - } - all := strings.Split(string(data), "\n") - if lines <= 0 || len(all) <= lines { - return strings.Join(all, "\n") - } - return strings.Join(all[len(all)-lines:], "\n") -} - -func (h *ClusterHarness) ConnectToNode(nodeID int) (*sql.DB, error) { - node := h.Nodes[nodeID-1] - node.mu.Lock() - defer node.mu.Unlock() - - if node.DB != nil { - if err := node.DB.Ping(); err == nil { - return node.DB, nil - } - node.DB.Close() - node.DB = nil - } - - dsn := fmt.Sprintf("root:@tcp(localhost:%d)/marmot", node.MySQLPort) - db, err := sql.Open("mysql", dsn) - if err != nil { - return nil, err - } - - if err := db.Ping(); err != nil { - db.Close() - return nil, err - } - - node.DB = db - return db, nil -} - -func (h *ClusterHarness) QueryNode(nodeID int, query string, args ...interface{}) (*sql.Rows, error) { - db, err := h.ConnectToNode(nodeID) - if err != nil { - return nil, err - } - defer noteClientCall() - return db.Query(query, args...) -} - -func (h *ClusterHarness) ExecNode(nodeID int, query string, args ...interface{}) (sql.Result, error) { - db, err := h.ConnectToNode(nodeID) - if err != nil { - return nil, err - } - defer noteClientCall() - return db.Exec(query, args...) -} - -// tableExistsOnNode checks if a table exists on a node using SHOW TABLES (MySQL-compatible) -func (h *ClusterHarness) tableExistsOnNode(nodeID int, tableName string) bool { - rows, err := h.QueryNode(nodeID, "SHOW TABLES") - if err != nil { - return false - } - defer rows.Close() - - for rows.Next() { - var name string - if err := rows.Scan(&name); err != nil { - continue - } - if name == tableName { - return true - } - } - return false -} - -// WaitForTableExists waits for a table to exist on all specified nodes -func (h *ClusterHarness) WaitForTableExists(tableName string, nodeIDs []int, timeout time.Duration) error { - deadline := time.Now().Add(timeout) - - for _, nodeID := range nodeIDs { - for time.Now().Before(deadline) { - if h.tableExistsOnNode(nodeID, tableName) { - h.t.Logf("Table %s exists on node %d", tableName, nodeID) - break - } - time.Sleep(500 * time.Millisecond) - } - - if !h.tableExistsOnNode(nodeID, tableName) { - return fmt.Errorf("table %s not found on node %d after %v", tableName, nodeID, timeout) - } - } - - return nil -} - -// getRowCount returns the row count for a table on a node, or -1 on error -func (h *ClusterHarness) getRowCount(nodeID int, tableName string) int { - rows, err := h.QueryNode(nodeID, fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName)) - if err != nil { - return -1 - } - defer rows.Close() - - var count int - if rows.Next() { - rows.Scan(&count) - } - return count -} - -// WaitForRowCount waits for all specified nodes to have at least expectedCount rows -func (h *ClusterHarness) WaitForRowCount(tableName string, nodes []int, expectedCount int, timeout time.Duration) error { - if timeout < 15*time.Second { - timeout = 15 * time.Second - } - deadline := time.Now().Add(timeout) - for time.Now().Before(deadline) { - allMatch := true - for _, nodeID := range nodes { - count := h.getRowCount(nodeID, tableName) - if count < expectedCount { - allMatch = false - break - } - } - if allMatch { - return nil - } - time.Sleep(100 * time.Millisecond) - } - return fmt.Errorf("timeout waiting for %d rows on nodes %v", expectedCount, nodes) -} - -// ClusterMember represents a node in the cluster membership response -type ClusterMember struct { - NodeID uint64 `json:"node_id"` - Address string `json:"address"` - Status string `json:"status"` - Incarnation uint64 `json:"incarnation"` -} - -// AdminAPIResponse wraps the admin API response format -type AdminAPIResponse struct { - Data []ClusterMember `json:"data"` -} - -// votesHeld asks node's admin API whether its AUTO_INCREMENT claim votes are -// held (GET /admin/cluster/autoinc/votes). -func (h *ClusterHarness) votesHeld(nodeID int) (bool, error) { - url := fmt.Sprintf("http://localhost:%d/admin/cluster/autoinc/votes", h.Nodes[nodeID-1].GRPCPort) - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return false, err - } - req.Header.Set("X-Marmot-Secret", "test-secret") - resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req) - noteClientCall() - if err != nil { - return false, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return false, fmt.Errorf("status %d: %s", resp.StatusCode, string(body)) - } - var apiResp struct { - Data struct { - VotesHeld bool `json:"votes_held"` - } `json:"data"` - } - if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil { - return false, err - } - return apiResp.Data.VotesHeld, nil -} - -// WaitForVotesReleased polls until node's AUTO_INCREMENT claim votes are not -// held: a node whose system database is new declines every claim at PREPARE -// until it merged claim bases from enough members, so a narrow -// AUTO_INCREMENT insert right after a cluster's first start can return 1205 -// until then. -func (h *ClusterHarness) WaitForVotesReleased(nodeID int, timeout time.Duration) error { - deadline := time.Now().Add(timeout) - for { - h.Progress() - held, err := h.votesHeld(nodeID) - if err == nil && !held { - return nil - } - if time.Now().After(deadline) { - return fmt.Errorf("node %d claim votes still held after %v (err=%v); log tail:\n%s", nodeID, timeout, err, h.getNodeLogTail(nodeID, 40)) - } - time.Sleep(clusterConvergencePoll) - } -} - -// WaitForReady polls until node is ready for any write: it reports itself -// ALIVE (a JOINING node is still catching up) and its claim votes are not -// held (WaitForVotesReleased), both within one named deadline. -func (h *ClusterHarness) WaitForReady(nodeID int, timeout time.Duration) error { - deadline := time.Now().Add(timeout) - for { - h.Progress() - if status, err := h.selfStatus(nodeID); err == nil && status == "ALIVE" { - break - } - if time.Now().After(deadline) { - return fmt.Errorf("node %d not ALIVE within %v; log tail:\n%s", nodeID, timeout, h.getNodeLogTail(nodeID, 40)) - } - time.Sleep(clusterConvergencePoll) - } - return h.WaitForVotesReleased(nodeID, time.Until(deadline)) -} - -// selfStatus is node's own membership status as its admin API reports it. -func (h *ClusterHarness) selfStatus(nodeID int) (string, error) { - members, err := h.getClusterStatus(nodeID) - if err != nil { - return "", err - } - for _, m := range members { - if m.NodeID == uint64(nodeID) { - return m.Status, nil - } - } - return "", fmt.Errorf("node %d absent from its own membership", nodeID) -} - -// getClusterStatus fetches cluster status from a node's admin API -func (h *ClusterHarness) getClusterStatus(nodeID int) ([]ClusterMember, error) { - node := h.Nodes[nodeID-1] - url := fmt.Sprintf("http://localhost:%d/admin/cluster/members", node.GRPCPort) - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - req.Header.Set("X-Marmot-Secret", "test-secret") - - client := &http.Client{Timeout: 5 * time.Second} - resp, err := client.Do(req) - noteClientCall() - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("status %d: %s", resp.StatusCode, string(body)) - } - - var apiResp AdminAPIResponse - if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil { - return nil, err - } - - return apiResp.Data, nil -} - -// WaitForClusterConvergence waits until every node sees every node ALIVE -// in its cluster membership, polling each node's admin API until timeout. -func (h *ClusterHarness) WaitForClusterConvergence(timeout time.Duration) error { - h.t.Logf("Waiting for cluster to converge (all nodes ALIVE on every node)...") - - deadline := time.Now().Add(timeout) - for node := 1; node <= numNodes; node++ { - for { - alive, err := h.aliveCount(node) - if err == nil && alive >= numNodes { - break - } - if time.Now().After(deadline) { - return fmt.Errorf("cluster did not converge on node %d within %v (alive=%d, err=%v)", node, timeout, alive, err) - } - time.Sleep(clusterConvergencePoll) - } - } - h.t.Logf("Cluster converged: every node sees %d nodes ALIVE", numNodes) - return nil -} - -// aliveCount returns how many members node's cluster membership reports -// ALIVE. -func (h *ClusterHarness) aliveCount(node int) (int, error) { - members, err := h.getClusterStatus(node) - if err != nil { - return 0, err - } - alive := 0 - for _, m := range members { - if m.Status == "ALIVE" { - alive++ - } - } - return alive, nil -} - -// dumpNodeLogs saves node logs to /tmp for debugging -func (h *ClusterHarness) dumpNodeLogs(prefix string) { - for nodeID := 1; nodeID <= numNodes; nodeID++ { - logPath := h.Nodes[nodeID-1].LogFile - data, err := os.ReadFile(logPath) - if err != nil { - h.t.Logf("Node %d log error: %v", nodeID, err) - continue - } +// kvTable is the (id, value) table most crash tests write. +const kvTable = "CREATE TABLE %s (id INT PRIMARY KEY, value TEXT)" - tmpLogPath := fmt.Sprintf("/tmp/%s_node%d.log", prefix, nodeID) - os.WriteFile(tmpLogPath, data, 0644) - h.t.Logf("Node %d log saved to %s", nodeID, tmpLogPath) - - // Print last 20 lines - lines := strings.Split(string(data), "\n") - start := 0 - if len(lines) > 20 { - start = len(lines) - 20 - } - h.t.Logf("=== Node %d last 20 log lines ===", nodeID) - for _, line := range lines[start:] { - if line != "" { - h.t.Logf(" %s", line) - } - } +// insertEach inserts ids from..to into table through node id, one +// transaction per row, and returns the rows the table must then hold. +func insertEach(c *cluster, id int, table string, from, to int, tag string) []string { + c.t.Helper() + var want []string + for i := from; i <= to; i++ { + c.mustExec(id, "marmot", fmt.Sprintf("INSERT INTO %s (id, value) VALUES (?, ?)", table), i, fmt.Sprintf("%s_%d", tag, i)) + want = append(want, fmt.Sprintf("%d|%s_%d", i, tag, i)) } + c.progress() + return want } -func (h *ClusterHarness) StartCluster() error { - h.t.Logf("Starting %d-node cluster...", numNodes) - - for i := 1; i <= numNodes; i++ { - if err := h.StartNode(i); err != nil { - return err - } - } - - h.t.Logf("Waiting for MySQL servers to be ready...") - for i := 1; i <= numNodes; i++ { - if err := h.WaitForAlive(i, 30*time.Second); err != nil { - return err - } - } - - // Wait for gossip cluster to fully converge (all nodes ALIVE, as seen - // by every node). - if err := h.WaitForClusterConvergence(30 * time.Second); err != nil { - h.dumpNodeLogs("cluster_start_fail") - return err - } - - // Every node on the harness's own binary must be ready for writes. A - // node on an overridden binary (NodeBin) may predate the readiness - // endpoint; a test mixing binaries waits for readiness itself. - for i := 1; i <= numNodes; i++ { - if _, overridden := h.NodeBin[i]; overridden { - continue - } - if err := h.WaitForReady(i, clusterReadyDeadline); err != nil { - h.dumpNodeLogs("cluster_start_not_ready") - return err - } - } - - h.t.Logf("Cluster started successfully") - return nil -} - -func (h *ClusterHarness) StopCluster() { - h.t.Logf("Stopping cluster...") - for i := 1; i <= len(h.Nodes); i++ { - h.StopNode(i) - } +// kvRows is the query that lists an (id, value) table's rows in order. +func kvRows(table string) string { + return fmt.Sprintf("SELECT id, value FROM %s ORDER BY id", table) } -func (h *ClusterHarness) Cleanup() { - h.cleanupOnce.Do(func() { - h.watchdog.Stop() - h.budgetOnce.Do(func() { close(h.budgetDone) }) - h.t.Logf("Cleaning up cluster harness...") - h.StopCluster() - time.Sleep(500 * time.Millisecond) - if h.t.Failed() { - h.t.Logf("Preserving test artifacts for debugging: %s", h.BaseDir) - } else { - os.RemoveAll(h.BaseDir) - } - h.t.Logf("Cleanup complete") - }) -} - -// ======================= -// INTEGRATION TESTS -// ======================= - -// TestDDLReplication verifies that CREATE TABLE replicates to all nodes +// TestDDLReplication: CREATE TABLE through one node reaches every node. func TestDDLReplication(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - - tableName := "ddl_test_table" - t.Logf("Creating table %s on node 1...", tableName) - - _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, value TEXT)", tableName)) - if err != nil { - t.Fatalf("Failed to create table: %v", err) - } - - // Wait for DDL to replicate to all nodes - allNodes := []int{1, 2, 3} - if err := harness.WaitForTableExists(tableName, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("ddl_test") - t.Fatalf("DDL replication failed: %v", err) - } - harness.Phase("converged") - - t.Logf("SUCCESS: Table %s replicated to all nodes", tableName) + c := newCluster(t) + c.start() + c.createTable(1, "marmot", "ddl_test_table", fmt.Sprintf(kvTable, "ddl_test_table")) } -// TestBasicReplication verifies that INSERT replicates to all nodes +// TestBasicReplication: rows inserted through one node reach every node. func TestBasicReplication(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - harness.Phase("start") - - tableName := "basic_test" - - // Create table and wait for DDL replication - _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, value TEXT)", tableName)) - if err != nil { - t.Fatalf("Failed to create table: %v", err) - } - - allNodes := []int{1, 2, 3} - if err := harness.WaitForTableExists(tableName, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("basic_test") - t.Fatalf("DDL replication failed: %v", err) - } - - // Insert data - t.Logf("Inserting 10 rows on node 1...") - for i := 1; i <= 10; i++ { - _, err := harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (%d, 'value_%d')", tableName, i, i)) - if err != nil { - t.Fatalf("Insert failed: %v", err) - } - } - - // Wait for replication - if err := harness.WaitForRowCount(tableName, allNodes, 10, 15*time.Second); err != nil { - harness.dumpNodeLogs("basic_replication_fail") - t.Fatalf("rows did not replicate: %v", err) - } - harness.Phase("converged") - - for _, nodeID := range allNodes { - t.Logf("Node %d has %d rows", nodeID, harness.getRowCount(nodeID, tableName)) - } - - t.Logf("SUCCESS: Basic replication verified") + c := newCluster(t) + c.start() + c.createTable(1, "marmot", "basic_test", fmt.Sprintf(kvTable, "basic_test")) + want := insertEach(c, 1, "basic_test", 1, 10, "value") + c.waitRows("marmot", kvRows("basic_test"), want, 1, 2, 3) } -// TestNodeRestartRecovery tests that a restarted node catches up missed writes +// TestNodeRestartRecovery: a node killed while rows are written gets them +// after it restarts. func TestNodeRestartRecovery(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - harness.Phase("start") - - tableName := "recovery_test" - - // Create table - _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, value TEXT)", tableName)) - if err != nil { - t.Fatalf("Failed to create table: %v", err) - } - - allNodes := []int{1, 2, 3} - if err := harness.WaitForTableExists(tableName, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("recovery_test_ddl") - t.Fatalf("DDL replication failed: %v", err) - } + c := newCluster(t) + c.start() + c.createTable(1, "marmot", "recovery_test", fmt.Sprintf(kvTable, "recovery_test")) + want := insertEach(c, 1, "recovery_test", 1, 5, "initial") + c.waitRows("marmot", kvRows("recovery_test"), want, 1, 2, 3) - // Write initial data - t.Logf("Writing initial 5 rows...") - for i := 1; i <= 5; i++ { - _, err := harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (%d, 'initial_%d')", tableName, i, i)) - if err != nil { - t.Fatalf("Insert failed: %v", err) - } - } - if err := harness.WaitForRowCount(tableName, allNodes, 5, 15*time.Second); err != nil { - t.Fatalf("initial rows did not replicate: %v", err) - } - t.Logf("All nodes have 5 rows before crash") - harness.Phase("initial-load") + c.kill(3) + want = append(want, insertEach(c, 1, "recovery_test", 6, 10, "missed")...) + c.waitRows("marmot", kvRows("recovery_test"), want, 1, 2) - // Kill node 3 - t.Logf("Killing node 3...") - if err := harness.KillNode(3); err != nil { - t.Fatalf("Failed to kill node 3: %v", err) - } - harness.Phase("node3-killed") - - // Write more data while node 3 is down - t.Logf("Writing 5 more rows while node 3 is down...") - for i := 6; i <= 10; i++ { - _, err := harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (%d, 'missed_%d')", tableName, i, i)) - if err != nil { - t.Fatalf("Insert failed while node 3 down: %v", err) - } - } - if err := harness.WaitForRowCount(tableName, []int{1, 2}, 10, 15*time.Second); err != nil { - t.Fatalf("rows did not replicate to nodes 1&2: %v", err) - } - t.Logf("Nodes 1 & 2 have 10 rows") - harness.Phase("missed-load") - - // Restart node 3 - t.Logf("Restarting node 3...") - if err := harness.StartNode(3); err != nil { - t.Fatalf("Failed to restart node 3: %v", err) - } - if err := harness.WaitForAlive(3, 20*time.Second); err != nil { - t.Fatalf("Node 3 did not become ALIVE: %v", err) - } - harness.Phase("node3-alive") - - // Wait for anti-entropy to catch up - t.Logf("Waiting for node 3 to catch up via anti-entropy...") - if err := harness.WaitForRowCount(tableName, []int{3}, 10, 20*time.Second); err != nil { - harness.dumpNodeLogs("recovery_test_fail") - t.Fatalf("Node 3 did not catch up: %v", err) - } - harness.Phase("caught-up") - - t.Logf("SUCCESS: Node 3 caught up after restart") + c.restart(3) + c.waitRows("marmot", kvRows("recovery_test"), want, 1, 2, 3) } -// TestRollingRestart tests cluster availability during rolling restart. -// This test verifies that: -// 1. Writes continue to succeed (on remaining nodes) during rolling restarts -// 2. After all restarts complete, anti-entropy brings all nodes to eventual consistency -// -// Every wait here is a poll on a named deadline (WaitForRowCount / WaitForAlive), -// not a flat sleep, so the test finishes as soon as the cluster actually -// converges, keeping it well under the 60 s cluster test budget. -func TestRollingRestart(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - harness.Phase("start") - - tableName := "rolling_test" - - // Create table - _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, value TEXT)", tableName)) - if err != nil { - t.Fatalf("Failed to create table: %v", err) - } - - allNodes := []int{1, 2, 3} - if err := harness.WaitForTableExists(tableName, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("rolling_test_ddl") - t.Fatalf("DDL replication failed: %v", err) - } - - // Insert baseline data before rolling restart - const baseline = 20 - t.Logf("Inserting baseline %d rows before rolling restart...", baseline) - for i := 1; i <= baseline; i++ { - _, err := harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (%d, 'baseline_%d')", tableName, i, i)) - if err != nil { - t.Fatalf("Baseline insert failed: %v", err) - } - } - if err := harness.WaitForRowCount(tableName, allNodes, baseline, 15*time.Second); err != nil { - t.Fatalf("baseline rows did not replicate: %v", err) - } - t.Logf("All nodes have %d baseline rows", baseline) - harness.Phase("baseline-load") - - // Rolling restart each node - writes continue against the nodes still up. - expected := baseline - for nodeID := 1; nodeID <= 3; nodeID++ { - t.Logf("Rolling restart: stopping node %d...", nodeID) - if err := harness.StopNode(nodeID); err != nil { - t.Fatalf("Failed to stop node %d: %v", nodeID, err) - } - - // Write 5 rows while node is down (to nodes still running) - targetNode := (nodeID % 3) + 1 // Pick a different node - if nodeID == targetNode { - targetNode = ((nodeID + 1) % 3) + 1 - } - t.Logf("Writing 5 rows to node %d while node %d is down...", targetNode, nodeID) - for i := 0; i < 5; i++ { - rowID := baseline + (nodeID-1)*5 + i + 1 // Unique IDs: 21-25, 26-30, 31-35 - _, err := harness.ExecNode(targetNode, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (%d, 'rolling_%d')", tableName, rowID, rowID)) - if err != nil { - t.Logf("Warning: write during rolling restart failed: %v", err) - continue - } - expected++ - } - - var stillUp []int - for _, n := range allNodes { - if n != nodeID { - stillUp = append(stillUp, n) - } - } - if err := harness.WaitForRowCount(tableName, stillUp, expected, 10*time.Second); err != nil { - t.Fatalf("writes during node %d's downtime did not land on the up nodes: %v", nodeID, err) - } - - if err := harness.StartNode(nodeID); err != nil { - t.Fatalf("Failed to restart node %d: %v", nodeID, err) - } - if err := harness.WaitForAlive(nodeID, 20*time.Second); err != nil { - t.Fatalf("Node %d did not become ALIVE: %v", nodeID, err) - } - t.Logf("Node %d restarted", nodeID) - - if err := harness.WaitForRowCount(tableName, []int{nodeID}, expected, 20*time.Second); err != nil { - harness.dumpNodeLogs("rolling_test_catchup") - t.Fatalf("node %d did not catch up via anti-entropy: %v", nodeID, err) - } - harness.Phase(fmt.Sprintf("node%d-rolled", nodeID)) - } - - // Final verification - all nodes converge on the same count. - if err := harness.WaitForRowCount(tableName, allNodes, expected, 15*time.Second); err != nil { - harness.dumpNodeLogs("rolling_test_fail") - t.Fatalf("cluster did not converge: %v", err) - } - harness.Phase("converged") - - finalCount := harness.getRowCount(1, tableName) - t.Logf("SUCCESS: Rolling restart maintained availability and consistency (final count: %d)", finalCount) -} - -// TestCrashMidTransaction tests that uncommitted transactions are rolled back +// TestCrashMidTransaction: rows an explicit transaction wrote but never +// committed are gone once its coordinator is killed: the other nodes never +// see them, and neither does the coordinator after it restarts and catches +// up. A committed marker row is the barrier that proves each node has +// applied everything the cluster committed before the check. func TestCrashMidTransaction(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - harness.Phase("start") - - tableName := "crash_txn_test" - - // Create table - _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, value TEXT)", tableName)) - if err != nil { - t.Fatalf("Failed to create table: %v", err) - } - - allNodes := []int{1, 2, 3} - if err := harness.WaitForTableExists(tableName, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("crash_txn_ddl") - t.Fatalf("DDL replication failed: %v", err) - } - - // Start a transaction - t.Logf("Starting transaction on node 1...") - db, err := harness.ConnectToNode(1) - if err != nil { - t.Fatalf("Failed to connect: %v", err) - } + c := newCluster(t) + c.start() + c.createTable(1, "marmot", "crash_txn_test", fmt.Sprintf(kvTable, "crash_txn_test")) - tx, err := db.Begin() + tx, err := c.db(1, "marmot").Begin() if err != nil { - t.Fatalf("Failed to begin transaction: %v", err) + t.Fatal(err) } - - // Insert in transaction for i := 1; i <= 5; i++ { - _, err := tx.Exec(fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (%d, 'uncommitted_%d')", tableName, i, i)) - if err != nil { - t.Fatalf("Insert in transaction failed: %v", err) + if _, err := tx.Exec("INSERT INTO crash_txn_test (id, value) VALUES (?, ?)", i, fmt.Sprintf("uncommitted_%d", i)); err != nil { + t.Fatalf("insert %d in the open transaction: %v", i, err) } } + c.kill(1) + _ = tx.Rollback() - // Kill node 1 before commit - t.Logf("Killing node 1 before commit...") - time.Sleep(100 * time.Millisecond) - harness.KillNode(1) - - time.Sleep(2 * time.Second) - - // Verify nodes 2 & 3 have no data (transaction not committed) - for _, nodeID := range []int{2, 3} { - count := harness.getRowCount(nodeID, tableName) - if count != 0 { - t.Fatalf("Node %d has %d rows, expected 0 (uncommitted transaction should not replicate)", nodeID, count) - } - } - t.Logf("Nodes 2 & 3 correctly have 0 rows") - - // Restart node 1 - t.Logf("Restarting node 1...") - if err := harness.StartNode(1); err != nil { - t.Fatalf("Failed to restart node 1: %v", err) - } - if err := harness.WaitForAlive(1, 15*time.Second); err != nil { - t.Fatalf("Node 1 did not become ALIVE: %v", err) - } - harness.Phase("node1-alive") - - time.Sleep(3 * time.Second) - - // Node 1 should also have 0 rows (transaction rolled back on crash) - count := harness.getRowCount(1, tableName) - if count != 0 { - t.Fatalf("Node 1 has %d rows after restart, expected 0", count) - } + marker := insertEach(c, 2, "crash_txn_test", 100, 100, "marker") + c.waitRows("marmot", kvRows("crash_txn_test"), marker, 2, 3) - t.Logf("SUCCESS: Uncommitted transaction correctly rolled back") + c.restart(1) + c.waitRows("marmot", kvRows("crash_txn_test"), marker, 1, 2, 3) } -// TestHighLoadRecovery tests node catch-up after missing writes during downtime +// TestHighLoadRecovery: a node killed while the cluster keeps committing +// rows has every one of them, and no other, after it restarts. func TestHighLoadRecovery(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() + c := newCluster(t) + c.start() + c.createTable(1, "marmot", "highload_test", fmt.Sprintf(kvTable, "highload_test")) + want := insertEach(c, 1, "highload_test", 1, 20, "initial") + c.waitRows("marmot", kvRows("highload_test"), want, 1, 2, 3) - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - harness.Phase("start") - - tableName := "highload_test" - allNodes := []int{1, 2, 3} - - // Create table - _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, value TEXT)", tableName)) - if err != nil { - t.Fatalf("Failed to create table: %v", err) - } - if err := harness.WaitForTableExists(tableName, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("highload_ddl") - t.Fatalf("DDL replication failed: %v", err) - } - - // Phase 1: Insert initial batch with all nodes up - initialRows := 20 - t.Logf("Phase 1: Inserting %d rows with all nodes up...", initialRows) - for i := 1; i <= initialRows; i++ { - _, err := harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (%d, 'initial_%d')", tableName, i, i)) - if err != nil { - t.Fatalf("Failed to insert row %d: %v", i, err) - } - } - - // Wait for all nodes to have initial rows - if err := harness.WaitForRowCount(tableName, allNodes, initialRows, 30*time.Second); err != nil { - harness.dumpNodeLogs("highload_initial") - t.Fatalf("Initial replication failed: %v", err) - } - t.Logf("Phase 1 complete: All nodes have %d rows", initialRows) - harness.Phase("initial-load") - - // Phase 2: Kill node 3 - t.Logf("Phase 2: Killing node 3...") - harness.KillNode(3) - harness.Phase("node3-killed") - - // Phase 3: Insert more rows while node 3 is down - additionalRows := 30 - t.Logf("Phase 3: Inserting %d rows while node 3 is down...", additionalRows) - for i := initialRows + 1; i <= initialRows+additionalRows; i++ { - _, err := harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (%d, 'during_down_%d')", tableName, i, i)) - if err != nil { - t.Fatalf("Failed to insert row %d: %v", i, err) - } - } - - // Verify nodes 1 & 2 have all rows - expectedTotal := initialRows + additionalRows - if err := harness.WaitForRowCount(tableName, []int{1, 2}, expectedTotal, 30*time.Second); err != nil { - harness.dumpNodeLogs("highload_replication") - t.Fatalf("Replication to nodes 1&2 failed: %v", err) - } - t.Logf("Phase 3 complete: Nodes 1 & 2 have %d rows", expectedTotal) - harness.Phase("missed-load") - - // Phase 4: Restart node 3 - t.Logf("Phase 4: Restarting node 3...") - if err := harness.StartNode(3); err != nil { - t.Fatalf("Failed to restart node 3: %v", err) - } - if err := harness.WaitForAlive(3, 30*time.Second); err != nil { - harness.dumpNodeLogs("highload_restart_alive") - t.Fatalf("Node 3 did not become ALIVE: %v", err) - } - harness.Phase("node3-alive") + c.kill(3) + want = append(want, insertEach(c, 1, "highload_test", 21, 50, "during_down")...) + c.waitRows("marmot", kvRows("highload_test"), want, 1, 2) - // Phase 5: Wait for node 3 to catch up - t.Logf("Phase 5: Waiting for node 3 to catch up to %d rows...", expectedTotal) - if err := harness.WaitForRowCount(tableName, []int{3}, expectedTotal, 25*time.Second); err != nil { - harness.dumpNodeLogs("highload_catchup") - t.Fatalf("Node 3 failed to catch up: %v", err) - } - harness.Phase("caught-up") - - // Final verification - all nodes have exact same count - for _, nodeID := range allNodes { - count := harness.getRowCount(nodeID, tableName) - if count != expectedTotal { - harness.dumpNodeLogs("highload_final") - t.Fatalf("Node %d has %d rows, expected %d", nodeID, count, expectedTotal) - } - t.Logf("Node %d has %d rows", nodeID, count) - } + c.restart(3) + c.waitRows("marmot", kvRows("highload_test"), want, 1, 2, 3) +} - t.Logf("SUCCESS: Node 3 recovered and caught up to %d rows", expectedTotal) +// TestRollingRestart: each node in turn is killed and restarted; every write +// through a surviving node while one is down succeeds, and each restarted node +// catches up, so the cluster ends with every row on every node. +func TestRollingRestart(t *testing.T) { + c := newCluster(t) + c.start() + c.createTable(1, "marmot", "rolling_test", fmt.Sprintf(kvTable, "rolling_test")) + want := insertEach(c, 1, "rolling_test", 1, 20, "baseline") + c.waitRows("marmot", kvRows("rolling_test"), want, 1, 2, 3) + c.phase("baseline") + + for down := 1; down <= 3; down++ { + c.kill(down) + up := slices.DeleteFunc([]int{1, 2, 3}, func(n int) bool { return n == down }) + from := 20 + (down-1)*5 + 1 + retried := 0 + for id := from; id < from+5; id++ { + retried += c.execAcrossReconnect(down%3+1, "marmot", "INSERT INTO rolling_test (id, value) VALUES (?, ?)", id, fmt.Sprintf("rolling_%d", id)) + want = append(want, fmt.Sprintf("%d|rolling_%d", id, id)) + } + t.Logf("node %d down: %d write attempts failed while a restarted peer reconnected", down, retried) + c.waitRows("marmot", kvRows("rolling_test"), want, up...) + c.restart(down) + c.waitRows("marmot", kvRows("rolling_test"), want, down) + c.phase(fmt.Sprintf("node-%d-restarted", down)) + } + c.waitRows("marmot", kvRows("rolling_test"), want, 1, 2, 3) } diff --git a/test/ddl_replication_test.go b/test/ddl_replication_test.go index ba0beeb9..2a9ad00a 100644 --- a/test/ddl_replication_test.go +++ b/test/ddl_replication_test.go @@ -4,8 +4,6 @@ import ( "context" "database/sql" "fmt" - "os" - "sync" "testing" "time" @@ -13,113 +11,9 @@ import ( "github.com/maxpert/marmot/db" "github.com/maxpert/marmot/hlc" "github.com/maxpert/marmot/protocol" - "github.com/maxpert/marmot/protocol/query/transform" "github.com/stretchr/testify/require" ) -// DDLMockDatabaseManager implements coordinator.DatabaseManager for testing -type DDLMockDatabaseManager struct { - mu sync.RWMutex - databases map[string]*sql.DB -} - -func NewDDLMockDatabaseManager() *DDLMockDatabaseManager { - return &DDLMockDatabaseManager{ - databases: make(map[string]*sql.DB), - } -} - -func (m *DDLMockDatabaseManager) ListDatabases() []string { - m.mu.RLock() - defer m.mu.RUnlock() - - names := make([]string, 0, len(m.databases)) - for name := range m.databases { - names = append(names, name) - } - return names -} - -func (m *DDLMockDatabaseManager) DatabaseExists(name string) bool { - m.mu.RLock() - defer m.mu.RUnlock() - _, exists := m.databases[name] - return exists -} - -func (m *DDLMockDatabaseManager) CreateDatabase(name string) error { - m.mu.Lock() - defer m.mu.Unlock() - - if _, exists := m.databases[name]; exists { - return nil // Idempotent - } - - // Create in-memory SQLite database - sqlDB, err := sql.Open("sqlite3_marmot", ":memory:") - if err != nil { - return err - } - - m.databases[name] = sqlDB - return nil -} - -func (m *DDLMockDatabaseManager) DropDatabase(name string) error { - m.mu.Lock() - defer m.mu.Unlock() - - if sqlDB, exists := m.databases[name]; exists { - sqlDB.Close() - delete(m.databases, name) - } - return nil -} - -func (m *DDLMockDatabaseManager) GetDatabaseConnection(name string) (*sql.DB, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - sqlDB, exists := m.databases[name] - if !exists { - return nil, fmt.Errorf("database '%s' does not exist", name) - } - return sqlDB, nil -} - -func (m *DDLMockDatabaseManager) GetDatabaseReadConnection(name string) (*sql.DB, error) { - return m.GetDatabaseConnection(name) -} - -func (m *DDLMockDatabaseManager) GetReplicatedDatabase(name string) (coordinator.ReplicatedDatabaseProvider, error) { - // Not implemented in DDL mock - return nil, fmt.Errorf("not implemented in DDL mock") -} - -func (m *DDLMockDatabaseManager) GetAutoIncrementColumn(database, table string) (string, error) { - // Not implemented in DDL mock - return empty - return "", nil -} - -func (m *DDLMockDatabaseManager) GetTranspilerSchema(database, table string) (*transform.SchemaInfo, error) { - // Not implemented in DDL mock - return nil, nil -} - -func (m *DDLMockDatabaseManager) GetVectorIndexManager() coordinator.VectorIndexManagerProvider { - return nil -} - -// RegistryKeyGeneration reports generation 1 for any database this mock is -// tracking (live) and 0 for any it is not (never seen); the DDL mock has no -// tombstone/generation history of its own to model. -func (m *DDLMockDatabaseManager) RegistryKeyGeneration(name string) (uint64, bool, error) { - if m.DatabaseExists(name) { - return 1, true, nil - } - return 0, false, nil -} - // DDLMockReader implements coordinator.Reader for testing type DDLMockReader struct{} @@ -352,9 +246,7 @@ func commitDDLForTest(t *testing.T, dbMgr *db.DatabaseManager, database, ddlSQL, // through SchemaVersionManager and bumped atomically by each committed DDL // transaction - not by a direct setter, which no longer exists. func TestSchemaVersionManager(t *testing.T) { - tmpDir, err := testDataDir("schema-version-test-*") - require.NoError(t, err) - defer os.RemoveAll(tmpDir) + tmpDir := testDir(t) clock := hlc.NewClock(1) dbMgr, err := db.NewDatabaseManager(tmpDir, 1, clock) @@ -495,13 +387,15 @@ func TestDDLLockManager(t *testing.T) { lock, err := shortLockMgr.AcquireLock("expiredb", 1, 3001, ts) require.NoError(t, err) require.NotNil(t, lock) - - // Wait for lock to expire - time.Sleep(150 * time.Millisecond) - - // Another transaction should be able to acquire the expired lock - lock2, err := shortLockMgr.AcquireLock("expiredb", 2, 3002, ts) - require.NoError(t, err) + _, err = shortLockMgr.AcquireLock("expiredb", 2, 3002, ts) + require.Error(t, err, "the lock is held until it expires") + + // Once the lock expires, another transaction acquires it. + var lock2 *coordinator.DDLLock + waitFor(t, "expired DDL lock acquired by another transaction", time.Second, func() (bool, string) { + lock2, err = shortLockMgr.AcquireLock("expiredb", 2, 3002, ts) + return err == nil, fmt.Sprint(err) + }) require.NotNil(t, lock2) require.Equal(t, uint64(2), lock2.NodeID) }) @@ -513,10 +407,8 @@ func TestDDLReplicationBasic(t *testing.T) { // Create temp directory for the real DatabaseManager. Schema versions are // now read from each database's own __marmot_schema_version SQLite table, // so DDL must commit for real through a real DatabaseManager - - // the DDLMockDatabaseManager has no such table. - tmpDir, err := testDataDir("ddl-replication-test-*") - require.NoError(t, err) - defer os.RemoveAll(tmpDir) + // a mock database manager has no such table. + tmpDir := testDir(t) clock := hlc.NewClock(1) dbMgr, err := db.NewDatabaseManager(tmpDir, 1, clock) @@ -611,9 +503,7 @@ func TestDDLWithConcurrentDML(t *testing.T) { // Setup test infrastructure. Schema versions are read from each // database's own __marmot_schema_version SQLite table, so this // needs a real DatabaseManager, not DDLMockDatabaseManager. - tmpDir, err := testDataDir("ddl-concurrent-dml-test-*") - require.NoError(t, err) - defer os.RemoveAll(tmpDir) + tmpDir := testDir(t) clock := hlc.NewClock(1) dbMgr, err := db.NewDatabaseManager(tmpDir, 1, clock) @@ -768,47 +658,20 @@ func TestDDLSerializationWithLocking(t *testing.T) { lockMgr := coordinator.NewDDLLockManager(5 * time.Second) clock := hlc.NewClock(1) - // Simulate two concurrent DDL operations on the same database + // Two DDL operations on the same database: while the first holds the + // lock, the second is refused; once the first releases it, the second + // acquires it. t.Run("Concurrent DDL operations serialized", func(t *testing.T) { - var wg sync.WaitGroup - errors := make([]error, 2) - - // Transaction 1: Acquire lock first - wg.Add(1) - go func() { - defer wg.Done() - ts := clock.Now() - lock, err := lockMgr.AcquireLock("testdb", 1, 1001, ts) - if err != nil { - errors[0] = err - return - } - - // Hold lock briefly - time.Sleep(100 * time.Millisecond) - - lockMgr.ReleaseLock("testdb", 1001) - _ = lock - }() - - // Transaction 2: Try to acquire lock (should wait/fail) - time.Sleep(10 * time.Millisecond) // Ensure txn1 acquires first - wg.Add(1) - go func() { - defer wg.Done() - ts := clock.Now() - _, err := lockMgr.AcquireLock("testdb", 2, 1002, ts) - errors[1] = err - }() - - wg.Wait() + _, err := lockMgr.AcquireLock("testdb", 1, 1001, clock.Now()) + require.NoError(t, err, "first DDL should acquire lock") - // First transaction should succeed - require.NoError(t, errors[0], "first DDL should acquire lock") + _, err = lockMgr.AcquireLock("testdb", 2, 1002, clock.Now()) + require.Error(t, err, "second DDL should fail to acquire lock") + require.Contains(t, err.Error(), "DDL lock for database 'testdb' is held") - // Second transaction should fail (lock held) - require.Error(t, errors[1], "second DDL should fail to acquire lock") - require.Contains(t, errors[1].Error(), "DDL lock for database 'testdb' is held") + require.NoError(t, lockMgr.ReleaseLock("testdb", 1001)) + _, err = lockMgr.AcquireLock("testdb", 2, 1002, clock.Now()) + require.NoError(t, err, "second DDL should acquire the released lock") }) } diff --git a/test/dml_operations_test.go b/test/dml_operations_test.go index 1dcb0ba3..7a6105a1 100644 --- a/test/dml_operations_test.go +++ b/test/dml_operations_test.go @@ -1,536 +1,91 @@ package test import ( - "database/sql" "fmt" "testing" - "time" ) -// waitForRowCountOnNode waits for expected row count on a specific node -func waitForRowCountOnNode(t *testing.T, harness *ClusterHarness, nodeID int, table string, expected int, timeout time.Duration) error { - if timeout < 15*time.Second { - timeout = 15 * time.Second - } - deadline := time.Now().Add(timeout) - for time.Now().Before(deadline) { - count := harness.getRowCount(nodeID, table) - if count == expected { - return nil - } - time.Sleep(100 * time.Millisecond) - } - actual := harness.getRowCount(nodeID, table) - return fmt.Errorf("node %d: timeout waiting for %d rows, got %d", nodeID, expected, actual) -} - -// verifyRowValue checks if a specific row has expected value -func verifyRowValue(t *testing.T, db *sql.DB, table string, id int, expectedValue string) error { - var value string - query := fmt.Sprintf("SELECT value FROM %s WHERE id = ?", table) - err := db.QueryRow(query, id).Scan(&value) - if err != nil { - return fmt.Errorf("failed to query row %d: %v", id, err) - } - if value != expectedValue { - return fmt.Errorf("row %d: expected value '%s', got '%s'", id, expectedValue, value) - } - return nil +// startKV starts a cluster with an (id, value) table named table. +func startKV(t *testing.T, table string) *cluster { + c := newCluster(t) + c.start() + c.createTable(1, "marmot", table, fmt.Sprintf(kvTable, table)) + return c } -// TestInsertReplication tests single and batch insert replication +// TestInsertReplication: a single-row INSERT, then several more through +// another node, reach every node. func TestInsertReplication(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - - tableName := "insert_test" - allNodes := []int{1, 2, 3} - - t.Logf("Creating table %s on node 1...", tableName) - _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, value TEXT)", tableName)) - if err != nil { - t.Fatalf("Failed to create table: %v", err) - } - - if err := harness.WaitForTableExists(tableName, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("insert_test_ddl") - t.Fatalf("DDL replication failed: %v", err) - } - - // Test 1: Insert single row on node 1 - t.Logf("Test 1: Inserting single row on node 1...") - _, err = harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (1, 'single_row')", tableName)) - if err != nil { - t.Fatalf("Failed to insert single row: %v", err) - } - - // Verify it appears on nodes 2 and 3 - for _, nodeID := range []int{2, 3} { - if err := waitForRowCountOnNode(t, harness, nodeID, tableName, 1, 5*time.Second); err != nil { - harness.dumpNodeLogs("insert_single_fail") - t.Fatalf("Single row insert replication failed on node %d: %v", nodeID, err) - } - t.Logf("Node %d has single row", nodeID) - } - - // Test 2: Insert multiple rows in a batch on node 2 - t.Logf("Test 2: Inserting 5 rows in batch on node 2...") - for i := 2; i <= 6; i++ { - _, err := harness.ExecNode(2, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (%d, 'batch_%d')", tableName, i, i)) - if err != nil { - t.Fatalf("Failed to insert batch row %d: %v", i, err) - } - } - - // Verify all 6 rows appear on nodes 1 and 3 - for _, nodeID := range []int{1, 3} { - if err := waitForRowCountOnNode(t, harness, nodeID, tableName, 6, 5*time.Second); err != nil { - harness.dumpNodeLogs("insert_batch_fail") - t.Fatalf("Batch insert replication failed on node %d: %v", nodeID, err) - } - t.Logf("Node %d has 6 rows", nodeID) - } - - t.Logf("SUCCESS: Insert replication verified") + c := startKV(t, "insert_test") + want := insertEach(c, 1, "insert_test", 1, 1, "single_row") + c.waitRows("marmot", kvRows("insert_test"), want, 1, 2, 3) + want = append(want, insertEach(c, 2, "insert_test", 2, 6, "batch")...) + c.waitRows("marmot", kvRows("insert_test"), want, 1, 2, 3) } -// TestUpdateReplication tests update replication +// TestUpdateReplication: a single-row UPDATE and a multi-row UPDATE, each +// through a different node, reach every node. func TestUpdateReplication(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - - tableName := "update_test" - allNodes := []int{1, 2, 3} - - t.Logf("Creating table %s on node 1...", tableName) - _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, value TEXT)", tableName)) - if err != nil { - t.Fatalf("Failed to create table: %v", err) - } - - if err := harness.WaitForTableExists(tableName, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("update_test_ddl") - t.Fatalf("DDL replication failed: %v", err) - } - - // Insert initial rows on node 1 - t.Logf("Inserting 3 initial rows on node 1...") - for i := 1; i <= 3; i++ { - _, err := harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (%d, 'initial_%d')", tableName, i, i)) - if err != nil { - t.Fatalf("Failed to insert row %d: %v", i, err) - } - } - - // Wait for replication - if err := harness.WaitForRowCount(tableName, allNodes, 3, 5*time.Second); err != nil { - harness.dumpNodeLogs("update_test_insert") - t.Fatalf("Insert replication failed: %v", err) - } - t.Logf("All nodes have 3 initial rows") - - // Test 1: Update single row on node 2 - t.Logf("Test 1: Updating row id=1 on node 2...") - _, err = harness.ExecNode(2, fmt.Sprintf( - "UPDATE %s SET value = 'updated_1' WHERE id = 1", tableName)) - if err != nil { - t.Fatalf("Failed to update row: %v", err) - } - - // Verify update propagates to nodes 1 and 3 - time.Sleep(2 * time.Second) - for _, nodeID := range []int{1, 3} { - db, err := harness.ConnectToNode(nodeID) - if err != nil { - t.Fatalf("Failed to connect to node %d: %v", nodeID, err) - } - if err := verifyRowValue(t, db, tableName, 1, "updated_1"); err != nil { - harness.dumpNodeLogs("update_single_fail") - t.Fatalf("Update replication failed on node %d: %v", nodeID, err) - } - t.Logf("Node %d has updated value for id=1", nodeID) - } - - // Test 2: Update multiple rows with WHERE clause - t.Logf("Test 2: Updating multiple rows on node 3...") - _, err = harness.ExecNode(3, fmt.Sprintf( - "UPDATE %s SET value = 'batch_updated' WHERE id > 1", tableName)) - if err != nil { - t.Fatalf("Failed to batch update: %v", err) - } - - // Verify updates propagate to nodes 1 and 2 - time.Sleep(2 * time.Second) - for _, nodeID := range []int{1, 2} { - db, err := harness.ConnectToNode(nodeID) - if err != nil { - t.Fatalf("Failed to connect to node %d: %v", nodeID, err) - } - for _, id := range []int{2, 3} { - if err := verifyRowValue(t, db, tableName, id, "batch_updated"); err != nil { - harness.dumpNodeLogs("update_batch_fail") - t.Fatalf("Batch update replication failed on node %d: %v", nodeID, err) - } - } - t.Logf("Node %d has batch updated values", nodeID) - } - - t.Logf("SUCCESS: Update replication verified") + c := startKV(t, "update_test") + insertEach(c, 1, "update_test", 1, 3, "initial") + c.mustExec(2, "marmot", "UPDATE update_test SET value = 'updated_1' WHERE id = 1") + c.waitRows("marmot", kvRows("update_test"), []string{"1|updated_1", "2|initial_2", "3|initial_3"}, 1, 2, 3) + c.mustExec(3, "marmot", "UPDATE update_test SET value = 'batch_updated' WHERE id > 1") + c.waitRows("marmot", kvRows("update_test"), []string{"1|updated_1", "2|batch_updated", "3|batch_updated"}, 1, 2, 3) } -// TestDeleteReplication tests delete replication +// TestDeleteReplication: a single-row DELETE and a multi-row DELETE, each +// through a different node, reach every node. func TestDeleteReplication(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - - tableName := "delete_test" - allNodes := []int{1, 2, 3} - - t.Logf("Creating table %s on node 1...", tableName) - _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, value TEXT)", tableName)) - if err != nil { - t.Fatalf("Failed to create table: %v", err) - } - - if err := harness.WaitForTableExists(tableName, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("delete_test_ddl") - t.Fatalf("DDL replication failed: %v", err) - } - - // Insert initial rows on node 1 - t.Logf("Inserting 5 initial rows on node 1...") - for i := 1; i <= 5; i++ { - _, err := harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (%d, 'row_%d')", tableName, i, i)) - if err != nil { - t.Fatalf("Failed to insert row %d: %v", i, err) - } - } - - // Wait for replication - if err := harness.WaitForRowCount(tableName, allNodes, 5, 5*time.Second); err != nil { - harness.dumpNodeLogs("delete_test_insert") - t.Fatalf("Insert replication failed: %v", err) - } - t.Logf("All nodes have 5 initial rows") - - // Test 1: Delete single row on node 2 - t.Logf("Test 1: Deleting row id=1 on node 2...") - _, err = harness.ExecNode(2, fmt.Sprintf( - "DELETE FROM %s WHERE id = 1", tableName)) - if err != nil { - t.Fatalf("Failed to delete row: %v", err) - } - - // Verify delete propagates to nodes 1 and 3 - for _, nodeID := range []int{1, 3} { - if err := waitForRowCountOnNode(t, harness, nodeID, tableName, 4, 5*time.Second); err != nil { - harness.dumpNodeLogs("delete_single_fail") - t.Fatalf("Delete replication failed on node %d: %v", nodeID, err) - } - t.Logf("Node %d has 4 rows after delete", nodeID) - } - - // Test 2: Delete multiple rows with WHERE clause - t.Logf("Test 2: Deleting multiple rows on node 3...") - _, err = harness.ExecNode(3, fmt.Sprintf( - "DELETE FROM %s WHERE id > 3", tableName)) - if err != nil { - t.Fatalf("Failed to batch delete: %v", err) - } - - // Verify deletes propagate to nodes 1 and 2 - for _, nodeID := range []int{1, 2} { - if err := waitForRowCountOnNode(t, harness, nodeID, tableName, 2, 5*time.Second); err != nil { - harness.dumpNodeLogs("delete_batch_fail") - t.Fatalf("Batch delete replication failed on node %d: %v", nodeID, err) - } - t.Logf("Node %d has 2 rows after batch delete", nodeID) - } - - t.Logf("SUCCESS: Delete replication verified") + c := startKV(t, "delete_test") + want := insertEach(c, 1, "delete_test", 1, 5, "row") + c.mustExec(2, "marmot", "DELETE FROM delete_test WHERE id = 1") + c.waitRows("marmot", kvRows("delete_test"), want[1:], 1, 2, 3) + c.mustExec(3, "marmot", "DELETE FROM delete_test WHERE id > 3") + c.waitRows("marmot", kvRows("delete_test"), want[1:3], 1, 2, 3) } -// TestInsertUpdateDelete_Sequential tests insert, update, delete on different nodes +// TestInsertUpdateDelete_Sequential: one row inserted, updated and deleted, +// each through a different node, is in the same state on every node after +// each step. func TestInsertUpdateDelete_Sequential(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - - tableName := "sequential_test" - allNodes := []int{1, 2, 3} - - t.Logf("Creating table %s on node 1...", tableName) - _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, value TEXT)", tableName)) - if err != nil { - t.Fatalf("Failed to create table: %v", err) - } - - if err := harness.WaitForTableExists(tableName, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("sequential_test_ddl") - t.Fatalf("DDL replication failed: %v", err) - } - - // Step 1: Insert row on node 1 - t.Logf("Step 1: Inserting row on node 1...") - _, err = harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (1, 'initial')", tableName)) - if err != nil { - t.Fatalf("Failed to insert row: %v", err) - } - - // Wait for insert replication - if err := harness.WaitForRowCount(tableName, allNodes, 1, 5*time.Second); err != nil { - harness.dumpNodeLogs("sequential_insert_fail") - t.Fatalf("Insert replication failed: %v", err) - } - t.Logf("All nodes have 1 row") - - // Step 2: Update row on node 2 - t.Logf("Step 2: Updating row on node 2...") - _, err = harness.ExecNode(2, fmt.Sprintf( - "UPDATE %s SET value = 'updated' WHERE id = 1", tableName)) - if err != nil { - t.Fatalf("Failed to update row: %v", err) - } - - // Wait and verify update - time.Sleep(2 * time.Second) - for _, nodeID := range allNodes { - db, err := harness.ConnectToNode(nodeID) - if err != nil { - t.Fatalf("Failed to connect to node %d: %v", nodeID, err) - } - if err := verifyRowValue(t, db, tableName, 1, "updated"); err != nil { - harness.dumpNodeLogs("sequential_update_fail") - t.Fatalf("Update replication failed on node %d: %v", nodeID, err) - } - } - t.Logf("All nodes have updated value") - - // Step 3: Delete row on node 3 - t.Logf("Step 3: Deleting row on node 3...") - _, err = harness.ExecNode(3, fmt.Sprintf( - "DELETE FROM %s WHERE id = 1", tableName)) - if err != nil { - t.Fatalf("Failed to delete row: %v", err) - } - - // Verify final state: row deleted on all nodes - for _, nodeID := range allNodes { - if err := waitForRowCountOnNode(t, harness, nodeID, tableName, 0, 5*time.Second); err != nil { - harness.dumpNodeLogs("sequential_delete_fail") - t.Fatalf("Delete replication failed on node %d: %v", nodeID, err) - } - } - t.Logf("All nodes have 0 rows (final state consistent)") - - t.Logf("SUCCESS: Sequential insert/update/delete verified") + c := startKV(t, "sequential_test") + c.mustExec(1, "marmot", "INSERT INTO sequential_test (id, value) VALUES (1, 'initial')") + c.waitRows("marmot", kvRows("sequential_test"), []string{"1|initial"}, 1, 2, 3) + c.mustExec(2, "marmot", "UPDATE sequential_test SET value = 'updated' WHERE id = 1") + c.waitRows("marmot", kvRows("sequential_test"), []string{"1|updated"}, 1, 2, 3) + c.mustExec(3, "marmot", "DELETE FROM sequential_test WHERE id = 1") + c.waitRows("marmot", kvRows("sequential_test"), nil, 1, 2, 3) } -// TestBulkInsertReplication tests bulk insert replication +// TestBulkInsertReplication: a hundred single-row INSERTs reach every node. func TestBulkInsertReplication(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - - tableName := "bulk_insert_test" - allNodes := []int{1, 2, 3} - - t.Logf("Creating table %s on node 1...", tableName) - _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, value TEXT)", tableName)) - if err != nil { - t.Fatalf("Failed to create table: %v", err) - } - - if err := harness.WaitForTableExists(tableName, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("bulk_insert_ddl") - t.Fatalf("DDL replication failed: %v", err) - } - - // Insert 100 rows on node 1 - t.Logf("Inserting 100 rows on node 1...") - for i := 1; i <= 100; i++ { - _, err := harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (%d, 'bulk_%d')", tableName, i, i)) - if err != nil { - t.Fatalf("Failed to insert row %d: %v", i, err) - } - } - - // Verify all 100 appear on nodes 2 and 3 - for _, nodeID := range []int{2, 3} { - if err := waitForRowCountOnNode(t, harness, nodeID, tableName, 100, 30*time.Second); err != nil { - harness.dumpNodeLogs("bulk_insert_fail") - t.Fatalf("Bulk insert replication failed on node %d: %v", nodeID, err) - } - t.Logf("Node %d has 100 rows", nodeID) - } - - t.Logf("SUCCESS: Bulk insert replication verified") + c := startKV(t, "bulk_insert_test") + want := insertEach(c, 1, "bulk_insert_test", 1, 100, "bulk") + c.waitRows("marmot", kvRows("bulk_insert_test"), want, 1, 2, 3) } -// TestBulkUpdateReplication tests bulk update replication +// TestBulkUpdateReplication: fifty single-row UPDATEs through one node reach +// every node, each row with its own value. func TestBulkUpdateReplication(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - - tableName := "bulk_update_test" - allNodes := []int{1, 2, 3} - - t.Logf("Creating table %s on node 1...", tableName) - _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, value TEXT)", tableName)) - if err != nil { - t.Fatalf("Failed to create table: %v", err) - } - - if err := harness.WaitForTableExists(tableName, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("bulk_update_ddl") - t.Fatalf("DDL replication failed: %v", err) - } - - // Insert 50 rows - t.Logf("Inserting 50 rows on node 1...") - for i := 1; i <= 50; i++ { - _, err := harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (%d, 'initial_%d')", tableName, i, i)) - if err != nil { - t.Fatalf("Failed to insert row %d: %v", i, err) - } - } - - // Wait for insert replication - if err := harness.WaitForRowCount(tableName, allNodes, 50, 30*time.Second); err != nil { - harness.dumpNodeLogs("bulk_update_insert") - t.Fatalf("Insert replication failed: %v", err) - } - t.Logf("All nodes have 50 initial rows") - - // Update all 50 rows on node 2 - t.Logf("Updating all 50 rows on node 2...") + c := startKV(t, "bulk_update_test") + insertEach(c, 1, "bulk_update_test", 1, 50, "initial") + var want []string for i := 1; i <= 50; i++ { - _, err := harness.ExecNode(2, fmt.Sprintf( - "UPDATE %s SET value = 'updated_%d' WHERE id = %d", tableName, i, i)) - if err != nil { - t.Fatalf("Failed to update row %d: %v", i, err) - } - } - - // Wait for updates to propagate - time.Sleep(5 * time.Second) - - // Verify updates propagated correctly by sampling a few rows - t.Logf("Verifying updates propagated to nodes 1 and 3...") - for _, nodeID := range []int{1, 3} { - db, err := harness.ConnectToNode(nodeID) - if err != nil { - t.Fatalf("Failed to connect to node %d: %v", nodeID, err) - } - - // Sample check: verify first, middle, and last rows - for _, id := range []int{1, 25, 50} { - expectedValue := fmt.Sprintf("updated_%d", id) - if err := verifyRowValue(t, db, tableName, id, expectedValue); err != nil { - harness.dumpNodeLogs("bulk_update_fail") - t.Fatalf("Bulk update replication failed on node %d: %v", nodeID, err) - } - } - t.Logf("Node %d has updated values (sampled)", nodeID) + c.mustExec(2, "marmot", "UPDATE bulk_update_test SET value = ? WHERE id = ?", fmt.Sprintf("updated_%d", i), i) + want = append(want, fmt.Sprintf("%d|updated_%d", i, i)) } - - t.Logf("SUCCESS: Bulk update replication verified") + c.waitRows("marmot", kvRows("bulk_update_test"), want, 1, 2, 3) } -// TestBulkDeleteReplication tests bulk delete replication +// TestBulkDeleteReplication: fifty single-row DELETEs through one node empty +// the table on every node. func TestBulkDeleteReplication(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - - tableName := "bulk_delete_test" - allNodes := []int{1, 2, 3} - - t.Logf("Creating table %s on node 1...", tableName) - _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, value TEXT)", tableName)) - if err != nil { - t.Fatalf("Failed to create table: %v", err) - } - - if err := harness.WaitForTableExists(tableName, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("bulk_delete_ddl") - t.Fatalf("DDL replication failed: %v", err) - } - - // Insert 50 rows - t.Logf("Inserting 50 rows on node 1...") + c := startKV(t, "bulk_delete_test") + want := insertEach(c, 1, "bulk_delete_test", 1, 50, "row") + c.waitRows("marmot", kvRows("bulk_delete_test"), want, 1, 2, 3) for i := 1; i <= 50; i++ { - _, err := harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, value) VALUES (%d, 'row_%d')", tableName, i, i)) - if err != nil { - t.Fatalf("Failed to insert row %d: %v", i, err) - } + c.mustExec(2, "marmot", "DELETE FROM bulk_delete_test WHERE id = ?", i) } - - // Wait for insert replication - if err := harness.WaitForRowCount(tableName, allNodes, 50, 30*time.Second); err != nil { - harness.dumpNodeLogs("bulk_delete_insert") - t.Fatalf("Insert replication failed: %v", err) - } - t.Logf("All nodes have 50 initial rows") - - // Delete all 50 rows on node 2 - t.Logf("Deleting all 50 rows on node 2...") - for i := 1; i <= 50; i++ { - _, err := harness.ExecNode(2, fmt.Sprintf( - "DELETE FROM %s WHERE id = %d", tableName, i)) - if err != nil { - t.Fatalf("Failed to delete row %d: %v", i, err) - } - } - - // Verify all deletes propagate (count should be 0 on all nodes) - for _, nodeID := range []int{1, 3} { - if err := waitForRowCountOnNode(t, harness, nodeID, tableName, 0, 30*time.Second); err != nil { - harness.dumpNodeLogs("bulk_delete_fail") - t.Fatalf("Bulk delete replication failed on node %d: %v", nodeID, err) - } - t.Logf("Node %d has 0 rows after bulk delete", nodeID) - } - - t.Logf("SUCCESS: Bulk delete replication verified") + c.waitRows("marmot", kvRows("bulk_delete_test"), nil, 1, 2, 3) } diff --git a/test/gap_detection_test.go b/test/gap_detection_test.go index 58f3a092..3cbb33e4 100644 --- a/test/gap_detection_test.go +++ b/test/gap_detection_test.go @@ -2,111 +2,45 @@ package test import ( "fmt" + "strings" "testing" - "time" ) -// Lagging Node Catchup Integration Test -// -// This test validates that a node which falls far behind can catch up via -// anti-entropy. It simulates a realistic scenario where a node is offline -// for an extended period while the cluster continues processing transactions. -// -// Background: -// - Marmot uses anti-entropy for eventual consistency -// - When a node is down, it misses transactions -// - On restart, anti-entropy detects the lag and syncs missing data -// -// Test Scenario: -// 1. A node goes offline (simulated crash) -// 2. The cluster continues processing many transactions -// 3. When the node restarts, anti-entropy syncs the missing data -// -// Success Criteria: -// - Node 3 successfully catches up after restart -// - Final data consistency across all nodes - -// TestLaggingNodeCatchup validates that a lagging node catches up via -// anti-entropy. Row counts and waits are sized to stay well under the 60 s -// cluster test budget: every wait is a poll on a named deadline (WaitForRowCount -// / WaitForAlive), never a flat sleep, so the test finishes as soon as the -// cluster actually converges. +// TestLaggingNodeCatchup: a node killed while the cluster commits a hundred +// transactions catches up through anti-entropy after it restarts, ending with +// exactly the rows every other node has. func TestLaggingNodeCatchup(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - // Start cluster - t.Log("Starting 3-node cluster...") - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - harness.Phase("start") - - // Create test table - t.Logf("Creating test table...") - _, err := harness.ExecNode(1, "CREATE TABLE test_lagging (id INT PRIMARY KEY, value TEXT)") - if err != nil { - t.Fatalf("Failed to create table: %v", err) - } - - // Wait for table to replicate - allNodes := []int{1, 2, 3} - if err := harness.WaitForTableExists("test_lagging", allNodes, 10*time.Second); err != nil { - t.Fatalf("Table did not replicate: %v", err) - } - - const initialRows = 50 - t.Logf("Inserting %d initial rows...", initialRows) - for i := 1; i <= initialRows; i++ { - _, err := harness.ExecNode(1, "INSERT INTO test_lagging (id, value) VALUES (?, ?)", i, fmt.Sprintf("value_%d", i)) - if err != nil { - t.Fatalf("Failed to insert row %d: %v", i, err) + c := newCluster(t) + c.start() + c.mustExec(1, "marmot", "CREATE TABLE lagging (id INT PRIMARY KEY, v TEXT)") + c.phase("start") + + const rowsPerTxn = 5 + var want []string + next := 1 + commit := func(txns int) { + for i := 0; i < txns; i++ { + values := make([]string, rowsPerTxn) + for r := range values { + values[r] = fmt.Sprintf("(%d, 'v%d')", next, next) + want = append(want, fmt.Sprintf("%d|v%d", next, next)) + next++ + } + c.mustExec(1, "marmot", "INSERT INTO lagging (id, v) VALUES "+strings.Join(values, ", ")) } + c.progress() } - if err := harness.WaitForRowCount("test_lagging", allNodes, initialRows, 15*time.Second); err != nil { - t.Fatalf("initial rows did not replicate: %v", err) - } - harness.Phase("initial-load") + const q = "SELECT id, v FROM lagging ORDER BY id" + commit(10) + c.waitRows("marmot", q, want, 1, 2, 3) + c.phase("initial-load") - // Kill node 3 - t.Logf("Killing node 3...") - if err := harness.KillNode(3); err != nil { - t.Fatalf("Failed to kill node 3: %v", err) - } - harness.Phase("node3-killed") - - // Insert enough rows while node 3 is down that it is meaningfully behind. - const laggingRows = 500 - total := initialRows + laggingRows - t.Logf("Inserting %d rows while node 3 is down...", laggingRows) - for i := initialRows + 1; i <= total; i++ { - _, err := harness.ExecNode(1, "INSERT INTO test_lagging (id, value) VALUES (?, ?)", i, fmt.Sprintf("value_%d", i)) - if err != nil { - t.Fatalf("Failed to insert row %d: %v", i, err) - } - } - if err := harness.WaitForRowCount("test_lagging", []int{1, 2}, total, 20*time.Second); err != nil { - t.Fatalf("rows did not replicate to nodes 1&2: %v", err) - } - harness.Phase("lagging-load") - - // Restart node 3 - t.Logf("Restarting node 3...") - if err := harness.StartNode(3); err != nil { - t.Fatalf("Failed to restart node 3: %v", err) - } - if err := harness.WaitForAlive(3, 20*time.Second); err != nil { - t.Fatalf("Node 3 did not become alive: %v", err) - } - harness.Phase("node3-alive") - - if err := harness.WaitForRowCount("test_lagging", []int{3}, total, 20*time.Second); err != nil { - harness.dumpNodeLogs("lagging_node_fail") - t.Fatalf("node 3 did not catch up: %v", err) - } - harness.Phase("caught-up") + c.kill(3) + commit(100) + c.waitRows("marmot", q, want, 1, 2) + c.phase("lagging-load") - count3 := harness.getRowCount(3, "test_lagging") - count1 := harness.getRowCount(1, "test_lagging") - t.Logf("SUCCESS: Node 3 caught up! Node 3: %d rows, Node 1: %d rows", count3, count1) + c.restart(3) + c.waitRows("marmot", q, want, 1, 2, 3) + c.phase("caught-up") } diff --git a/test/graceful_restart_cluster_test.go b/test/graceful_restart_cluster_test.go index 30d05a3d..55b72f13 100644 --- a/test/graceful_restart_cluster_test.go +++ b/test/graceful_restart_cluster_test.go @@ -1,87 +1,31 @@ package test import ( - "encoding/json" "fmt" - "io" - "net/http" "testing" - "time" ) -// gracefulStopDeadline bounds a node's own shutdown after SIGTERM. -const gracefulStopDeadline = 30 * time.Second - -// memberStatus returns nodeID's status in observer's membership view. -func memberStatus(h *ClusterHarness, observer int, nodeID uint64) (string, error) { - url := fmt.Sprintf("http://localhost:%d/admin/cluster/members", h.Nodes[observer-1].GRPCPort) - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return "", err - } - req.Header.Set("X-Marmot-Secret", "test-secret") - resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - var wrapped struct { - Data []struct { - NodeID uint64 `json:"node_id"` - Status string `json:"status"` - } `json:"data"` - } - if err := json.Unmarshal(body, &wrapped); err != nil { - return "", fmt.Errorf("members on node %d: %w: %s", observer, err, body) - } - for _, m := range wrapped.Data { - if m.NodeID == nodeID { - return m.Status, nil - } - } - return "", fmt.Errorf("node %d absent from node %d's membership", nodeID, observer) -} - // TestGracefulStopThenRestartRejoins: a node stopped with SIGTERM announces -// LEAVING and its peers keep that record. Restarted, it must rejoin as ALIVE and -// serve writes, not read its own departure as a decommission and shut down. +// that it is leaving, and its peers keep that record. Restarted, it must +// rejoin as ALIVE in its peers' view and its own and serve writes, not read +// its own departure as a decommission and shut down again. func TestGracefulStopThenRestartRejoins(t *testing.T) { - h := NewClusterHarness(t) - defer h.Cleanup() - tables := map[string][]string{"marmot": {"t"}} - startWithTables(t, h, tables) - - if err := h.GracefulStopNode(3, gracefulStopDeadline); err != nil { - t.Fatal(err) - } - h.Phase("node-3-stopped-gracefully") - if err := h.StartNode(3); err != nil { - t.Fatal(err) - } - if err := h.WaitForAlive(3, restartDeadline); err != nil { - t.Fatal(err) - } - - deadline := time.Now().Add(restartDeadline) - for { - s1, err1 := memberStatus(h, 1, 3) - s3, err3 := memberStatus(h, 3, 3) - if err1 == nil && err3 == nil && s1 == "ALIVE" && s3 == "ALIVE" { - break - } - if time.Now().After(deadline) { - t.Fatalf("node 3 not ALIVE within %v of restart: node1 view=%q (%v), own view=%q (%v); log tail:\n%s", - restartDeadline, s1, err1, s3, err3, h.getNodeLogTail(3, 30)) - } - time.Sleep(convergencePoll) - } - h.Phase("node-3-alive") + c := newCluster(t) + c.start() + createAutoIncTables(c, 1, []target{{"marmot", "t"}}) + c.stop(3) + waitFor(t, "node 1 records node 3's departure", readyDeadline, func() (bool, string) { + statuses, err := c.statuses(1) + return err == nil && statuses[3] != "" && statuses[3] != "ALIVE", fmt.Sprint(statuses, err) + }) - l := newKeyLedger(h) - insertFromEveryNode(t, h, l, "after-restart") - waitKeysConverged(t, h, l, tables, convergenceDeadline) + c.restart(3) + l := newRowLedger() + for _, id := range c.all() { + v := fmt.Sprintf("after-restart-n%d", id) + res := c.mustExec(id, "marmot", "INSERT INTO t (v) VALUES (?)", v) + rowID, _ := res.LastInsertId() + l.inserted("w", ledgerKey("marmot", "t", rowID), v) + } + l.waitConverged(c, []target{{"marmot", "t"}}) } diff --git a/test/harness_test.go b/test/harness_test.go new file mode 100644 index 00000000..264300ed --- /dev/null +++ b/test/harness_test.go @@ -0,0 +1,995 @@ +package test + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand/v2" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/go-sql-driver/mysql" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + + marmotgrpc "github.com/maxpert/marmot/grpc" + "github.com/maxpert/marmot/protocol/mysqlcode" + "github.com/maxpert/marmot/test/stallwatch" +) + +const ( + // testBudget is the wall-clock budget of every cluster test: the harness + // fails a test whose cluster lives longer and stops its nodes. + testBudget = 10 * time.Second + // stallTimeout fails a cluster that wrote nothing, answered no client + // call and reported no progress for this long. + stallTimeout = 5 * time.Second + // pollInterval is how often waitFor re-checks its condition. + pollInterval = 20 * time.Millisecond + // readyDeadline bounds a node's wait from start to ready for writes. + readyDeadline = 6 * time.Second + // replicationDeadline bounds a wait for a write to reach every node. + replicationDeadline = 5 * time.Second + // reconnectDeadline bounds execAcrossReconnect's retries: grpc's default + // reconnect backoff starts at one second, with jitter. + reconnectDeadline = 3 * time.Second + // stopDeadline bounds a graceful stop (SIGTERM until exit). + stopDeadline = 5 * time.Second + // clientTimeout bounds every client call a test makes to a node. + clientTimeout = 3 * time.Second + // stallDumpWait bounds the wait for stalled nodes to write their + // goroutine dumps and exit. + stallDumpWait = 2 * time.Second + // logTailLines is how much of each node's log a failure report shows. + logTailLines = 30 + // clusterSecret is every test cluster's cluster_secret. + clusterSecret = "test-secret" +) + +// waitFor polls cond every pollInterval until it reports done, failing the +// test once deadline passes. what names the wait; cond's state describes what +// it last saw, and is what the failure prints. It is the only place a test +// sleeps. +func waitFor(t testing.TB, what string, deadline time.Duration, cond func() (done bool, state string)) { + t.Helper() + if state, ok := poll(deadline, cond); !ok { + t.Fatalf("%s: not reached within %s; last state: %s", what, deadline, state) + } +} + +// poll is waitFor without the test: it reports whether cond was done within +// deadline and the state it last described. +func poll(deadline time.Duration, cond func() (bool, string)) (string, bool) { + end := time.Now().Add(deadline) + for { + done, state := cond() + if done { + return state, true + } + if time.Now().After(end) { + return state, false + } + time.Sleep(pollInterval) + } +} + +const ( + // retryBackoffBase and retryBackoffCap bound retryBackoff's waits. + retryBackoffBase = 5 * time.Millisecond + retryBackoffCap = 250 * time.Millisecond +) + +// retryBackoff retries attempt the way a real client retries a refused +// write: after each failure it waits a random time up to an exponentially +// growing bound (full jitter), until attempt succeeds or deadline passes. +// Two clients refused by the same conflict so retry at different times, +// where retrying on one clock would make them collide again. It reports the +// state attempt last described and whether it succeeded. +func retryBackoff(deadline time.Duration, attempt func() (bool, string)) (string, bool) { + end := time.Now().Add(deadline) + bound := retryBackoffBase + for { + done, state := attempt() + if done { + return state, true + } + if time.Now().After(end) { + return state, false + } + time.Sleep(rand.N(bound) + time.Millisecond) + bound = min(2*bound, retryBackoffCap) + } +} + +// node is one Marmot process of a test cluster. +type node struct { + id int + grpcPort int + mysqlPort int + dir string + config string + log string + // bin is the binary the node runs; "" means the tree under test. + bin string + + mu sync.Mutex + cmd *exec.Cmd + exited chan struct{} + // waitErr is the process's exit error, set before exited closes. + waitErr error + conns map[string]*sql.DB +} + +// clusterConfig holds the per-test settings a node's config file is written +// from. Every field is written explicitly: the defaults below are small so a +// cluster starts, releases its claim votes and catches up within about a +// second. +type clusterConfig struct { + size int + gossipIntervalMS int + suspectTimeoutMS int + deadTimeoutMS int + antiEntropySeconds int + gcIntervalSeconds int + gcMinRetentionHours int + deltaSyncThresholdSecs int + heartbeatTimeoutSecs int + lockWaitTimeoutSecs int + shutdownGraceMS int + autoIncMergeMS int + autoIncBaseSyncMS int + // strictPrepareSync makes a PREPARE durable before it is ACKed. + strictPrepareSync bool +} + +func defaultClusterConfig() clusterConfig { + return clusterConfig{ + size: 3, + gossipIntervalMS: 100, + suspectTimeoutMS: 1500, + deadTimeoutMS: 3000, + antiEntropySeconds: 1, + gcIntervalSeconds: 30, + gcMinRetentionHours: 2, + deltaSyncThresholdSecs: 3600, + heartbeatTimeoutSecs: 10, + lockWaitTimeoutSecs: 5, + shutdownGraceMS: 1000, + autoIncMergeMS: 50, + autoIncBaseSyncMS: 1000, + } +} + +// cluster is one test's Marmot cluster: its own ports, its own directory +// under the run directory, and nodes that are always stopped when the test +// ends, however it ends. +type cluster struct { + t *testing.T + dir string + cfg clusterConfig + nodes []*node + + watchdog *stallwatch.Watchdog + budget *time.Timer + // lastPhase is when the previous phase ended (phase). + lastPhase time.Time +} + +// phase logs how long the test's phase that just ended took. +func (c *cluster) phase(name string) { + now := time.Now() + c.t.Logf("phase=%s elapsed=%s", name, now.Sub(c.lastPhase).Round(time.Millisecond)) + c.lastPhase = now + c.progress() +} + +// newCluster writes a cluster's node configs and registers its cleanup; it +// starts nothing. Options adjust the config before it is written. +func newCluster(t *testing.T, opts ...func(*clusterConfig)) *cluster { + t.Helper() + cfg := defaultClusterConfig() + for _, opt := range opts { + opt(&cfg) + } + dir := testDir(t) + c := &cluster{t: t, dir: dir, cfg: cfg, lastPhase: time.Now()} + ports := freePorts(t, 2*cfg.size) + for i := 0; i < cfg.size; i++ { + c.nodes = append(c.nodes, &node{ + id: i + 1, + grpcPort: ports[2*i], + mysqlPort: ports[2*i+1], + dir: filepath.Join(dir, fmt.Sprintf("node%d", i+1)), + conns: map[string]*sql.DB{}, + }) + } + for _, n := range c.nodes { + c.writeConfig(n) + } + c.watchdog = stallwatch.Start(stallTimeout, pollInterval*10, c.activity, c.reportStall) + c.budget = time.AfterFunc(testBudget, c.reportBudgetExceeded) + t.Cleanup(c.cleanup) + return c +} + +// testDir creates and returns t's own directory under runDir. +func testDir(t testing.TB) string { + t.Helper() + dir := filepath.Join(runDir, strings.ReplaceAll(t.Name(), "/", "_")) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + return dir +} + +// freshDir creates and returns a new, empty directory under t's own +// directory. +func freshDir(t testing.TB) string { + t.Helper() + dir, err := os.MkdirTemp(testDir(t), "dir-") + if err != nil { + t.Fatal(err) + } + return dir +} + +// freePorts reserves n distinct free TCP ports by listening on each until all +// are chosen, so no two nodes of one cluster are handed the same port. +func freePorts(t *testing.T, n int) []int { + t.Helper() + ports := make([]int, 0, n) + for i := 0; i < n; i++ { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("reserve a port: %v", err) + } + defer l.Close() + ports = append(ports, l.Addr().(*net.TCPAddr).Port) + } + return ports +} + +// seeds is node's seed list: every node but the first joins through node 1. +func (c *cluster) seeds(n *node) string { + if n.id == 1 { + return "" + } + return fmt.Sprintf("%q", fmt.Sprintf("127.0.0.1:%d", c.nodes[0].grpcPort)) +} + +// writeConfig writes node's config file from the cluster's config. +func (c *cluster) writeConfig(n *node) { + if err := os.MkdirAll(n.dir, 0o755); err != nil { + c.t.Fatal(err) + } + n.config = filepath.Join(n.dir, "config.toml") + n.log = filepath.Join(n.dir, "marmot.log") + cfg := c.cfg + body := fmt.Sprintf(`node_id = %d +data_dir = %q + +[transaction] +heartbeat_timeout_seconds = %d +lock_wait_timeout_seconds = %d + +[coordinator] +prepare_timeout_ms = 2000 +commit_timeout_ms = 2000 +abort_timeout_ms = 2000 + +[cluster] +grpc_bind_address = "127.0.0.1" +grpc_advertise_address = "127.0.0.1:%d" +grpc_port = %d +seed_nodes = [%s] +gossip_interval_ms = %d +suspect_timeout_ms = %d +dead_timeout_ms = %d +shutdown_grace_period_ms = %d +autoinc_merge_interval_ms = %d +autoinc_base_sync_interval_ms = %d +cluster_secret = %q + +[cluster.promotion] +check_interval_seconds = 1 +min_healthy_duration_sec = 0 + +[replication] +anti_entropy_interval_seconds = %d +gc_interval_seconds = %d +gc_min_retention_hours = %d +delta_sync_threshold_seconds = %d + +[metastore] +strict_prepare_sync = %t + +[mysql] +bind_address = "127.0.0.1" +port = %d + +[logging] +format = "console" + +[prometheus] +enabled = false +`, n.id, n.dir, cfg.heartbeatTimeoutSecs, cfg.lockWaitTimeoutSecs, n.grpcPort, n.grpcPort, c.seeds(n), + cfg.gossipIntervalMS, cfg.suspectTimeoutMS, cfg.deadTimeoutMS, cfg.shutdownGraceMS, + cfg.autoIncMergeMS, cfg.autoIncBaseSyncMS, clusterSecret, + cfg.antiEntropySeconds, cfg.gcIntervalSeconds, cfg.gcMinRetentionHours, cfg.deltaSyncThresholdSecs, cfg.strictPrepareSync, n.mysqlPort) + if err := os.WriteFile(n.config, []byte(body), 0o644); err != nil { + c.t.Fatal(err) + } +} + +// node returns node id (1-based). +func (c *cluster) node(id int) *node { return c.nodes[id-1] } + +// startNode launches node id's process without waiting for it; its output +// is appended to its log, one header per incarnation. +func (c *cluster) startNode(id int) { + c.t.Helper() + n := c.node(id) + n.mu.Lock() + defer n.mu.Unlock() + if n.cmd != nil { + c.t.Fatalf("node %d is already running", id) + } + logFile, err := os.OpenFile(n.log, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + c.t.Fatal(err) + } + defer logFile.Close() + fmt.Fprintf(logFile, "=== node %d start %s ===\n", id, time.Now().Format(time.RFC3339Nano)) + bin := marmotBin + if n.bin != "" { + bin = n.bin + } + cmd := exec.Command(bin, "--config", n.config) + cmd.Stdout, cmd.Stderr = logFile, logFile + if err := cmd.Start(); err != nil { + c.t.Fatalf("start node %d: %v", id, err) + } + exited := make(chan struct{}) + go func() { + err := cmd.Wait() + n.mu.Lock() + n.waitErr = err + n.mu.Unlock() + close(exited) + }() + n.cmd, n.exited = cmd, exited +} + +// kill SIGKILLs node id and waits for it to exit: a crash, with nothing +// flushed or announced. +func (c *cluster) kill(id int) { + c.t.Helper() + cmd, exited := c.detach(id) + if err := cmd.Process.Signal(syscall.SIGKILL); err != nil { + c.t.Fatalf("SIGKILL node %d: %v", id, err) + } + <-exited +} + +// stop SIGTERMs node id and waits for its own shutdown (the path that +// announces LEAVING to its peers), failing the test unless it exits cleanly +// within stopDeadline. +func (c *cluster) stop(id int) { + c.t.Helper() + cmd, exited := c.detach(id) + if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { + c.t.Fatalf("SIGTERM node %d: %v", id, err) + } + select { + case <-exited: + case <-time.After(stopDeadline): + _ = cmd.Process.Kill() + <-exited + c.t.Fatalf("node %d did not exit within %s of SIGTERM; log tail:\n%s", id, stopDeadline, c.logTail(id)) + } + n := c.node(id) + n.mu.Lock() + err := n.waitErr + n.mu.Unlock() + if err != nil { + c.t.Fatalf("node %d exited with an error after SIGTERM: %v; log tail:\n%s", id, err, c.logTail(id)) + } +} + +// detach marks node id stopped and closes its client connections, returning +// its process for the caller to end. +func (c *cluster) detach(id int) (*exec.Cmd, chan struct{}) { + c.t.Helper() + n := c.node(id) + n.mu.Lock() + defer n.mu.Unlock() + if n.cmd == nil { + c.t.Fatalf("node %d is not running", id) + } + for name, conn := range n.conns { + conn.Close() + delete(n.conns, name) + } + cmd, exited := n.cmd, n.exited + n.cmd = nil + return cmd, exited +} + +// killAll SIGKILLs every running node. Unlike kill it never fails the test, +// so the watchdogs' goroutines and cleanup can call it. +func (c *cluster) killAll() { + for _, n := range c.nodes { + n.mu.Lock() + cmd, exited := n.cmd, n.exited + n.cmd = nil + for name, conn := range n.conns { + conn.Close() + delete(n.conns, name) + } + n.mu.Unlock() + if cmd != nil { + _ = cmd.Process.Signal(syscall.SIGKILL) + <-exited + } + } +} + +// cleanup stops every node; it keeps the cluster's directory, and prints +// each node's log tail, when the test failed. +func (c *cluster) cleanup() { + c.budget.Stop() + c.watchdog.Stop() + if c.t.Failed() { + for _, n := range c.nodes { + c.t.Logf("=== node %d: last %d log lines ===\n%s", n.id, logTailLines, c.logTail(n.id)) + } + } + c.killAll() + if c.t.Failed() { + c.t.Logf("cluster directory kept: %s", c.dir) + return + } + if err := os.RemoveAll(c.dir); err != nil { + c.t.Logf("remove %s: %v", c.dir, err) + } +} + +// logTail is the last logTailLines lines of node id's log. +func (c *cluster) logTail(id int) string { + data, err := os.ReadFile(c.node(id).log) + if err != nil { + return err.Error() + } + lines := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + return strings.Join(lines[max(len(lines)-logTailLines, 0):], "\n") +} + +// clientCalls counts client calls to any node that returned, with or without +// an error: a cluster answering its clients is not stalled. +var clientCalls atomic.Int64 + +// activity digests what the stall watchdog counts as progress: every file +// write under the cluster's directory and every returned client call. +func (c *cluster) activity() int64 { + total := clientCalls.Load() + _ = filepath.WalkDir(c.dir, func(_ string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + if info, err := d.Info(); err == nil { + total += info.Size() + info.ModTime().UnixNano() + } + return nil + }) + return total +} + +// progress tells the stall watchdog the test saw the cluster make progress. +func (c *cluster) progress() { c.watchdog.Progress() } + +// reportStall fails a cluster that made no progress for stallTimeout. Each +// running node gets SIGQUIT, so the Go runtime writes every goroutine's stack +// into its log before it exits; the nodes then stop, so the test's own calls +// fail at once. +func (c *cluster) reportStall(idle time.Duration) { + c.t.Errorf("cluster stalled: nothing written, no client call returned and no progress for %s; goroutine dumps are in each node's log under %s", + idle.Round(time.Millisecond), c.dir) + var dumping []chan struct{} + for _, n := range c.nodes { + n.mu.Lock() + if n.cmd != nil && n.cmd.Process.Signal(syscall.SIGQUIT) == nil { + dumping = append(dumping, n.exited) + } + n.mu.Unlock() + } + dumpDeadline := time.After(stallDumpWait) + for _, exited := range dumping { + select { + case <-exited: + case <-dumpDeadline: + } + } + c.killAll() +} + +// reportBudgetExceeded fails a test whose cluster outlived testBudget and +// stops its nodes, so the test's own calls fail at once. +func (c *cluster) reportBudgetExceeded() { + c.t.Errorf("cluster test exceeded its %s budget", testBudget) + c.killAll() +} + +// dsn is the MySQL DSN for database on node id; every call through it is +// bounded by clientTimeout. +func (c *cluster) dsn(id int, database string) string { + return fmt.Sprintf("root:@tcp(127.0.0.1:%d)/%s?timeout=%s&readTimeout=%s&writeTimeout=%s", + c.node(id).mysqlPort, database, clientTimeout, clientTimeout, clientTimeout) +} + +// db is the connection pool for database on node id, opened on first use and +// closed when the node stops. +func (c *cluster) db(id int, database string) *sql.DB { + c.t.Helper() + n := c.node(id) + n.mu.Lock() + defer n.mu.Unlock() + if conn, ok := n.conns[database]; ok { + return conn + } + conn, err := sql.Open("mysql", c.dsn(id, database)) + if err != nil { + c.t.Fatalf("open %s on node %d: %v", database, id, err) + } + n.conns[database] = conn + return conn +} + +// exec runs q on database through node id. +func (c *cluster) exec(id int, database, q string, args ...any) (sql.Result, error) { + defer clientCalls.Add(1) + return c.db(id, database).Exec(q, args...) +} + +// execAcrossReconnect runs q through node id like mustExec, but retries while +// it fails with 1105 for up to reconnectDeadline. A coordinator's gRPC +// connection to a peer that just restarted redials only after grpc's +// reconnect backoff, and until then a PREPARE to that peer fails at once, so +// with another node down the write cannot reach a quorum. It reports how many +// attempts failed. +func (c *cluster) execAcrossReconnect(id int, database, q string, args ...any) int { + c.t.Helper() + failed := 0 + waitFor(c.t, fmt.Sprintf("node %d accepts %s", id, q), reconnectDeadline, func() (bool, string) { + _, err := c.exec(id, database, q, args...) + if err == nil { + return true, "" + } + if mysqlCode(err) != mysqlcode.ErrCodeUnknown { + c.t.Fatalf("node %d: %s: %v", id, q, err) + } + failed++ + return false, err.Error() + }) + return failed +} + +// ping checks that node id answers a MySQL query. +func (c *cluster) ping(id int) error { + defer clientCalls.Add(1) + var one int + return c.db(id, "marmot").QueryRow("SELECT 1").Scan(&one) +} + +// waitMySQL waits until node id answers MySQL queries. +func (c *cluster) waitMySQL(id int) { + c.t.Helper() + waitFor(c.t, fmt.Sprintf("node %d answers MySQL", id), readyDeadline, func() (bool, string) { + err := c.ping(id) + return err == nil, fmt.Sprint(err) + }) +} + +// mustExec is exec that fails the test on an error. +func (c *cluster) mustExec(id int, database, q string, args ...any) sql.Result { + c.t.Helper() + res, err := c.exec(id, database, q, args...) + if err != nil { + c.t.Fatalf("node %d: %s: %v", id, q, err) + } + return res +} + +// rows runs query on database through node id and returns each row as its +// columns joined by "|" (NULL as "NULL"), in the order the query gives. +func (c *cluster) rows(id int, database, query string, args ...any) ([]string, error) { + defer clientCalls.Add(1) + rs, err := c.db(id, database).Query(query, args...) + if err != nil { + return nil, err + } + defer rs.Close() + cols, err := rs.Columns() + if err != nil { + return nil, err + } + var out []string + vals := make([]sql.NullString, len(cols)) + ptrs := make([]any, len(cols)) + for i := range vals { + ptrs[i] = &vals[i] + } + for rs.Next() { + if err := rs.Scan(ptrs...); err != nil { + return nil, err + } + fields := make([]string, len(vals)) + for i, v := range vals { + fields[i] = "NULL" + if v.Valid { + fields[i] = v.String + } + } + out = append(out, strings.Join(fields, "|")) + } + return out, rs.Err() +} + +// mustRows is rows that fails the test on an error. +func (c *cluster) mustRows(id int, database, query string, args ...any) []string { + c.t.Helper() + out, err := c.rows(id, database, query, args...) + if err != nil { + c.t.Fatalf("node %d: %s: %v", id, query, err) + } + return out +} + +// waitRows waits until query (which must order its rows) returns exactly +// want on database through every node in ids. +func (c *cluster) waitRows(database, query string, want []string, ids ...int) { + c.t.Helper() + wantText := strings.Join(want, ",") + waitFor(c.t, fmt.Sprintf("nodes %v: %s = %d rows", ids, query, len(want)), replicationDeadline, func() (bool, string) { + var state []string + for _, id := range ids { + got, err := c.rows(id, database, query) + if err != nil { + return false, fmt.Sprintf("node %d: %v", id, err) + } + if gotText := strings.Join(got, ","); gotText != wantText { + return false, fmt.Sprintf("node %d has %d rows: %s", id, len(got), abbreviate(gotText)) + } + state = append(state, fmt.Sprintf("node %d ok", id)) + } + return true, strings.Join(state, "; ") + }) +} + +// waitSameRows waits until query (which must order its rows) returns the +// same rows on database through every node in ids, and returns them. +func (c *cluster) waitSameRows(database, query string, ids ...int) []string { + c.t.Helper() + var agreed []string + waitFor(c.t, fmt.Sprintf("nodes %v agree on %s", ids, query), replicationDeadline, func() (bool, string) { + var first string + var state []string + for i, id := range ids { + got, err := c.rows(id, database, query) + if err != nil { + return false, fmt.Sprintf("node %d: %v", id, err) + } + text := strings.Join(got, ",") + state = append(state, fmt.Sprintf("node %d: %d rows %s", id, len(got), abbreviate(text))) + if i == 0 { + first, agreed = text, got + } else if text != first { + return false, strings.Join(state, "; ") + } + } + return true, "" + }) + return agreed +} + +// hasTable reports whether database on node id lists table. +func (c *cluster) hasTable(id int, database, table string) (bool, error) { + tables, err := c.rows(id, database, "SHOW TABLES") + if err != nil { + return false, err + } + return slices.Contains(tables, table), nil +} + +// waitTable waits until database on every node in ids lists table. +func (c *cluster) waitTable(database, table string, ids ...int) { + c.t.Helper() + waitFor(c.t, fmt.Sprintf("table %s.%s on nodes %v", database, table, ids), replicationDeadline, func() (bool, string) { + for _, id := range ids { + if ok, err := c.hasTable(id, database, table); !ok { + return false, fmt.Sprintf("node %d: absent (err=%v)", id, err) + } + } + return true, "" + }) +} + +// waitDatabase waits until SHOW DATABASES on every node in ids lists +// database (present) or does not (!present). +func (c *cluster) waitDatabase(database string, present bool, ids ...int) { + c.t.Helper() + waitFor(c.t, fmt.Sprintf("database %s present=%v on nodes %v", database, present, ids), replicationDeadline, func() (bool, string) { + for _, id := range ids { + if ok, err := c.hasDatabase(id, database); err != nil || ok != present { + return false, fmt.Sprintf("node %d: present=%v (err=%v)", id, ok, err) + } + } + return true, "" + }) +} + +// hasDatabase reports whether SHOW DATABASES on node id lists database. +func (c *cluster) hasDatabase(id int, database string) (bool, error) { + databases, err := c.rows(id, "marmot", "SHOW DATABASES") + return slices.Contains(databases, database), err +} + +// createDatabase creates database through node id and waits until every +// running node has it. +func (c *cluster) createDatabase(id int, database string) { + c.t.Helper() + c.mustExec(id, "marmot", "CREATE DATABASE "+database) + c.waitDatabase(database, true, c.running()...) +} + +// createTable runs ddl, which creates table in database, through node id and +// waits until every running node has the table. +func (c *cluster) createTable(id int, database, table, ddl string) { + c.t.Helper() + c.mustExec(id, database, ddl) + c.waitTable(database, table, c.running()...) +} + +// idValueRows is ids from..to of a (id, value) table whose value is tag_id, +// as a VALUES list and as the rows rows() returns for them. +func idValueRows(from, to int, tag string) (values string, want []string) { + var tuples []string + for i := from; i <= to; i++ { + tuples = append(tuples, fmt.Sprintf("(%d, '%s_%d')", i, tag, i)) + want = append(want, fmt.Sprintf("%d|%s_%d", i, tag, i)) + } + return strings.Join(tuples, ", "), want +} + +// mysqlCode is err's MySQL error number, or 0 when err is not a MySQL error. +func mysqlCode(err error) uint16 { + var mysqlErr *mysql.MySQLError + if errors.As(err, &mysqlErr) { + return mysqlErr.Number + } + return 0 +} + +// abbreviate shortens s for a failure message. +func abbreviate(s string) string { + const limit = 300 + if len(s) <= limit { + return s + } + return s[:limit] + "..." +} + +// member is one entry of a node's membership, as its admin API reports it. +type member struct { + NodeID uint64 `json:"node_id"` + Status string `json:"status"` +} + +// admin sends method path to node id's admin API, waiting at most timeout, +// and decodes the response's data field into out. +func (c *cluster) admin(id int, method, path string, timeout time.Duration, out any) error { + defer clientCalls.Add(1) + req, err := http.NewRequest(method, fmt.Sprintf("http://127.0.0.1:%d/admin%s", c.node(id).grpcPort, path), nil) + if err != nil { + return err + } + req.Header.Set("X-Marmot-Secret", clusterSecret) + resp, err := (&http.Client{Timeout: timeout}).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("%s %s on node %d: status %d: %s", method, path, id, resp.StatusCode, body) + } + return json.NewDecoder(resp.Body).Decode(&struct { + Data any `json:"data"` + }{Data: out}) +} + +// statuses is node id's view of every member's status, keyed by node id. +func (c *cluster) statuses(id int) (map[uint64]string, error) { + var members []member + if err := c.admin(id, http.MethodGet, "/cluster/members", clientTimeout, &members); err != nil { + return nil, err + } + out := make(map[uint64]string, len(members)) + for _, m := range members { + out[m.NodeID] = m.Status + } + return out, nil +} + +// autoIncState asks node id over gRPC for its AUTO_INCREMENT claim bases and +// whether its claim votes are held. GetAutoIncBases predates the admin votes +// endpoint, so this also answers for a node on an older release. +func (c *cluster) autoIncState(id int) (*marmotgrpc.AutoIncBasesResponse, error) { + defer clientCalls.Add(1) + conn, err := grpc.NewClient(fmt.Sprintf("127.0.0.1:%d", c.node(id).grpcPort), + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, err + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), clientTimeout) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, marmotgrpc.ClusterSecretHeader, clusterSecret) + return marmotgrpc.NewMarmotServiceClient(conn).GetAutoIncBases(ctx, &marmotgrpc.AutoIncBasesRequest{}) +} + +// votesHeld reports whether node id's AUTO_INCREMENT claim votes are held. +func (c *cluster) votesHeld(id int) (bool, error) { + state, err := c.autoIncState(id) + if err != nil { + return false, err + } + return state.GetVotesHeld(), nil +} + +// autoIncBase is node id's AUTO_INCREMENT claim base for database.table (the +// largest of its committed, seed and merged floors); found is false while +// the node has no claim row for the table. +func (c *cluster) autoIncBase(id int, database, table string) (base uint64, found bool, err error) { + state, err := c.autoIncState(id) + if err != nil { + return 0, false, err + } + for _, b := range state.GetBases() { + if b.GetDatabase() == database && b.GetTable() == table { + return b.GetBase(), true, nil + } + } + return 0, false, nil +} + +// waitAutoIncBaseAtLeast waits until every node in ids holds a claim base of +// at least min for database.table. +func (c *cluster) waitAutoIncBaseAtLeast(database, table string, min uint64, ids ...int) { + c.t.Helper() + waitFor(c.t, fmt.Sprintf("claim base of %s.%s >= %d on nodes %v", database, table, min, ids), replicationDeadline, func() (bool, string) { + for _, id := range ids { + base, found, err := c.autoIncBase(id, database, table) + if err != nil || !found || base < min { + return false, fmt.Sprintf("node %d: base %d found %v err %v", id, base, found, err) + } + } + return true, "" + }) +} + +// readiness describes why node id is not ready for writes yet, or returns "" +// once it is: it sees itself and every node in running ALIVE, and its claim +// votes are not held (a node that has not merged claim bases from enough +// members declines every claim, so a narrow AUTO_INCREMENT insert through it +// returns 1205). +func (c *cluster) readiness(id int, running []int) string { + // The MySQL listener starts last, once the node finished starting: only + // then is its own status the one it keeps (a restarted node restores its + // membership before it marks itself JOINING). + if err := c.ping(id); err != nil { + return fmt.Sprintf("node %d MySQL: %v", id, err) + } + statuses, err := c.statuses(id) + if err != nil { + return fmt.Sprintf("node %d: %v", id, err) + } + for _, peer := range running { + if s := statuses[uint64(peer)]; s != "ALIVE" { + return fmt.Sprintf("node %d sees node %d as %q (all: %v)", id, peer, s, statuses) + } + } + held, err := c.votesHeld(id) + if err != nil { + return fmt.Sprintf("node %d votes: %v", id, err) + } + if held { + return fmt.Sprintf("node %d claim votes held", id) + } + return "" +} + +// running lists the ids of the nodes whose process is running. +func (c *cluster) running() []int { + var ids []int + for _, n := range c.nodes { + n.mu.Lock() + if n.cmd != nil { + ids = append(ids, n.id) + } + n.mu.Unlock() + } + return ids +} + +// waitReady waits until every running node is ready for writes (readiness). +func (c *cluster) waitReady() { + c.t.Helper() + running := c.running() + waitFor(c.t, fmt.Sprintf("nodes %v ready", running), readyDeadline, func() (bool, string) { + for _, id := range running { + if why := c.readiness(id, running); why != "" { + return false, why + } + } + return true, "" + }) +} + +// start starts every node and waits until the cluster is ready. Node 1, every +// other node's seed, starts first: a joiner that cannot reach its seed at boot +// stays outside the cluster. +func (c *cluster) start() { + c.t.Helper() + c.startNodes(c.all()...) +} + +// all lists every node id. +func (c *cluster) all() []int { + ids := make([]int, len(c.nodes)) + for i := range ids { + ids[i] = i + 1 + } + return ids +} + +// startNodes starts nodes ids, which begin with the seed node 1, as start +// does; the other nodes stay unstarted, unknown to the cluster. +func (c *cluster) startNodes(ids ...int) { + c.t.Helper() + if ids[0] != 1 { + c.t.Fatalf("startNodes(%v): the seed node 1 starts first", ids) + } + c.startNode(1) + waitFor(c.t, "seed node 1 ALIVE", readyDeadline, func() (bool, string) { + statuses, err := c.statuses(1) + if err != nil { + return false, err.Error() + } + return statuses[1] == "ALIVE", fmt.Sprint(statuses) + }) + for _, id := range ids[1:] { + c.startNode(id) + } + c.waitReady() +} + +// restart starts node id again and waits until the cluster is ready. +func (c *cluster) restart(id int) { + c.t.Helper() + c.startNode(id) + c.waitReady() +} diff --git a/test/information_schema_cluster_test.go b/test/information_schema_cluster_test.go index 23e6c81e..9fbbb0f2 100644 --- a/test/information_schema_cluster_test.go +++ b/test/information_schema_cluster_test.go @@ -1,7 +1,7 @@ package test // INFORMATION_SCHEMA over the real MySQL wire protocol, text and prepared, -// against a 3-node ClusterHarness. Every query carries its own timeout: the +// against a 3-node cluster. Every query carries its own timeout: the // failure these tests exist for is a query that never answers. import ( @@ -11,7 +11,6 @@ import ( "reflect" "strconv" "testing" - "time" ) // isAnswer is one INFORMATION_SCHEMA answer, normalised so the text @@ -21,13 +20,13 @@ type isAnswer struct { rows [][]string } -// queryAnswer runs one query under clusterQueryTimeout. With args, the driver +// queryAnswer runs one query under clientTimeout. With args, the driver // prepares it on the server (COM_STMT_PREPARE/EXECUTE); without, it is a text // query. func queryAnswer(conn *sql.DB, query string, args ...interface{}) (isAnswer, error) { - ctx, cancel := context.WithTimeout(context.Background(), clusterQueryTimeout) + ctx, cancel := context.WithTimeout(context.Background(), clientTimeout) defer cancel() - defer noteClientCall() + defer clientCalls.Add(1) rows, err := conn.QueryContext(ctx, query, args...) if err != nil { return isAnswer{}, err @@ -80,28 +79,17 @@ func queryAnswer(conn *sql.DB, query string, args ...interface{}) (isAnswer, err // times out; a prepared statement answers a result set only for SELECT - the // prepared answers come back with no columns. func TestInformationSchemaAnswersOnEveryPath(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - if err := harness.StartCluster(); err != nil { - t.Fatalf("StartCluster: %v", err) - } - - if _, err := execTimed(openTimedNodeDatabase(t, harness, 1, "marmot"), "CREATE DATABASE isdb"); err != nil { - t.Fatalf("CREATE DATABASE: %v", err) - } - waitForDatabase(t, harness, "isdb", 30*time.Second) - isdb := openTimedNodeDatabase(t, harness, 1, "isdb") + c := newCluster(t) + c.start() + c.createDatabase(1, "isdb") for _, ddl := range []string{ "CREATE TABLE users (id INT AUTO_INCREMENT PRIMARY KEY, name VARCHAR(64) NOT NULL, email VARCHAR(128))", "CREATE INDEX idx_users_email ON users (email)", "CREATE TABLE posts (id BIGINT PRIMARY KEY, body TEXT)", } { - if _, err := execTimed(isdb, ddl); err != nil { - t.Fatalf("%s: %v", ddl, err) - } + c.mustExec(1, "isdb", ddl) } - waitForTableIn(t, harness, "isdb", "posts", 30*time.Second) - + c.waitTable("isdb", "posts", 1, 2, 3) cases := []struct { name string database string @@ -148,10 +136,9 @@ func TestInformationSchemaAnswersOnEveryPath(t *testing.T) { // Prepared first: before the fix the first prepared query wedged the // node, so every later query on it would time out. for _, tc := range cases { - conn := openTimedNodeDatabase(t, harness, nodeID, tc.database) + conn := c.db(nodeID, tc.database) prepared, err := queryAnswer(conn, tc.prepared, tc.args...) if err != nil { - harness.dumpNodeLogs("is_prepared") t.Fatalf("node %d %s, prepared: %v", nodeID, tc.name, err) } text, err := queryAnswer(conn, tc.text) @@ -174,8 +161,7 @@ func TestInformationSchemaAnswersOnEveryPath(t *testing.T) { // database they read, completes. An index on posts leaves every answer // above unchanged for the next node. ddl := fmt.Sprintf("CREATE INDEX after_is_%d ON posts (body)", nodeID) - if _, err := execTimed(openTimedNodeDatabase(t, harness, nodeID, "isdb"), ddl); err != nil { - harness.dumpNodeLogs("is_ddl") + if _, err := c.exec(nodeID, "isdb", ddl); err != nil { t.Fatalf("node %d: %s after INFORMATION_SCHEMA queries: %v", nodeID, ddl, err) } } diff --git a/test/main_test.go b/test/main_test.go new file mode 100644 index 00000000..fe15f7d5 --- /dev/null +++ b/test/main_test.go @@ -0,0 +1,117 @@ +package test + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/maxpert/marmot/protocol" +) + +// testDataRoot holds every run's clusters and binaries: Marmot's runtime +// files live under /tmp/marmot. +const testDataRoot = "/tmp/marmot/test" + +// buildTags are the tags every Marmot binary is built with. +const buildTags = "sqlite_preupdate_hook sqlite_fts5 sqlite_json sqlite_math_functions sqlite_foreign_keys sqlite_stat4 sqlite_vacuum_incr" + +// preLogPullCommit is the last commit before nodes pulled each other's commit +// logs: the release a rolling upgrade to the log-pull protocol starts from. +const preLogPullCommit = "7a66596" + +var ( + // runDir is this package run's directory: one subdirectory per test. + runDir string + // marmotBin is the binary built from the tree under test. + marmotBin string + // preLogPullBin is the preLogPullCommit binary. + preLogPullBin string +) + +// TestMain builds the binaries every cluster test runs, once per package +// run, so no test pays for a build. +func TestMain(m *testing.M) { + if os.Getenv("MARMOT_RUN_CLUSTER_INTEGRATION_TESTS") != "1" { + fmt.Fprintln(os.Stderr, "skipping cluster integration tests; set MARMOT_RUN_CLUSTER_INTEGRATION_TESTS=1 to run") + os.Exit(0) + } + if err := protocol.InitializePipeline(10000, nil); err != nil { + fmt.Fprintln(os.Stderr, "initialize query pipeline:", err) + os.Exit(1) + } + // A re-exec'd kill -9 child (TestClaimRangeKillNineChild) runs one + // in-process test in its parent's directories: it builds nothing and owns + // no run directory. + if os.Getenv(killNineDataRootEnv) != "" { + os.Exit(m.Run()) + } + if err := setUp(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + if code == 0 { + _ = os.RemoveAll(runDir) + } + os.Exit(code) +} + +// setUp creates runDir and builds marmotBin and preLogPullBin. +func setUp() error { + if err := os.MkdirAll(testDataRoot, 0o755); err != nil { + return err + } + dir, err := os.MkdirTemp(testDataRoot, time.Now().Format("run-20060102-150405-")) + if err != nil { + return err + } + runDir = dir + marmotBin = filepath.Join(runDir, "marmot") + if err := build(repoRoot(), marmotBin); err != nil { + return err + } + preLogPullBin, err = buildPreLogPull() + return err +} + +// repoRoot is the module root, taken from this file's own path so the +// harness builds the tree it is compiled from. +func repoRoot() string { + _, file, _, _ := runtime.Caller(0) + return filepath.Dir(filepath.Dir(file)) +} + +// build builds the Marmot binary from src into bin. +func build(src, bin string) error { + cmd := exec.Command("go", "build", "-tags", buildTags, "-o", bin, ".") + cmd.Dir = src + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("build %s: %v\n%s", src, err, out) + } + return nil +} + +// buildPreLogPull returns the preLogPullCommit binary, building it from a +// read-only `git archive` of that commit unless a previous run left it under +// testDataRoot. +func buildPreLogPull() (string, error) { + dir := filepath.Join(testDataRoot, "marmot_pref2_"+preLogPullCommit) + bin := filepath.Join(dir, "marmot") + if _, err := os.Stat(bin); err == nil { + return bin, nil + } + src := filepath.Join(dir, "src") + if err := os.MkdirAll(src, 0o755); err != nil { + return "", err + } + archive := exec.Command("sh", "-c", fmt.Sprintf("git archive %s | tar -x -C %s", preLogPullCommit, src)) + archive.Dir = repoRoot() + if out, err := archive.CombinedOutput(); err != nil { + return "", fmt.Errorf("git archive %s: %v\n%s", preLogPullCommit, err, out) + } + return bin, build(src, bin) +} diff --git a/test/node_outage_convergence_cluster_test.go b/test/node_outage_convergence_cluster_test.go index a334e9e4..7b3504f2 100644 --- a/test/node_outage_convergence_cluster_test.go +++ b/test/node_outage_convergence_cluster_test.go @@ -6,876 +6,427 @@ package test // catch-up, convergence with GC actually running, and transactions a killed // node had prepared but never committed. // -// These check row presence (missing=0) and duplicate ids only. Row values -// converging across nodes is checked separately -// (row_version_convergence_cluster_test.go). -// // A transaction a node holds locally PENDING (its own coordinator killed // before its local commit, or a participant COMMIT that failed against a // pinned session) must not block that database's log pull until the -// stale-transaction GC ends it, or an ACKed row stays missing well past -// convergenceDeadline: the puller resolves such a record in the same round -// (grpc.LogPuller's resolveLocalPending). grpc/log_puller_test.go pins each -// resolution deterministically, these tests the end-to-end outcome. +// stale-transaction GC ends it: the puller resolves such a record in the same +// round (grpc.LogPuller's resolveLocalPending). grpc/log_puller_test.go pins +// each resolution deterministically; these tests the end-to-end outcome. import ( - "context" - "database/sql" - "encoding/json" "errors" "fmt" - "io" "net/http" - "os" - "sort" "strings" - "sync" "testing" - "time" - "github.com/go-sql-driver/mysql" "github.com/maxpert/marmot/protocol/mysqlcode" ) -// keyLedger records every ACKed row key (database.table/id) across possibly -// several databases, and each writer's own keys so a writer can UPDATE one of -// its own rows. Unlike rowLedger (f2_convergence_cluster_test.go, one -// database) it does not compare values across nodes - see the file comment. -type keyLedger struct { - harness *ClusterHarness - mu sync.Mutex - keys map[string]bool - own map[string][]string - dups []string -} - -func newKeyLedger(h *ClusterHarness) *keyLedger { - return &keyLedger{harness: h, keys: map[string]bool{}, own: map[string][]string{}} -} - -func ledgerKey(database, table string, id int64) string { - return fmt.Sprintf("%s.%s/%d", database, table, id) -} - -// acked records key as ACKed by writer. A duplicate key (the same id -// generated twice) is recorded, never silently overwritten. -func (l *keyLedger) acked(writer, key string) { - l.harness.Progress() - l.mu.Lock() - defer l.mu.Unlock() - if l.keys[key] { - l.dups = append(l.dups, key) - } - l.keys[key] = true - l.own[writer] = append(l.own[writer], key) -} - -// pick returns one of writer's own ACKed keys, deterministically spread by i. -func (l *keyLedger) pick(writer string, i int) (string, bool) { - l.mu.Lock() - defer l.mu.Unlock() - keys := l.own[writer] - if len(keys) == 0 { - return "", false - } - return keys[(i*7919)%len(keys)], true -} - -func (l *keyLedger) size() int { - l.mu.Lock() - defer l.mu.Unlock() - return len(l.keys) -} - -// rowIDsOn reads every row id (not value: see file comment) of every -// database.table on node. -func rowIDsOn(t *testing.T, h *ClusterHarness, node int, tables map[string][]string) (map[string]bool, error) { - got := map[string]bool{} - for database, ts := range tables { - conn := openTimedNodeDatabase(t, h, node, database) - for _, table := range ts { - a, err := queryAnswer(conn, "SELECT id FROM "+table) - if err != nil { - return nil, fmt.Errorf("node %d %s.%s: %v", node, database, table, err) - } - for _, r := range a.rows { - got[database+"."+table+"/"+r[0]] = true - } - } - } - return got, nil -} - -// waitKeysConverged polls until every node holds every ACKed row's id, within -// deadline. -func waitKeysConverged(t *testing.T, h *ClusterHarness, l *keyLedger, tables map[string][]string, deadline time.Duration) { - t.Helper() - start := time.Now() - for { - h.Progress() - var missing []string - for node := 1; node <= numNodes; node++ { - got, err := rowIDsOn(t, h, node, tables) - if err != nil { - missing = append(missing, err.Error()) - continue - } - l.mu.Lock() - for k := range l.keys { - if !got[k] { - missing = append(missing, fmt.Sprintf("node %d %s absent", node, k)) - } - } - l.mu.Unlock() - } - if len(missing) == 0 { - t.Logf("converged: %d ACKed rows on every node, %s after load stopped", l.size(), time.Since(start).Round(time.Millisecond)) - return - } - if time.Since(start) > deadline { - sort.Strings(missing) - if len(missing) > 20 { - missing = append(missing[:20], fmt.Sprintf("... %d more", len(missing)-20)) - } - h.dumpNodeLogs("cluster_" + strings.ReplaceAll(t.Name(), "/", "_")) - t.Fatalf("ACKed rows still missing %s after load stopped:\n%s", deadline, strings.Join(missing, "\n")) - } - time.Sleep(convergencePoll) - } -} - -// runKeyWriters runs one writer per node in nodes: it inserts round-robin into -// every database.table and every third statement updates one of its own -// ACKed rows (the id, and so the ledger key, does not change). A statement -// that fails is simply retried on the next iteration. -func runKeyWriters(t *testing.T, h *ClusterHarness, l *keyLedger, nodes []int, targets [][2]string, stop <-chan struct{}) *sync.WaitGroup { - var wg sync.WaitGroup - for _, n := range nodes { - wg.Add(1) - go func(n int) { - defer wg.Done() - conns := map[string]*sql.DB{} - for _, tg := range targets { - if conns[tg[0]] == nil { - conns[tg[0]] = openTimedNodeDatabase(t, h, n, tg[0]) - } - } - writer := fmt.Sprintf("n%d", n) - for i := 0; ; i++ { - select { - case <-stop: - return - default: - } - tg := targets[i%len(targets)] - if i%3 == 2 { - if key, ok := l.pick(writer, i); ok { - parts := strings.SplitN(key, "/", 2) - dt := strings.SplitN(parts[0], ".", 2) - v := fmt.Sprintf("n%d#%d-u", n, i) - if _, err := execTimed(conns[dt[0]], "UPDATE "+dt[1]+" SET v = ? WHERE id = "+parts[1], v); err != nil { - time.Sleep(convergencePoll) - } - continue - } - } - conn := conns[tg[0]] - v := fmt.Sprintf("n%d#%d", n, i) - res, err := execTimed(conn, "INSERT INTO "+tg[1]+" (v) VALUES (?)", v) - if err != nil { - time.Sleep(convergencePoll) - continue - } - if id, err := res.LastInsertId(); err == nil { - l.acked(writer, ledgerKey(tg[0], tg[1], id)) - } - } - }(n) - } - return &wg -} - -// sleepWithProgress sleeps d while reporting progress to the stall watchdog, so an -// intentionally idle phase (e.g. "node stopped, about to restart") is never -// mistaken for a stall. -func sleepWithProgress(h *ClusterHarness, d time.Duration) { - end := time.Now().Add(d) - for time.Now().Before(end) { - h.Progress() - time.Sleep(convergencePoll) - } -} - -// startWithTables starts a cluster and creates each database (unless "marmot") and -// each database.table (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT). -func startWithTables(t *testing.T, h *ClusterHarness, tables map[string][]string) { - t.Helper() - if err := h.StartCluster(); err != nil { - h.Cleanup() - t.Fatalf("StartCluster: %v", err) - } - createTables(t, h, []int{1, 2, 3}, tables) -} - -func createTables(t *testing.T, h *ClusterHarness, nodes []int, tables map[string][]string) { - t.Helper() - for database, ts := range tables { - if database != "marmot" { - if _, err := h.ExecNode(1, "CREATE DATABASE "+database); err != nil { - t.Fatalf("CREATE DATABASE %s: %v", database, err) - } - waitDatabasePresence(t, h, nodes, database, true, 30*time.Second) - } - for _, table := range ts { - if _, err := execTimed(openTimedNodeDatabase(t, h, nodes[0], database), "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)"); err != nil { - t.Fatalf("CREATE TABLE %s.%s: %v", database, table, err) - } - for _, n := range nodes { - waitTableOn(t, h, n, database, table, 30*time.Second) - } - } - } -} - -func hasDatabase(t *testing.T, h *ClusterHarness, node int, database string) (bool, error) { - a, err := queryAnswer(openTimedNodeDatabase(t, h, node, "marmot"), "SHOW DATABASES") - if err != nil { - return false, err - } - for _, r := range a.rows { - if strings.EqualFold(r[0], database) { - return true, nil - } - } - return false, nil -} - -func waitDatabasePresence(t *testing.T, h *ClusterHarness, nodes []int, database string, present bool, timeout time.Duration) { - t.Helper() - deadline := time.Now().Add(timeout) - for _, n := range nodes { - for { - h.Progress() - ok, err := hasDatabase(t, h, n, database) - if err == nil && ok == present { - break +// createAutoIncTables creates each database of targets (other than marmot) +// and each table as (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT) through node +// id, waiting until every running node has them. +func createAutoIncTables(c *cluster, id int, targets []target) { + c.t.Helper() + for _, tg := range targets { + if tg.database != "marmot" { + if ok, err := c.hasDatabase(id, tg.database); err != nil || !ok { + c.createDatabase(id, tg.database) } - if time.Now().After(deadline) { - h.dumpNodeLogs("cluster_db_" + database) - t.Fatalf("database %s present=%v not reached on node %d within %s (err=%v)", database, present, n, timeout, err) - } - time.Sleep(convergencePoll) - } - } -} - -func waitTableOn(t *testing.T, h *ClusterHarness, node int, database, table string, timeout time.Duration) { - t.Helper() - deadline := time.Now().Add(timeout) - conn := openTimedNodeDatabase(t, h, node, database) - for { - h.Progress() - if _, err := queryAnswer(conn, "SELECT COUNT(*) FROM "+table); err == nil { - return } - if time.Now().After(deadline) { - h.dumpNodeLogs("cluster_table_" + table) - t.Fatalf("%s.%s not on node %d within %s", database, table, node, timeout) - } - time.Sleep(convergencePoll) - } -} - -// schemaVersionOf reads database's schema version through node's admin API. -func schemaVersionOf(h *ClusterHarness, node int, database string) (string, error) { - url := fmt.Sprintf("http://localhost:%d/admin/%s/metadata/schema/version", h.Nodes[node-1].GRPCPort, database) - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return "", err - } - req.Header.Set("X-Marmot-Secret", "test-secret") - resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req) - noteClientCall() - if err != nil { - return "", err + c.createTable(id, tg.database, tg.table, "CREATE TABLE "+tg.table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") } - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - var v interface{} - if err := json.Unmarshal(body, &v); err != nil { - return "", fmt.Errorf("status %d body %s", resp.StatusCode, body) - } - return fmt.Sprintf("%d:%s", resp.StatusCode, strings.TrimSpace(string(body))), nil } -// waitSchemaVersionsEqual waits until every node reports the same schema -// version for every database. -func waitSchemaVersionsEqual(t *testing.T, h *ClusterHarness, databases []string, timeout time.Duration) { - t.Helper() - deadline := time.Now().Add(timeout) - for { - h.Progress() - var diff []string - for _, database := range databases { - var vs []string - for n := 1; n <= numNodes; n++ { - v, err := schemaVersionOf(h, n, database) - if err != nil { - v = "err:" + err.Error() - } - vs = append(vs, v) - } - if vs[0] != vs[1] || vs[0] != vs[2] { - diff = append(diff, fmt.Sprintf("%s: %v", database, vs)) - } - } - if len(diff) == 0 { - return - } - if time.Now().After(deadline) { - t.Fatalf("schema versions differ after %s: %v", timeout, diff) - } - time.Sleep(time.Second) - } -} - -// waitLogLine waits for the n-th occurrence of line in node's log. -func waitLogLine(h *ClusterHarness, node int, line string, n int, timeout time.Duration) bool { - deadline := time.Now().Add(timeout) - for time.Now().Before(deadline) { - data, _ := os.ReadFile(h.Nodes[node-1].LogFile) - if strings.Count(string(data), line) >= n { - return true - } - time.Sleep(20 * time.Millisecond) - } - return false -} - -// TestNodeDownTwoDatabasesConverges: a node stopped for 15s under -// insert+update load across two databases, then returned: every ACKed row on -// every node within convergenceDeadline. +// TestNodeDownTwoDatabasesConverges: a node killed under insert and update +// load across two databases, then restarted: every node holds every ACKed +// row with its ACKed value. func TestNodeDownTwoDatabasesConverges(t *testing.T) { - h := NewClusterHarness(t) - defer h.Cleanup() - tables := map[string][]string{"outagea": {"t"}, "outageb": {"t"}} - startWithTables(t, h, tables) - h.Phase("start") - - l := newKeyLedger(h) - stop := make(chan struct{}) - wg := runKeyWriters(t, h, l, []int{1, 2, 3}, [][2]string{{"outagea", "t"}, {"outageb", "t"}}, stop) - sleepWithProgress(h, 3*time.Second) - h.Phase("load") - - if err := h.StopNode(3); err != nil { - t.Fatal(err) - } - h.Phase("node3-stopped") - sleepWithProgress(h, 15*time.Second) - - if err := h.StartNode(3); err != nil { - t.Fatal(err) - } - if err := h.WaitForAlive(3, restartDeadline); err != nil { - t.Fatal(err) - } - h.Phase("node3-alive") - sleepWithProgress(h, 3*time.Second) - close(stop) - wg.Wait() - h.Phase("load-stopped") - - if len(l.dups) > 0 { - t.Fatalf("duplicate ids: %v", l.dups) - } - waitKeysConverged(t, h, l, tables, convergenceDeadline) - h.Phase("converged") + c := newCluster(t) + c.start() + targets := []target{{"outagea", "t"}, {"outageb", "t"}} + createAutoIncTables(c, 1, targets) + l := newRowLedger() + stop := runWriters(c, l, []int{1, 2, 3}, targets, true) + l.waitForMore(c, ledgerStep, "before the outage") + c.kill(3) + l.waitForMore(c, ledgerStep, "while node 3 is down") + c.restart(3) + l.waitForMore(c, ledgerStep, "after node 3 returned") + stop() + l.waitConverged(c, targets) } // TestExplicitTxnAcrossNodeDownConverges: an explicit transaction (BEGIN; -// several DML; COMMIT) held open on node 1, with node 3 stopped during it: -// delivered everywhere. +// several DML; COMMIT) held open on node 1 while node 3 is killed during it +// is delivered everywhere, along with the load nodes 2 and 3 wrote around it. +// Its first inserts into xa and xb claim node 1's id ranges inside BEGIN +// while nodes 2 and 3 claim theirs concurrently. +// +// Mutation: flush the batch committer before a claim-only commit +// (TransactionManager.CommitTransactionAfter). The xb claim waits on DML +// queued behind the writer the transaction's own session holds, and the +// transaction fails. func TestExplicitTxnAcrossNodeDownConverges(t *testing.T) { - h := NewClusterHarness(t) - defer h.Cleanup() - tables := map[string][]string{"marmot": {"xa", "xb"}} - startWithTables(t, h, tables) - h.Phase("start") - - l := newKeyLedger(h) - stop := make(chan struct{}) - wg := runKeyWriters(t, h, l, []int{2, 3}, [][2]string{{"marmot", "xa"}, {"marmot", "xb"}}, stop) - sleepWithProgress(h, 3*time.Second) - h.Phase("load") - - db1 := openTimedNodeDatabase(t, h, 1, "marmot") - var txnKeys []string - for attempt := 1; ; attempt++ { - keys, err := runExplicitTxnAcrossOutage(t, h, l, db1, attempt == 1) - if err == nil { - txnKeys = keys + c := newCluster(t) + c.start() + targets := []target{{"marmot", "xa"}, {"marmot", "xb"}} + createAutoIncTables(c, 1, targets) + l := newRowLedger() + stop := runWriters(c, l, []int{2, 3}, targets, true) + l.waitForMore(c, ledgerStep, "before the transaction") + // A COMMIT refused with 1213 (a write-write conflict with the concurrent + // writers) committed nothing and is documented retryable: the client runs + // the transaction again. Only the first attempt kills node 3. A 1213 on a + // statement before COMMIT is not retried: there it means the transaction's + // pinned session outlived the lock wait, which is the stall this test + // guards. + var err error + for attempt := 1; attempt <= explicitTxnAttempts; attempt++ { + err = runExplicitTxnAcrossOutage(c, l, attempt == 1) + var refused *commitRefusedError + if !errors.As(err, &refused) || mysqlCode(err) != mysqlcode.ErrCodeDeadlock { break } - if !isDeadlockRetry(err) || attempt == explicitTxnAttempts { - t.Fatalf("explicit txn attempt %d: %v", attempt, err) - } - // 1213 is documented retryable: nothing of the transaction was - // committed, and its ids were already returned, so the product cannot - // re-claim them at COMMIT - the client retries the whole transaction. - t.Logf("explicit txn attempt %d refused with retryable 1213, retrying: %v", attempt, err) - sleepWithProgress(h, explicitTxnBackoff) - } - h.Phase("commit") - for _, k := range txnKeys { - l.acked("txn", k) } - t.Logf("explicit txn committed %d rows, ACKed=%d", len(txnKeys), l.size()) - sleepWithProgress(h, 3*time.Second) - - if err := h.StartNode(3); err != nil { - t.Fatal(err) - } - if err := h.WaitForAlive(3, restartDeadline); err != nil { - t.Fatal(err) - } - h.Phase("node3-alive") - sleepWithProgress(h, 2*time.Second) - close(stop) - wg.Wait() - h.Phase("load-stopped") - - if len(l.dups) > 0 { - t.Fatalf("duplicate ids: %v", l.dups) + if err != nil { + t.Fatalf("explicit transaction: %v", err) } - waitKeysConverged(t, h, l, tables, convergenceDeadline) - h.Phase("converged") + c.restart(3) + l.waitForMore(c, ledgerStep, "after node 3 returned") + stop() + l.waitConverged(c, targets) } -// explicitTxnAttempts caps how often TestExplicitTxnAcrossNodeDownConverges -// runs its explicit transaction on the documented-retryable 1213, and -// explicitTxnBackoff is the pause before each retry. -const ( - explicitTxnAttempts = 5 - explicitTxnBackoff = time.Second -) +// commitRefusedError is an explicit transaction's COMMIT failing, as opposed +// to one of its statements. +type commitRefusedError struct{ err error } -// isDeadlockRetry reports whether err is MySQL 1213, the retryable refusal -// of a whole transaction. -func isDeadlockRetry(err error) bool { - var mysqlErr *mysql.MySQLError - return errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlcode.ErrCodeDeadlock -} +func (e *commitRefusedError) Error() string { return "COMMIT: " + e.err.Error() } +func (e *commitRefusedError) Unwrap() error { return e.err } -// runExplicitTxnAcrossOutage runs the explicit transaction of -// TestExplicitTxnAcrossNodeDownConverges on node 1 - three -// inserts, node 3 stopped mid-transaction, three more inserts and an UPDATE -// of a row node 2 committed, held open a while before COMMIT - and returns -// the keys it inserted once COMMIT succeeds. Only the first attempt stops -// node 3 and holds the transaction open; a retry runs straight through. Any -// failure rolls it back. -func runExplicitTxnAcrossOutage(t *testing.T, h *ClusterHarness, l *keyLedger, db1 *sql.DB, firstAttempt bool) ([]string, error) { - conn, err := db1.Conn(context.Background()) +// explicitTxnAttempts caps TestExplicitTxnAcrossNodeDownConverges' runs of +// its explicit transaction on a COMMIT refused with 1213. +const explicitTxnAttempts = 5 + +// runExplicitTxnAcrossOutage runs, on one connection to node 1: BEGIN, three +// inserts, (first attempt only) a kill of node 3, three more inserts and an +// UPDATE of a row node 2 committed, then COMMIT, recording the rows once the +// COMMIT is ACKed. Any failure rolls the transaction back. +func runExplicitTxnAcrossOutage(c *cluster, l *rowLedger, killNode3 bool) error { + tx, err := c.db(1, "marmot").Begin() if err != nil { - return nil, err + return err } - defer conn.Close() - committed := false - defer func() { - if !committed { - _, _ = connExecTimed(conn, "ROLLBACK") - } - }() - var keys []string + defer tx.Rollback() + type row struct{ key, v string } + var rows []row insert := func(table, v string) error { - res, err := connExecTimed(conn, "INSERT INTO "+table+" (v) VALUES (?)", v) + res, err := tx.Exec("INSERT INTO "+table+" (v) VALUES (?)", v) if err != nil { - return err + return fmt.Errorf("INSERT %s %s: %w", table, v, err) } id, _ := res.LastInsertId() - keys = append(keys, ledgerKey("marmot", table, id)) + rows = append(rows, row{ledgerKey("marmot", table, id), v}) return nil } - if _, err := connExecTimed(conn, "BEGIN"); err != nil { - return nil, err - } - h.Phase("begin") for i, table := range []string{"xa", "xb", "xa"} { if err := insert(table, fmt.Sprintf("txn-pre-%d", i)); err != nil { - return nil, err + return err } - } - if firstAttempt { - sleepWithProgress(h, 2*time.Second) - if err := h.StopNode(3); err != nil { - t.Fatal(err) + if i == 0 { + // The transaction now holds node 1's SQLite writer: let nodes 2 + // and 3 commit while it does, so their commits queue on node 1 + // before the xb insert claims a range. + l.waitForMore(c, 3, "while node 1's transaction holds its writer") } } + if killNode3 { + c.kill(3) + } for i := 0; i < 3; i++ { - // xa: node 1 already holds an id range for it, so no claim (which - // needs a quorum while node 3 is not yet suspected) is needed. if err := insert("xa", fmt.Sprintf("txn-post-%d", i)); err != nil { - return nil, err + return err } } - // Update a row node 2 committed before this transaction. - if key, ok := l.pick("n2", 1); ok && strings.HasPrefix(key, "marmot.xa/") { - if _, err := connExecTimed(conn, "UPDATE xa SET v = ? WHERE id = "+strings.TrimPrefix(key, "marmot.xa/"), "txn-upd"); err != nil { - return nil, err + updated, ok := l.pick("n2", 1) + if ok && strings.HasPrefix(updated, "marmot.xa/") { + if _, err := tx.Exec("UPDATE xa SET v = 'txn-upd' WHERE id = " + strings.TrimPrefix(updated, "marmot.xa/")); err != nil { + return fmt.Errorf("UPDATE %s: %w", updated, err) } } - if firstAttempt { - sleepWithProgress(h, 3*time.Second) + if err := tx.Commit(); err != nil { + return &commitRefusedError{err: err} } - if _, err := connExecTimed(conn, "COMMIT"); err != nil { - return nil, err + for _, r := range rows { + l.inserted("txn", r.key, r.v) } - committed = true - return keys, nil -} - -// connExecTimed runs one statement on an already-open *sql.Conn under -// clusterQueryTimeout (execTimed takes a *sql.DB, which an explicit -// transaction's pinned connection is not). -func connExecTimed(conn *sql.Conn, q string, args ...interface{}) (sql.Result, error) { - ctx, cancel := context.WithTimeout(context.Background(), clusterQueryTimeout) - defer cancel() - defer noteClientCall() - return conn.ExecContext(ctx, q, args...) + if ok { + // Node 2's writer may update the same row concurrently: only its + // presence is certain. + l.updated(updated, "txn-upd", false) + } + return nil } -// TestDatabaseOpsWhileNodeDownConverge: CREATE DATABASE and DROP DATABASE -// (and a drop+recreate) while a node is down: the returning node converges -// and no dropped database is resurrected. +// TestDatabaseOpsWhileNodeDownConverge: DROP DATABASE, CREATE DATABASE and a +// drop and recreate while a node is down: the returning node ends with the +// same databases and rows as the others, and no dropped database comes back. func TestDatabaseOpsWhileNodeDownConverge(t *testing.T) { - h := NewClusterHarness(t) - defer h.Cleanup() - startWithTables(t, h, map[string][]string{"holddb": {"t"}, "recreatedb": {"t"}}) - h.Phase("start") - - l := newKeyLedger(h) + c := newCluster(t) + c.start() + createAutoIncTables(c, 1, []target{{"holddb", "t"}, {"recreatedb", "t"}}) for i := 0; i < 5; i++ { for _, database := range []string{"holddb", "recreatedb"} { - if _, err := execTimed(openTimedNodeDatabase(t, h, 1, database), "INSERT INTO t (v) VALUES (?)", fmt.Sprintf("old-%d", i)); err != nil { - t.Fatal(err) - } + c.mustExec(1, database, "INSERT INTO t (v) VALUES (?)", fmt.Sprintf("old-%d", i)) } } - for n := 1; n <= numNodes; n++ { - for _, database := range []string{"holddb", "recreatedb"} { - for { - a, err := queryAnswer(openTimedNodeDatabase(t, h, n, database), "SELECT COUNT(*) FROM t") - if err == nil && a.rows[0][0] == "5" { - break - } - sleepWithProgress(h, convergencePoll) - } - } + for _, database := range []string{"holddb", "recreatedb"} { + c.waitRows(database, "SELECT COUNT(*) FROM t", []string{"5"}, 1, 2, 3) } - h.Phase("baseline-load") - if err := h.StopNode(3); err != nil { - t.Fatal(err) - } - h.Phase("node3-stopped") - up := []int{1, 2} - if _, err := h.ExecNode(2, "DROP DATABASE holddb"); err != nil { - t.Fatalf("DROP holddb: %v", err) - } - if _, err := h.ExecNode(1, "DROP DATABASE recreatedb"); err != nil { - t.Fatalf("DROP recreatedb: %v", err) - } - waitDatabasePresence(t, h, up, "recreatedb", false, 30*time.Second) - createTables(t, h, up, map[string][]string{"newdb": {"t"}, "recreatedb": {"t"}}) + c.kill(3) + c.mustExec(2, "marmot", "DROP DATABASE holddb") + c.mustExec(1, "marmot", "DROP DATABASE recreatedb") + c.waitDatabase("recreatedb", false, 1, 2) + c.waitDatabase("holddb", false, 1, 2) + targets := []target{{"newdb", "t"}, {"recreatedb", "t"}} + createAutoIncTables(c, 1, targets) + l := newRowLedger() for i := 0; i < 20; i++ { - for _, database := range []string{"newdb", "recreatedb"} { + for _, tg := range targets { v := fmt.Sprintf("new-%d", i) - res, err := execTimed(openTimedNodeDatabase(t, h, up[i%2], database), "INSERT INTO t (v) VALUES (?)", v) - if err != nil { - t.Fatalf("insert %s: %v", database, err) - } + res := c.mustExec(1+i%2, tg.database, "INSERT INTO t (v) VALUES (?)", v) id, _ := res.LastInsertId() - l.acked("w", ledgerKey(database, "t", id)) + l.inserted("w", ledgerKey(tg.database, "t", id), v) } } - h.Phase("ops-done-node3-down") - - if err := h.StartNode(3); err != nil { - t.Fatal(err) - } - if err := h.WaitForAlive(3, restartDeadline); err != nil { - t.Fatal(err) - } - h.Phase("node3-alive") - waitDatabasePresence(t, h, []int{3}, "holddb", false, convergenceDeadline) - waitDatabasePresence(t, h, []int{3}, "newdb", true, convergenceDeadline) - waitKeysConverged(t, h, l, map[string][]string{"newdb": {"t"}, "recreatedb": {"t"}}, convergenceDeadline) - h.Phase("converged") - // One more anti-entropy round: an old peer must not bring holddb back. - sleepWithProgress(h, 6*time.Second) - for n := 1; n <= numNodes; n++ { - ok, err := hasDatabase(t, h, n, "holddb") - if err != nil || ok { - t.Fatalf("node %d: holddb present=%v err=%v after the drop", n, ok, err) + c.restart(3) + c.waitDatabase("holddb", false, 3) + c.waitDatabase("newdb", true, 3) + l.waitConverged(c, targets) + for _, id := range c.all() { + if ok, err := c.hasDatabase(id, "holddb"); err != nil || ok { + t.Fatalf("node %d: holddb present=%v (err %v) after the drop", id, ok, err) } } - h.Phase("no-resurrection-checked") } -// TestKillMidCatchUpConverges: kill -9 of a node mid catch-up (DDL and DML -// missed while down), then restart: converges, no duplicate apply, equal -// schema versions. The kill lands 150ms after the startup pull began, inside -// the 20ms-300ms range over which this was seen to hold. -func TestKillMidCatchUpConverges(t *testing.T) { - h := NewClusterHarness(t) - defer h.Cleanup() - tables := map[string][]string{"marmot": {"catchupa"}, "catchup": {"catchupb"}} - startWithTables(t, h, tables) - h.Phase("start") - - l := newKeyLedger(h) - stop := make(chan struct{}) - targets := [][2]string{{"marmot", "catchupa"}, {"catchup", "catchupb"}} - wg := runKeyWriters(t, h, l, []int{1, 2}, targets, stop) - sleepWithProgress(h, 3*time.Second) - h.Phase("load") - - if err := h.StopNode(3); err != nil { - t.Fatal(err) +// pulledSeqs sums node id's pull cursors over every peer and database, as its +// admin API reports them: it grows as the node applies peers' commit logs. +func (c *cluster) pulledSeqs(id int) (uint64, error) { + var state struct { + Peers []struct { + Databases []struct { + Seq uint64 `json:"local_pull_cursor_seq"` + } `json:"databases"` + } `json:"peers"` } - h.Phase("node3-stopped") - sleepWithProgress(h, 4*time.Second) - - c1 := openTimedNodeDatabase(t, h, 1, "marmot") - for _, q := range []string{"ALTER TABLE catchupa ADD COLUMN c1 INT", "CREATE TABLE catchupd (id INT PRIMARY KEY)"} { - if _, err := execTimed(c1, q); err != nil { - t.Fatalf("%s: %v", q, err) - } - } - if _, err := execTimed(openTimedNodeDatabase(t, h, 2, "catchup"), "ALTER TABLE catchupb ADD COLUMN c2 INT"); err != nil { - t.Fatalf("alter catchupb: %v", err) + if err := c.admin(id, http.MethodGet, "/cluster/replication", clientTimeout, &state); err != nil { + return 0, err } - sleepWithProgress(h, 5*time.Second) - h.Phase("missed-ddl") - - const pullLine = "Starting startup log pull" - before, _ := os.ReadFile(h.Nodes[2].LogFile) - if err := h.StartNode(3); err != nil { - t.Fatal(err) - } - if !waitLogLine(h, 3, pullLine, strings.Count(string(before), pullLine)+1, 20*time.Second) { - t.Logf("startup pull line not seen; killing anyway") - } - time.Sleep(150 * time.Millisecond) - if err := h.KillNode(3); err != nil { - t.Fatal(err) + var total uint64 + for _, p := range state.Peers { + for _, d := range p.Databases { + total += d.Seq + } } - h.Phase("killed-mid-catchup") - sleepWithProgress(h, 2*time.Second) + return total, nil +} - if err := h.StartNode(3); err != nil { - t.Fatal(err) - } - if err := h.WaitForAlive(3, restartDeadline); err != nil { +// TestKillMidCatchUpConverges: a node that missed DML and DDL while down is +// SIGKILLed again as soon as its restart has begun pulling its peers' logs, +// then restarted: every node converges on every ACKed row, applies every DDL +// once, and reports the same schema versions. +func TestKillMidCatchUpConverges(t *testing.T) { + c := newCluster(t) + c.start() + targets := []target{{"marmot", "catchupa"}, {"catchup", "catchupb"}} + createAutoIncTables(c, 1, targets) + l := newRowLedger() + stop := runWriters(c, l, []int{1, 2}, targets, true) + l.waitForMore(c, ledgerStep, "before the outage") + before, err := c.pulledSeqs(3) + if err != nil { t.Fatal(err) } - h.Phase("node3-alive") - sleepWithProgress(h, 2*time.Second) - close(stop) - wg.Wait() - h.Phase("load-stopped") - - if len(l.dups) > 0 { - t.Fatalf("duplicate ids: %v", l.dups) - } - waitKeysConverged(t, h, l, tables, convergenceDeadline) - h.Phase("converged") - waitSchemaVersionsEqual(t, h, []string{"marmot", "catchup"}, 15*time.Second) - h.Phase("schema-versions-equal") - - for n := 1; n <= numNodes; n++ { - for _, q := range []struct{ db, q string }{ + c.kill(3) + l.waitForMore(c, 3*ledgerStep, "while node 3 is down") + c.mustExec(1, "marmot", "ALTER TABLE catchupa ADD COLUMN c1 INT") + c.mustExec(1, "marmot", "CREATE TABLE catchupd (id INT PRIMARY KEY)") + c.mustExec(2, "catchup", "ALTER TABLE catchupb ADD COLUMN c2 INT") + l.waitForMore(c, ledgerStep, "after the DDL") + + c.startNode(3) + waitFor(t, "node 3 begins pulling its peers' logs", readyDeadline, func() (bool, string) { + pulled, err := c.pulledSeqs(3) + return err == nil && pulled > before, fmt.Sprintf("pulled %d (before %d), err %v", pulled, before, err) + }) + c.kill(3) + c.restart(3) + l.waitForMore(c, ledgerStep, "after node 3 returned") + stop() + l.waitConverged(c, targets) + c.waitSchemaVersionsEqual("marmot", "catchup") + for _, id := range c.all() { + for _, q := range []struct{ database, query string }{ {"marmot", "SELECT COUNT(*) FROM catchupd"}, {"marmot", "SELECT c1 FROM catchupa LIMIT 1"}, {"catchup", "SELECT c2 FROM catchupb LIMIT 1"}, } { - if _, err := queryAnswer(openTimedNodeDatabase(t, h, n, q.db), q.q); err != nil { - t.Errorf("node %d %s: %v", n, q.q, err) + if _, err := c.rows(id, q.database, q.query); err != nil { + t.Errorf("node %d: %s: %v", id, q.query, err) } } } } -// TestConvergesWithLiveGC runs the cluster with GC actually deleting log -// entries (gc_interval_seconds lives in [replication]; a harness that wrote -// it elsewhere would silently run the 60s default and never see GC delete -// anything). With GC on and running every 5s, writes settle -// enough to be GC'd (seen via the log line GC logs when it deletes committed -// records, db/transaction.go cleanupOldTransactionRecords), and a node killed -// and restarted afterward still converges to every ACKed row - GC must not -// remove anything a still-behind node needs to catch up on. +// committedTxnCount is node id's count of committed transaction records in +// database, which GC lowers as it deletes them. +func (c *cluster) committedTxnCount(id int, database string) (int64, error) { + var counters struct { + Count int64 `json:"committed_txn_count"` + } + err := c.admin(id, http.MethodGet, "/"+database+"/metadata/counters", clientTimeout, &counters) + return counters.Count, err +} + +// TestConvergesWithLiveGC runs the cluster with GC deleting log records every +// second (both retention floors at 0, so a record is eligible once every +// peer's watermark passed it). Once GC has deleted records on every node, a +// node killed and restarted still converges to every ACKed row: GC must not +// remove anything a node that is behind needs to catch up. func TestConvergesWithLiveGC(t *testing.T) { - const table = "livegc" - harness := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)", func(h *ClusterHarness) { - // gc_interval_seconds must stay >= anti_entropy_interval_seconds (the - // harness's 5s) and gc_min_retention_hours must stay >= - // delta_sync_threshold_seconds in hours (cfg.Validate enforces - // both), so both retention knobs are dropped to 0: a committed - // record becomes GC-eligible as soon as every peer's watermark has - // passed it, instead of after 2h. - h.GCIntervalSeconds = 5 - h.GCMinRetentionHours = 0 - h.DeltaSyncThresholdSeconds = 0 + c := newCluster(t, func(cfg *clusterConfig) { + cfg.gcIntervalSeconds = 1 + cfg.gcMinRetentionHours = 0 + cfg.deltaSyncThresholdSecs = 0 }) - defer harness.Cleanup() - harness.Phase("start") - - tables := []string{table} - ledger := newRowLedger(harness) - stop := make(chan struct{}) - wg := runLedgerWriters(t, harness, ledger, []int{1, 2, 3}, tables, stop) - ledger.waitForMore(t, 150, "before the GC pass") - harness.Phase("load") - - const gcLine = "GC: Cleaned up old transaction records" - deadline := time.Now().Add(20 * time.Second) - for n := 1; n <= numNodes; n++ { - for { - data, _ := os.ReadFile(harness.Nodes[n-1].LogFile) - if strings.Contains(string(data), gcLine) { - break + c.start() + targets := []target{{"marmot", "livegc"}} + createAutoIncTables(c, 1, targets) + l := newRowLedger() + stop := runWriters(c, l, []int{1, 2, 3}, targets, false) + l.waitForMore(c, ledgerStep, "before the GC pass") + + peak := map[int]int64{} + waitFor(t, "GC deleted committed records on every node", readyDeadline, func() (bool, string) { + c.progress() + deleted := 0 + for _, id := range c.all() { + n, err := c.committedTxnCount(id, "marmot") + if err != nil { + return false, err.Error() } - if time.Now().After(deadline) { - close(stop) - wg.Wait() - harness.dumpNodeLogs("live_gc_no_pass") - t.Fatalf("node %d: no %q within %s of live GC running", n, gcLine, 20*time.Second) + peak[id] = max(peak[id], n) + if n < peak[id] { + deleted++ } - harness.Progress() - time.Sleep(convergencePoll) - } - } - harness.Phase("gc-pass-seen-on-every-node") - - ledger.waitForMore(t, 50, "after the GC pass") - if err := harness.KillNode(3); err != nil { - t.Fatalf("kill node 3: %v", err) - } - harness.Phase("node3-killed") - ledger.waitForMore(t, 50, "while node 3 is down") - if err := harness.StartNode(3); err != nil { - t.Fatalf("restart node 3: %v", err) - } - if err := harness.WaitForAlive(3, restartDeadline); err != nil { - t.Fatalf("node 3 did not come back: %v", err) - } - harness.Phase("node3-alive") - ledger.waitForMore(t, 50, "after node 3 returned") - close(stop) - wg.Wait() - harness.Phase("load-stopped") - - if len(ledger.dups) > 0 { - t.Fatalf("duplicate ids: %v", ledger.dups) - } - waitLedgerConverged(t, harness, ledger, tables) - harness.Phase("converged") -} - -// missingKeysOn lists every ACKed row of tables that node lacks. -func missingKeysOn(t *testing.T, h *ClusterHarness, l *keyLedger, node int, tables map[string][]string) ([]string, error) { - got, err := rowIDsOn(t, h, node, tables) - if err != nil { - return nil, err - } - l.mu.Lock() - defer l.mu.Unlock() - var missing []string - for k := range l.keys { - if !got[k] { - missing = append(missing, k) } - } - sort.Strings(missing) - return missing, nil + return deleted == len(c.nodes), fmt.Sprintf("peaks %v", peak) + }) + c.kill(3) + l.waitForMore(c, ledgerStep, "while node 3 is down") + c.restart(3) + l.waitForMore(c, ledgerStep, "after node 3 returned") + stop() + l.waitConverged(c, targets) } // TestPreparedOnKilledNodeCommittedByPeersConverges: transactions node 3 -// durably prepared but never committed, which its -// peers did commit: an explicit transaction's pinned session holds node 3's -// SQLite writer while nodes 1 and 2 commit writes, so every COMMIT to node 3 -// waits on that writer; node 3 is then killed (kill -9) and restarted with -// those transactions recovered PENDING. The stale-transaction GC is kept out -// of the test's window (heartbeat_timeout_seconds far above it), so only the -// log pull resolving those records - committing them through the local -// commit path - can deliver the rows. Node 3 must hold every ACKed row the -// moment it reports itself ALIVE (JOINING -> ALIVE only once caught up), and -// every node must converge within convergenceDeadline. +// durably prepared (strict_prepare_sync: a PREPARE is synced before it is +// ACKed) but never committed, which its peers did commit. An +// explicit transaction's pinned session holds node 3's SQLite writer while +// nodes 1 and 2 commit writes, so every COMMIT to node 3 waits on that +// writer; node 3 is then SIGKILLed and restarted with those transactions +// recovered PENDING. The stale-transaction GC is kept out of the test +// (heartbeat_timeout_seconds far above its budget), so only the log pull +// resolving those records can deliver the rows. Node 3 must hold every +// ACKed row the moment it reports itself ALIVE (JOINING -> ALIVE only once +// caught up), and every node must converge. func TestPreparedOnKilledNodeCommittedByPeersConverges(t *testing.T) { - h := NewClusterHarness(t, func(h *ClusterHarness) { h.HeartbeatTimeoutSeconds = 600 }) - defer h.Cleanup() - tables := map[string][]string{"marmot": {"pa", "pb"}} - startWithTables(t, h, tables) - h.Phase("start") - - l := newKeyLedger(h) - stop := make(chan struct{}) - wg := runKeyWriters(t, h, l, []int{1, 2}, [][2]string{{"marmot", "pa"}}, stop) - sleepWithProgress(h, 2*time.Second) - h.Phase("load") - - conn, err := openTimedNodeDatabase(t, h, 3, "marmot").Conn(context.Background()) + c := newCluster(t, func(cfg *clusterConfig) { + cfg.heartbeatTimeoutSecs = 600 + cfg.strictPrepareSync = true + }) + c.start() + targets := []target{{"marmot", "pa"}, {"marmot", "pb"}} + createAutoIncTables(c, 1, targets) + l := newRowLedger() + stop := runWriters(c, l, []int{1, 2}, targets[:1], false) + l.waitForMore(c, ledgerStep, "before the pin") + + pin, err := c.db(3, "marmot").Begin() if err != nil { t.Fatal(err) } - defer conn.Close() - for _, q := range []string{"BEGIN", "INSERT INTO pb (v) VALUES ('pinned')"} { - if _, err := connExecTimed(conn, q); err != nil { - t.Fatalf("%s: %v", q, err) - } + if _, err := pin.Exec("INSERT INTO pb (v) VALUES ('pinned')"); err != nil { + t.Fatalf("pin node 3's writer: %v", err) } - h.Phase("writer-pinned") - before := l.size() - sleepWithProgress(h, 4*time.Second) - if l.size() == before { - t.Fatal("no write committed while node 3's writer was pinned") - } - if err := h.KillNode(3); err != nil { - t.Fatal(err) - } - close(stop) - wg.Wait() - h.Phase("node3-killed") - - if err := h.StartNode(3); err != nil { - t.Fatal(err) - } - deadline := time.Now().Add(convergenceDeadline) - for { - h.Progress() - status, err := h.selfStatus(3) - if err == nil && status == "ALIVE" { - // Read right after observing ALIVE; a read error only means - // node 3's MySQL listener is not up yet. - missing, readErr := missingKeysOn(t, h, l, 3, tables) - if readErr == nil && len(missing) > 0 { - t.Fatalf("node 3 promoted ALIVE while missing %d ACKed rows: %v", len(missing), missing) - } - if readErr == nil { - break + l.waitForMore(c, ledgerStep, "while node 3's writer is pinned") + waitFor(t, "node 3 holds transactions its peers committed PENDING", readyDeadline, func() (bool, string) { + var pending []struct { + NodeID uint64 `json:"node_id"` + } + if err := c.admin(3, http.MethodGet, "/marmot/metadata/transactions/pending", clientTimeout, &pending); err != nil { + return false, err.Error() + } + peers := 0 + for _, p := range pending { + if p.NodeID != 3 { + peers++ } } - if time.Now().After(deadline) { - missing, readErr := missingKeysOn(t, h, l, 3, tables) - h.dumpNodeLogs("cluster_prepared_killed") - t.Fatalf("node 3 not ALIVE within %s (status=%q err=%v, read err=%v); missing %d ACKed rows: %v", - convergenceDeadline, status, err, readErr, len(missing), missing) + return peers > 0, fmt.Sprintf("%d pending, %d from peers", len(pending), peers) + }) + c.kill(3) + _ = pin.Rollback() + stop() + + c.startNode(3) + waitFor(t, "node 3 ALIVE", readyDeadline, func() (bool, string) { + statuses, err := c.statuses(3) + if err != nil || statuses[3] != "ALIVE" || c.ping(3) != nil { + return false, fmt.Sprint(statuses, err) } - time.Sleep(convergencePoll) - } - h.Phase("node3-alive") + if bad := l.mismatchesOn(c, 3, targets[:1]); len(bad) > 0 { + t.Fatalf("node 3 reported ALIVE while missing %d ACKed rows, first: %s", len(bad), bad[0]) + } + return true, "" + }) + c.waitReady() + l.waitConverged(c, targets[:1]) +} + +// schemaVersion is database's schema version as node id's admin API reports +// it. +func (c *cluster) schemaVersion(id int, database string) (string, error) { + var v any + err := c.admin(id, http.MethodGet, "/"+database+"/metadata/schema/version", clientTimeout, &v) + return fmt.Sprint(v), err +} - waitKeysConverged(t, h, l, tables, convergenceDeadline) - h.Phase("converged") +// waitSchemaVersionsEqual waits until every node reports the same schema +// version for every database. +func (c *cluster) waitSchemaVersionsEqual(databases ...string) { + c.t.Helper() + waitFor(c.t, fmt.Sprintf("equal schema versions of %v", databases), replicationDeadline, func() (bool, string) { + for _, database := range databases { + var versions []string + for _, id := range c.all() { + v, err := c.schemaVersion(id, database) + if err != nil { + return false, fmt.Sprintf("node %d %s: %v", id, database, err) + } + versions = append(versions, v) + } + for _, v := range versions[1:] { + if v != versions[0] { + return false, fmt.Sprintf("%s: %v", database, versions) + } + } + } + return true, "" + }) } diff --git a/test/partition_test.go b/test/partition_test.go index 33003b8a..7df0da7d 100644 --- a/test/partition_test.go +++ b/test/partition_test.go @@ -3,6 +3,7 @@ package test import ( "context" "fmt" + "runtime" "sync" "testing" "time" @@ -370,12 +371,10 @@ func TestConcurrentWritesDuringPartition(t *testing.T) { node1 := cluster.GetNode(1) node2 := cluster.GetNode(2) - // Add delay to ensure different timestamps - time.Sleep(1 * time.Millisecond) txn1, _ := node1.Write("key", "value_from_node1") - - time.Sleep(1 * time.Millisecond) - txn2, _ := node2.Write("key", "value_from_node2") // This should win (later timestamp) + // node2's clock has seen txn1, so its write is strictly later and wins. + node2.clock.Update(txn1.Timestamp) + txn2, _ := node2.Write("key", "value_from_node2") // Replicate within partitions (will fail quorum for node1) cluster.ReplicateToQuorum(txn1) @@ -489,9 +488,9 @@ func TestConsistencyInvariant(t *testing.T) { default: // Create random partition cluster.partitionSim.PartitionNodes([]uint64{1}, []uint64{2, 3}) - time.Sleep(50 * time.Millisecond) + runtime.Gosched() cluster.partitionSim.Reset() - time.Sleep(50 * time.Millisecond) + runtime.Gosched() } } }() diff --git a/test/quorum_of_one_test.go b/test/quorum_of_one_test.go index b948f14d..40bd56ea 100644 --- a/test/quorum_of_one_test.go +++ b/test/quorum_of_one_test.go @@ -1,158 +1,68 @@ package test import ( - "fmt" - "strings" + "slices" "testing" - "time" + + "github.com/maxpert/marmot/protocol/mysqlcode" ) -// TestLoneRestartedNodeRefusesToBeItsOwnQuorum is F2: a node restarted while its -// peers are unreachable must not commit alone. -// -// Before this step the registry started with self only and was never persisted, -// so a restarted node computed a total membership of 1, was its own majority, -// and committed writes its peers never saw. Two mechanisms now prevent it: the -// registry restores its last known membership from disk, so the quorum -// denominator is 3 rather than 1; and, when no snapshot is readable, the -// coordinator refuses to act as a quorum of one on any node configured with -// seeds. +// TestLoneRestartedNodeRefusesToBeItsOwnQuorum: a node restarted while its +// peers are unreachable must not commit alone. Its registry restores its +// last known membership from disk, so the quorum denominator is 3 rather +// than 1 (the write fails with 1105, quorum not achieved); and when no +// membership is readable, the coordinator refuses to act as a quorum of one +// on a node configured with seeds (1205). Either way nothing is written +// locally, and once the peers are back the same write succeeds everywhere. // // Mutation: drop restoreMembershipLocked in the registry constructor AND the -// belt in GetClusterState. The lone node accepts the write and the row-count -// assertion below fires. +// belt in GetClusterState. The lone node accepts the write. func TestLoneRestartedNodeRefusesToBeItsOwnQuorum(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - if err := harness.StartCluster(); err != nil { - t.Fatalf("Failed to start cluster: %v", err) - } - + c := newCluster(t) + c.start() const table = "f2_quorum" - allNodes := []int{1, 2, 3} - - if _, err := harness.ExecNode(1, fmt.Sprintf( - "CREATE TABLE %s (id INT PRIMARY KEY, v TEXT)", table)); err != nil { - t.Fatalf("CREATE TABLE: %v", err) - } - if err := harness.WaitForTableExists(table, allNodes, 15*time.Second); err != nil { - harness.dumpNodeLogs("f2_ddl") - t.Fatalf("DDL did not replicate: %v", err) - } - - if _, err := harness.ExecNode(1, fmt.Sprintf( - "INSERT INTO %s (id, v) VALUES (1, 'before')", table)); err != nil { - t.Fatalf("initial INSERT: %v", err) + c.createTable(1, "marmot", table, "CREATE TABLE "+table+" (id INT PRIMARY KEY, v TEXT)") + c.mustExec(1, "marmot", "INSERT INTO "+table+" (id, v) VALUES (1, 'before')") + c.waitRows("marmot", "SELECT id, v FROM "+table+" ORDER BY id", []string{"1|before"}, 1, 2, 3) + + // Node 3 dies abruptly, then its peers go away: it comes back knowing + // nothing and reaching no one. + c.kill(3) + c.kill(1) + c.kill(2) + c.startNode(3) + c.waitMySQL(3) + _, err := c.exec(3, "marmot", "INSERT INTO "+table+" (id, v) VALUES (2, 'alone')") + if code := mysqlCode(err); !slices.Contains([]uint16{mysqlcode.ErrCodeUnknown, mysqlcode.ErrCodeLockTimeout}, code) { + t.Fatalf("a lone restarted node answered the write with %v, want refusal 1105 or 1205", err) } - if err := harness.WaitForRowCount(table, allNodes, 1, 15*time.Second); err != nil { - harness.dumpNodeLogs("f2_initial") - t.Fatalf("initial row did not replicate: %v", err) - } - - // Node 3 dies abruptly, then its peers go away. This is the window the - // defect lived in: node 3 comes back knowing nothing and reaching no one. - if err := harness.KillNode(3); err != nil { - t.Fatalf("KillNode(3): %v", err) - } - if err := harness.StopNode(1); err != nil { - t.Fatalf("StopNode(1): %v", err) - } - if err := harness.StopNode(2); err != nil { - t.Fatalf("StopNode(2): %v", err) - } - - if err := harness.StartNode(3); err != nil { - t.Fatalf("StartNode(3): %v", err) - } - if err := harness.WaitForAlive(3, 30*time.Second); err != nil { - harness.dumpNodeLogs("f2_restart") - t.Fatalf("node 3 did not come up alone: %v", err) - } - - // The invariant. Whichever mechanism fires - a restored denominator of 3 - // that cannot be met, or the seed-node belt - the write must be refused. - _, err := harness.ExecNode(3, fmt.Sprintf( - "INSERT INTO %s (id, v) VALUES (2, 'alone')", table)) - if err == nil { - harness.dumpNodeLogs("f2_lone_accepted") - t.Fatal("a lone restarted node accepted a write: it acted as its own quorum") - } - t.Logf("lone node refused the write, as required: %v", err) - - // And it did not write locally either. An error returned after a local - // commit would still have diverged from the peers that were down. // Mutation: refuse only after applying locally. - if got := harness.getRowCount(3, table); got != 1 { - t.Errorf("row count on the lone node = %d, want 1; the refused write must not have been applied", got) - } - - // Once the cluster is back, the same statement must succeed and replicate: - // the refusal is transient, not a node that has wedged itself. - // Mutation: make the refusal permanent (e.g. latch it); this fires. - if err := harness.StartNode(1); err != nil { - t.Fatalf("StartNode(1): %v", err) - } - if err := harness.StartNode(2); err != nil { - t.Fatalf("StartNode(2): %v", err) - } - if err := harness.WaitForClusterConvergence(60 * time.Second); err != nil { - harness.dumpNodeLogs("f2_reconverge") - t.Fatalf("cluster did not reconverge: %v", err) - } - - var insertErr error - deadline := time.Now().Add(30 * time.Second) - for time.Now().Before(deadline) { - _, insertErr = harness.ExecNode(3, fmt.Sprintf( - "INSERT INTO %s (id, v) VALUES (2, 'after')", table)) - if insertErr == nil { - break - } - time.Sleep(time.Second) - } - if insertErr != nil { - harness.dumpNodeLogs("f2_after_reconverge") - t.Fatalf("INSERT still refused after the cluster reconverged: %v", insertErr) - } - if err := harness.WaitForRowCount(table, allNodes, 2, 30*time.Second); err != nil { - harness.dumpNodeLogs("f2_after_replicate") - t.Fatalf("the write did not replicate after reconvergence: %v", err) - } + c.waitRows("marmot", "SELECT id, v FROM "+table+" ORDER BY id", []string{"1|before"}, 3) + + // Mutation: make the refusal permanent (latch it). The write below fails. + c.startNode(1) + c.startNode(2) + c.waitReady() + c.execAcrossReconnect(3, "marmot", "INSERT INTO "+table+" (id, v) VALUES (2, 'after')") + c.waitRows("marmot", "SELECT id, v FROM "+table+" ORDER BY id", []string{"1|before", "2|after"}, 1, 2, 3) } -// TestFreshNodeWithUnreachableSeedsRefusesWrites covers the belt on its own: a -// node that has never known any membership, so there is no snapshot to restore, -// and whose seeds are down. Its membership is genuinely one, and the only thing -// distinguishing it from a legitimate single-node deployment is that it was -// configured with seeds. +// TestFreshNodeWithUnreachableSeedsRefusesWrites covers the belt on its own: +// a node that has never known any membership, so there is none to restore, +// and whose seeds are down. Only its configured seeds tell it apart from a +// single-node deployment; it must refuse writes with the retryable 1205, as +// the seeds may come up at any moment. // -// Mutation: drop the HasSeedNodes() condition and this passes for the wrong -// reason; drop the belt entirely and the write is accepted. +// Mutations: drop the belt - the write is accepted; return a non-retryable +// code from ErrMembershipNotLearned - the code check fires. func TestFreshNodeWithUnreachableSeedsRefusesWrites(t *testing.T) { - harness := NewClusterHarness(t) - defer harness.Cleanup() - - // Start only node 2, whose configured seeds (node 1) are never started. - if err := harness.StartNode(2); err != nil { - t.Fatalf("StartNode(2): %v", err) - } - if err := harness.WaitForAlive(2, 30*time.Second); err != nil { - harness.dumpNodeLogs("f2_fresh_boot") - t.Fatalf("node 2 did not come up: %v", err) - } - - _, err := harness.ExecNode(2, "CREATE TABLE f2_fresh (id INT PRIMARY KEY, v TEXT)") - if err == nil { - // A DDL is a quorum write too; if it is accepted the belt did not fire. - harness.dumpNodeLogs("f2_fresh_accepted") - t.Fatal("a fresh node with unreachable seeds accepted a write") - } - - // The client must be told to retry: the seeds may come up at any moment. - // Mutation: return a non-retryable code from ErrMembershipNotLearned. - if !strings.Contains(err.Error(), "1205") && !strings.Contains(strings.ToLower(err.Error()), "membership") { - t.Errorf("error %q names neither the retryable code 1205 nor the membership condition", err) + c := newCluster(t) + // Only node 2 starts; its seed, node 1, never does. + c.startNode(2) + c.waitMySQL(2) + // A DDL is a quorum write too. + _, err := c.exec(2, "marmot", "CREATE TABLE f2_fresh (id INT PRIMARY KEY, v TEXT)") + if code := mysqlCode(err); code != mysqlcode.ErrCodeLockTimeout { + t.Fatalf("a fresh node with unreachable seeds answered a write with %v, want 1205", err) } - t.Logf("fresh node with unreachable seeds refused the write: %v", err) } diff --git a/test/rolling_upgrade_cluster_test.go b/test/rolling_upgrade_cluster_test.go index 7e9f5557..b0deae66 100644 --- a/test/rolling_upgrade_cluster_test.go +++ b/test/rolling_upgrade_cluster_test.go @@ -1,216 +1,102 @@ package test import ( - "errors" "fmt" - "os" - "os/exec" - "path/filepath" "strings" - "sync" "testing" - "time" - "github.com/go-sql-driver/mysql" "github.com/maxpert/marmot/protocol/mysqlcode" ) -// preLogPullCommit is the last commit before nodes pulled each other's commit -// logs: the release a rolling upgrade to the log-pull protocol starts from. -const preLogPullCommit = "7a66596" - -// upgradeGossipDeadline bounds each wait for gossip to carry a node's new log -// protocol version to its peers after that node restarts. -const upgradeGossipDeadline = 20 * time.Second - -var ( - preLogPullBinOnce sync.Once - preLogPullBinPath string - preLogPullBinErr error -) - -// preLogPullBinary builds the preLogPullCommit binary once per test process, from a -// read-only `git archive` of that commit, and reuses a binary a previous run -// left under testDataRoot. Call it before NewClusterHarness: the build is not -// part of the cluster's time budget. -func preLogPullBinary(t *testing.T) string { - t.Helper() - preLogPullBinOnce.Do(func() { - dir := filepath.Join(testDataRoot, "marmot_pref2_"+preLogPullCommit) - bin := filepath.Join(dir, "marmot") - if _, err := os.Stat(bin); err == nil { - preLogPullBinPath = bin - return - } - src := filepath.Join(dir, "src") - if err := os.MkdirAll(src, 0o755); err != nil { - preLogPullBinErr = err - return - } - archive := exec.Command("sh", "-c", fmt.Sprintf("git archive %s | tar -x -C %s", preLogPullCommit, src)) - archive.Dir = repoRoot() - if out, err := archive.CombinedOutput(); err != nil { - preLogPullBinErr = fmt.Errorf("git archive %s: %v\n%s", preLogPullCommit, err, out) - return - } - build := exec.Command("go", "build", "-tags", "sqlite_preupdate_hook sqlite_fts5 sqlite_json sqlite_math_functions sqlite_foreign_keys sqlite_stat4 sqlite_vacuum_incr", "-o", bin, ".") - build.Dir = src - if out, err := build.CombinedOutput(); err != nil { - preLogPullBinErr = fmt.Errorf("build %s: %v\n%s", preLogPullCommit, err, out) - return - } - preLogPullBinPath = bin - }) - if preLogPullBinErr != nil { - t.Fatalf("pre-log-pull binary: %v", preLogPullBinErr) - } - return preLogPullBinPath -} - -// restartOnBinary stops node, switches it to bin ("" for the harness's own -// binary) and starts it again, waiting until it answers. -func restartOnBinary(t *testing.T, h *ClusterHarness, node int, bin string) { - t.Helper() - if err := h.StopNode(node); err != nil { - t.Fatal(err) - } - if bin == "" { - delete(h.NodeBin, node) - } else { - h.NodeBin[node] = bin - } - if err := h.StartNode(node); err != nil { - t.Fatal(err) - } - if err := h.WaitForAlive(node, restartDeadline); err != nil { - t.Fatal(err) - } -} - -// ddlRefusedNaming reports whether err is the rolling-upgrade refusal (the -// retryable 1213) naming exactly the members in legacy. -func ddlRefusedNaming(err error, legacy string) bool { - var mysqlErr *mysql.MySQLError - return errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlcode.ErrCodeDeadlock && - strings.Contains(mysqlErr.Message, "node(s) "+legacy+" ") +// restartOnBinary kills node id, switches it to bin ("" for the tree under +// test) and restarts it, waiting until the cluster is ready. +func (c *cluster) restartOnBinary(id int, bin string) { + c.t.Helper() + c.kill(id) + c.node(id).bin = bin + c.restart(id) } -// waitDDLRefusedNaming retries ddl on node until it is refused naming exactly -// legacy (gossip carries each restarted node's protocol version to its peers -// asynchronously), failing after upgradeGossipDeadline. -func waitDDLRefusedNaming(t *testing.T, h *ClusterHarness, node int, ddl, legacy string) { - t.Helper() - deadline := time.Now().Add(upgradeGossipDeadline) - for { - h.Progress() - _, err := execTimed(openTimedNodeDatabase(t, h, node, "marmot"), ddl) - if ddlRefusedNaming(err, legacy) { - return - } +// waitDDLRefusedNaming retries ddl through node id until it is refused with +// the rolling-upgrade refusal - the retryable 1213 - naming exactly the +// members in legacy (gossip carries each restarted node's protocol version +// to its peers asynchronously). It fails at once if the DDL commits. +func (c *cluster) waitDDLRefusedNaming(id int, ddl, legacy string) { + c.t.Helper() + waitFor(c.t, fmt.Sprintf("node %d refuses %q naming %s", id, ddl, legacy), replicationDeadline, func() (bool, string) { + _, err := c.exec(id, "marmot", ddl) if err == nil { - t.Fatalf("node %d accepted %q while node(s) %s still run the older release", node, ddl, legacy) - } - if time.Now().After(deadline) { - t.Fatalf("node %d never refused %q naming %s within %s; last error: %v", node, ddl, legacy, upgradeGossipDeadline, err) + c.t.Fatalf("node %d accepted %q while node(s) %s still run the older release", id, ddl, legacy) } - time.Sleep(convergencePoll) - } -} - -// waitDDLAccepted retries ddl on node until it succeeds, failing after -// upgradeGossipDeadline. -func waitDDLAccepted(t *testing.T, h *ClusterHarness, node int, ddl string) { - t.Helper() - deadline := time.Now().Add(upgradeGossipDeadline) - for { - h.Progress() - _, err := execTimed(openTimedNodeDatabase(t, h, node, "marmot"), ddl) - if err == nil { - return - } - if time.Now().After(deadline) { - t.Fatalf("node %d still refuses %q %s after the upgrade finished: %v", node, ddl, upgradeGossipDeadline, err) - } - time.Sleep(convergencePoll) - } -} - -// insertFromEveryNode inserts one row into marmot.t through each node and -// records it in l. -func insertFromEveryNode(t *testing.T, h *ClusterHarness, l *keyLedger, tag string) { - t.Helper() - for n := 1; n <= numNodes; n++ { - res, err := execTimed(openTimedNodeDatabase(t, h, n, "marmot"), "INSERT INTO t (v) VALUES (?)", fmt.Sprintf("%s-n%d", tag, n)) - if err != nil { - t.Fatalf("%s: INSERT through node %d: %v", tag, n, err) - } - id, err := res.LastInsertId() - if err != nil { - t.Fatal(err) - } - l.acked(fmt.Sprintf("n%d", n), ledgerKey("marmot", "t", id)) - } + refused := mysqlCode(err) == mysqlcode.ErrCodeDeadlock && strings.Contains(err.Error(), "node(s) "+legacy+" ") + return refused, err.Error() + }) } // TestRollingUpgradeRefusesDDLUntilEveryNodeServesLogPull: a rolling upgrade // from the release before the log-pull protocol. A cluster started on -// preLogPullCommit gets nodes 1 and 2 upgraded; while node 3 still runs the old -// binary, DDL is refused cluster-wide with the retryable error naming node 3 -// (through an upgraded coordinator) and does not commit through the old -// coordinator either, while DML keeps replicating. Once node 3 is upgraded +// preLogPullCommit gets nodes 1 and 2 upgraded; while node 3 still runs the +// old binary, DDL is refused cluster-wide with the retryable 1213 naming node +// 3 (through an upgraded coordinator), does not commit through the old +// coordinator either, and DML keeps replicating. Once node 3 is upgraded // too, every node coordinates both writes and DDL, and the schema versions // agree. +// +// The old release has no readiness endpoint of its own, so readiness asks +// every node over gRPC whether its claim votes are held: a narrow insert +// through the old node 3 before its votes are released returns 1205, which +// is how this test used to flake. func TestRollingUpgradeRefusesDDLUntilEveryNodeServesLogPull(t *testing.T) { - oldBin := preLogPullBinary(t) - h := NewClusterHarness(t, func(h *ClusterHarness) { - h.NodeBin = map[int]string{1: oldBin, 2: oldBin, 3: oldBin} - }) - defer h.Cleanup() - tables := map[string][]string{"marmot": {"t"}} - startWithTables(t, h, tables) - h.Phase("start-old-release") - - restartOnBinary(t, h, 1, "") - restartOnBinary(t, h, 2, "") - // The old release started the cluster without the readiness endpoint - // StartCluster waits on; the upgraded nodes answer it. - for _, n := range []int{1, 2} { - if err := h.WaitForVotesReleased(n, upgradeGossipDeadline); err != nil { - t.Fatal(err) - } + c := newCluster(t) + for _, n := range c.nodes { + n.bin = preLogPullBin } - h.Phase("nodes-1-2-upgraded") + c.start() + createAutoIncTables(c, 1, []target{{"marmot", "t"}}) + c.phase("start-old-release") + + c.restartOnBinary(1, "") + c.restartOnBinary(2, "") + c.phase("nodes-1-2-upgraded") - waitDDLRefusedNaming(t, h, 1, "CREATE TABLE mixed1 (id INT PRIMARY KEY)", "[3]") - waitDDLRefusedNaming(t, h, 2, "CREATE TABLE mixed2 (id INT PRIMARY KEY)", "[3]") - if _, err := execTimed(openTimedNodeDatabase(t, h, 3, "marmot"), "CREATE TABLE mixed3 (id INT PRIMARY KEY)"); err == nil { + c.waitDDLRefusedNaming(1, "CREATE TABLE mixed1 (id INT PRIMARY KEY)", "[3]") + c.waitDDLRefusedNaming(2, "CREATE TABLE mixed2 (id INT PRIMARY KEY)", "[3]") + if _, err := c.exec(3, "marmot", "CREATE TABLE mixed3 (id INT PRIMARY KEY)"); err == nil { t.Fatal("the old coordinator's DDL committed although its upgraded participants must refuse it") } - l := newKeyLedger(h) - insertFromEveryNode(t, h, l, "mixed") - waitKeysConverged(t, h, l, tables, convergenceDeadline) - h.Phase("mixed-checked") - - restartOnBinary(t, h, 3, "") - h.Phase("node-3-upgraded") - - for n := 1; n <= numNodes; n++ { - table := fmt.Sprintf("after%d", n) - waitDDLAccepted(t, h, n, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - tables["marmot"] = append(tables["marmot"], table) - for m := 1; m <= numNodes; m++ { - waitTableOn(t, h, m, "marmot", table, upgradeGossipDeadline) - } + l := newRowLedger() + insertFromEveryNode(c, l, "mixed") + l.waitConverged(c, []target{{"marmot", "t"}}) + c.phase("mixed-checked") + + c.restartOnBinary(3, "") + c.phase("node-3-upgraded") + for _, id := range c.all() { + table := fmt.Sprintf("after%d", id) + c.createTable(id, "marmot", table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") } - for n := 1; n <= numNodes; n++ { - for m := 1; m <= numNodes; m++ { - if h.tableExistsOnNode(m, fmt.Sprintf("mixed%d", n)) { - t.Fatalf("DDL refused while mixed left table mixed%d on node %d", n, m) + for _, id := range c.all() { + for _, mixed := range []string{"mixed1", "mixed2", "mixed3"} { + if ok, err := c.hasTable(id, "marmot", mixed); err != nil || ok { + t.Fatalf("node %d: %s present=%v (err %v): a DDL refused while mixed was applied", id, mixed, ok, err) } } } - insertFromEveryNode(t, h, l, "upgraded") - waitKeysConverged(t, h, l, map[string][]string{"marmot": {"t"}}, convergenceDeadline) - waitSchemaVersionsEqual(t, h, []string{"marmot"}, upgradeGossipDeadline) - h.Phase("upgraded-checked") + insertFromEveryNode(c, l, "upgraded") + l.waitConverged(c, []target{{"marmot", "t"}}) + c.waitSchemaVersionsEqual("marmot") +} + +// insertFromEveryNode inserts one generated row into marmot.t through each +// node and records it in l. +func insertFromEveryNode(c *cluster, l *rowLedger, tag string) { + c.t.Helper() + for _, id := range c.all() { + v := fmt.Sprintf("%s-n%d", tag, id) + rowID, err := insertNarrow(c, id, "marmot", "INSERT INTO t (v) VALUES (?)", v) + if err != nil { + c.t.Fatalf("%s: INSERT through node %d: %v", tag, id, err) + } + l.inserted(tag, ledgerKey("marmot", "t", rowID), v) + } } diff --git a/test/row_version_convergence_cluster_test.go b/test/row_version_convergence_cluster_test.go index b0ef6b7a..d8acefed 100644 --- a/test/row_version_convergence_cluster_test.go +++ b/test/row_version_convergence_cluster_test.go @@ -2,145 +2,103 @@ package test // Cluster test for row-value convergence: nodes 1 and 2 keep overwriting the // same rows while node 3 is down and for a moment after it returns, then -// node 3 writes each row once more. Every node must end with the same value -// for every row. The order in which a node meets older and newer images of -// a row is pinned deterministically by grpc's log puller tests; this checks -// the end-to-end outcome. +// node 3 writes each row once more. Every node must end with exactly node +// 3's last value for every row. The order in which a node meets older and +// newer images of a row is pinned deterministically by grpc's log puller +// tests; this checks the end-to-end outcome. import ( - "crypto/sha256" - "encoding/hex" "fmt" - "strings" "sync" + "sync/atomic" "testing" - "time" ) -const ( - // hotRows is how many rows the writers keep overwriting. - hotRows = 10 - // hotOutage is how long node 3 stays down under load. - hotOutage = 10 * time.Second - // hotAfterRestart is how long the load continues once node 3 is back. - hotAfterRestart = 2 * time.Second -) +// hotRows is how many rows the hot writers keep overwriting. +const hotRows = 10 -// tableChecksum hashes every (id, v) of table on node, in id order. -func tableChecksum(t *testing.T, h *ClusterHarness, node int, database, table string) (string, error) { - a, err := queryAnswer(openTimedNodeDatabase(t, h, node, database), "SELECT id, v FROM "+table+" ORDER BY id") - if err != nil { - return "", fmt.Errorf("node %d: %v", node, err) - } - sum := sha256.New() - for _, r := range a.rows { - fmt.Fprintf(sum, "%s=%s\n", r[0], r[1]) - } - return hex.EncodeToString(sum.Sum(nil))[:16], nil -} - -// waitChecksumsEqual polls until table's checksum is identical on every -// node, within deadline. -func waitChecksumsEqual(t *testing.T, h *ClusterHarness, database, table string, deadline time.Duration) { - t.Helper() - start := time.Now() - for { - h.Progress() - sums := make([]string, 0, numNodes) - for node := 1; node <= numNodes; node++ { - sum, err := tableChecksum(t, h, node, database, table) - if err != nil { - sum = err.Error() - } - sums = append(sums, sum) - } - equal := true - for _, s := range sums[1:] { - equal = equal && s == sums[0] - } - if equal { - t.Logf("checksums equal on every node %s after load stopped", time.Since(start).Round(time.Millisecond)) - return - } - if time.Since(start) > deadline { - h.dumpNodeLogs("cluster_" + strings.ReplaceAll(t.Name(), "/", "_")) - t.Fatalf("%s.%s checksums still differ %s after load stopped: %v", database, table, deadline, sums) - } - time.Sleep(convergencePoll) - } +// hotWriters counts the ACKed updates of runHotWriters and keeps the last +// failure, for a wait that times out to print. +type hotWriters struct { + acked atomic.Int64 + lastErr atomic.Value } // runHotWriters has each node in nodes overwrite rows 1..hotRows in turn -// until stop closes. A statement that fails is retried on the next turn. -func runHotWriters(t *testing.T, h *ClusterHarness, database, table string, nodes []int, stop <-chan struct{}) *sync.WaitGroup { +// until the returned stop is called, counting ACKed updates in w. +func runHotWriters(c *cluster, database, table string, nodes []int, w *hotWriters) (stop func()) { + done := make(chan struct{}) var wg sync.WaitGroup for _, n := range nodes { wg.Add(1) go func(n int) { defer wg.Done() - conn := openTimedNodeDatabase(t, h, n, database) for i := 0; ; i++ { select { - case <-stop: + case <-done: return default: } - id := i%hotRows + 1 - if _, err := execTimed(conn, "UPDATE "+table+" SET v = ? WHERE id = ?", fmt.Sprintf("n%d#%d", n, i), id); err != nil { - time.Sleep(convergencePoll) - continue + q := "UPDATE " + table + " SET v = ? WHERE id = ?" + if _, ok, lastErr := writeRetrying(c, n, database, done, q, fmt.Sprintf("n%d#%d", n, i), i%hotRows+1); ok { + w.acked.Add(1) + } else if lastErr != "" { + w.lastErr.Store(fmt.Sprintf("node %d: %s", n, lastErr)) } - h.Progress() } }(n) } - return &wg + return func() { + close(done) + wg.Wait() + } } +// waitAcked waits until w's ACKed updates have grown by n. +func waitAcked(c *cluster, w *hotWriters, n int64, phase string) { + c.t.Helper() + want := w.acked.Load() + n + waitFor(c.t, fmt.Sprintf("%s: %d ACKed updates", phase, want), replicationDeadline, func() (bool, string) { + c.progress() + return w.acked.Load() >= want, fmt.Sprintf("%d ACKed, last error: %v", w.acked.Load(), w.lastErr.Load()) + }) +} + +// TestSameRowsOverwrittenAcrossOutageConvergeByValue: rows overwritten on +// nodes 1 and 2 while node 3 is down, and just after it returns, then +// written once more by node 3 before it may have pulled what it missed, end +// with node 3's last value on every node. func TestSameRowsOverwrittenAcrossOutageConvergeByValue(t *testing.T) { - h := NewClusterHarness(t) - defer h.Cleanup() const database, table = "hotrows", "t" - startWithTables(t, h, map[string][]string{database: {table}}) - conn := openTimedNodeDatabase(t, h, 1, database) - for id := 1; id <= hotRows; id++ { - if _, err := execTimed(conn, "INSERT INTO "+table+" (id, v) VALUES (?, 'seed')", id); err != nil { - t.Fatalf("seed row %d: %v", id, err) - } - } - waitChecksumsEqual(t, h, database, table, convergenceDeadline) - h.Phase("seeded") - - if err := h.StopNode(3); err != nil { - t.Fatal(err) - } - h.Phase("node3-stopped") - stop := make(chan struct{}) - wg := runHotWriters(t, h, database, table, []int{1, 2}, stop) - sleepWithProgress(h, hotOutage) + // Two writers overwriting the same rows block on each other's intents; a + // short lock wait turns each such conflict into a quick retryable 1205. + c := newCluster(t, func(cfg *clusterConfig) { cfg.lockWaitTimeoutSecs = 1 }) + c.start() + c.createDatabase(1, database) + c.createTable(1, database, table, "CREATE TABLE "+table+" (id INT PRIMARY KEY, v TEXT)") + values, _ := idValueRows(1, hotRows, "seed") + c.mustExec(1, database, "INSERT INTO "+table+" (id, v) VALUES "+values) + c.waitSameRows(database, "SELECT id, v FROM "+table+" ORDER BY id", 1, 2, 3) - if err := h.StartNode(3); err != nil { - t.Fatal(err) - } - if err := h.WaitForAlive(3, restartDeadline); err != nil { - t.Fatal(err) - } - h.Phase("node3-alive") - sleepWithProgress(h, hotAfterRestart) - close(stop) - wg.Wait() - h.Phase("load-stopped") + c.phase("seeded") + c.kill(3) + var w hotWriters + stop := runHotWriters(c, database, table, []int{1, 2}, &w) + waitAcked(c, &w, 3*ledgerStep, "while node 3 is down") + c.phase("outage-load") + c.startNode(3) + c.waitMySQL(3) + c.phase("restart") + waitAcked(c, &w, ledgerStep, "after node 3 returned") + stop() + c.phase("after-load") - // One last live write per row, coordinated by node 3 itself, while the - // writes it missed during the outage may still be waiting to be pulled. - conn3 := openTimedNodeDatabase(t, h, 3, database) + var want []string for id := 1; id <= hotRows; id++ { - if _, err := execTimed(conn3, "UPDATE "+table+" SET v = ? WHERE id = ?", fmt.Sprintf("final#%d", id), id); err != nil { - t.Fatalf("final update of row %d: %v", id, err) - } + c.mustExec(3, database, "UPDATE "+table+" SET v = ? WHERE id = ?", fmt.Sprintf("final#%d", id), id) + want = append(want, fmt.Sprintf("%d|final#%d", id, id)) } - h.Phase("final-writes") - - waitChecksumsEqual(t, h, database, table, convergenceDeadline) - h.Phase("converged") + c.phase("final-writes") + c.waitRows(database, "SELECT id, v FROM "+table+" ORDER BY id", want, 1, 2, 3) + c.phase("converged") } diff --git a/test/schema_version_kill_cluster_test.go b/test/schema_version_kill_cluster_test.go index ca5a5432..e6a48a07 100644 --- a/test/schema_version_kill_cluster_test.go +++ b/test/schema_version_kill_cluster_test.go @@ -1,36 +1,23 @@ package test -import ( - "testing" - "time" -) +import "testing" -// TestSchemaVersionSurvivesKillNine: a node applies DDL, is killed with -// SIGKILL and restarts with its tables intact. It must also keep its schema -// version: a node reporting a lower version than the schema it runs refuses -// every peer transaction that requires the newer one, and nothing but a -// snapshot restore heals it. With node 3 stopped, node 2's write needs node -// 1's vote, so it commits only if node 1 kept its version. +// TestSchemaVersionSurvivesKillNine: a node applies DDL, is SIGKILLed and +// restarts with its tables intact. It must also keep its schema version: a +// node reporting a lower version than the schema it runs refuses every peer +// transaction that requires the newer one, and nothing but a snapshot +// restore heals it. With node 3 killed, node 2's write needs node 1's vote, +// so it commits only if node 1 kept its version. // // Mutation: write the schema version with pebble.NoSync -// (PebbleMetaStore.UpdateSchemaVersion). "node 2 cannot commit a replicated -// write" fires: node 1 restarts at version 0. +// (PebbleMetaStore.UpdateSchemaVersion). Node 1 restarts at version 0 and +// node 2's write never commits. func TestSchemaVersionSurvivesKillNine(t *testing.T) { - harness := startNarrowCluster(t, "sv", "CREATE TABLE sv (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - defer harness.Cleanup() - createQuorumProbe(t, harness) - - if err := harness.StopNode(1); err != nil { - t.Fatalf("StopNode(1): %v", err) - } - if err := harness.StartNode(1); err != nil { - t.Fatalf("StartNode(1): %v", err) - } - if err := harness.WaitForAlive(1, 90*time.Second); err != nil { - t.Fatalf("node 1 did not come back: %v", err) - } - if err := harness.StopNode(3); err != nil { - t.Fatalf("StopNode(3): %v", err) - } - waitForQuorumWrites(t, harness, 60*time.Second, 2) + c := startNarrowCluster(t, "sv", "CREATE TABLE sv (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + c.createTable(1, "marmot", "probe", "CREATE TABLE probe (k INT PRIMARY KEY, v TEXT)") + c.kill(1) + c.restart(1) + c.kill(3) + c.execAcrossReconnect(2, "marmot", "REPLACE INTO probe (k, v) VALUES (2, 'after-kill')") + c.waitRows("marmot", "SELECT k, v FROM probe", []string{"2|after-kill"}, 1, 2) } diff --git a/test/setup_test.go b/test/setup_test.go deleted file mode 100644 index fff148ef..00000000 --- a/test/setup_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package test - -import ( - "fmt" - "os" - "testing" - - "github.com/maxpert/marmot/protocol" -) - -// TestMain initializes the query pipeline before running any tests -func TestMain(m *testing.M) { - if os.Getenv("MARMOT_RUN_CLUSTER_INTEGRATION_TESTS") != "1" { - fmt.Fprintln(os.Stderr, "skipping external cluster integration tests; set MARMOT_RUN_CLUSTER_INTEGRATION_TESTS=1 to run") - os.Exit(0) - } - - // Initialize pipeline with default test values (nil ID generator for tests) - if err := protocol.InitializePipeline(10000, nil); err != nil { - panic("Failed to initialize query pipeline for tests: " + err.Error()) - } - - // Run all tests - code := m.Run() - - // Exit with test result code - os.Exit(code) -} diff --git a/test/verification_test.go b/test/verification_test.go index 96d80512..22b067bc 100644 --- a/test/verification_test.go +++ b/test/verification_test.go @@ -128,7 +128,7 @@ func (tdm *TestDatabaseManager) RegistryKeyGeneration(name string) (uint64, bool func TestMySQLServerIntegration(t *testing.T) { // Setup temporary DB with MetaStore - tmpDir := t.TempDir() + tmpDir := testDir(t) dbPath := tmpDir + "/test.db" // Create MetaStore using factory (PebbleDB) @@ -158,15 +158,20 @@ func TestMySQLServerIntegration(t *testing.T) { handler := coordinator.NewCoordinatorHandler(1, writeCoord, readCoord, clock, dbMgr, nil, nil, nil) // Setup server on an ephemeral port to avoid collisions with other tests. - port := findFreeTCPPort(t) + port := freePorts(t, 1)[0] server := protocol.NewMySQLServer(fmt.Sprintf("127.0.0.1:%d", port), "", 0, handler) err = server.Start() require.NoError(t, err) defer server.Stop() - - // Give server time to start - time.Sleep(100 * time.Millisecond) + waitFor(t, "MySQL server listening", time.Second, func() (bool, string) { + conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), clientTimeout) + if err != nil { + return false, err.Error() + } + conn.Close() + return true, "" + }) // Connect with MySQL client cfg := mysql.Config{ @@ -219,11 +224,3 @@ func TestMySQLServerIntegration(t *testing.T) { // We can't easily verify internal snapshot state here without exposing it, // but success means the read path is working. } - -func findFreeTCPPort(t *testing.T) int { - t.Helper() - l, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - defer l.Close() - return l.Addr().(*net.TCPAddr).Port -} diff --git a/test/write_ledger_convergence_cluster_test.go b/test/write_ledger_convergence_cluster_test.go index a05ff8ab..2c6b74ee 100644 --- a/test/write_ledger_convergence_cluster_test.go +++ b/test/write_ledger_convergence_cluster_test.go @@ -2,243 +2,265 @@ package test import ( "fmt" - "sort" + "slices" "strings" "sync" "testing" "time" ) -// convergenceDeadline bounds how long after the write load stops every node -// must hold every ACKed row: the harness's 5 s anti-entropy interval, a -// second round for a pair held back by a transaction still settling, and -// margin for the per-pair pulls themselves. -const convergenceDeadline = 30 * time.Second - const ( - // loadWindow is how long the ClaimsUnderDDL load runs. - loadWindow = 20 * time.Second - // rowLedgerStep is how many further ACKed rows each phase of the - // kill-and-restart test waits for, within phaseDeadline. - rowLedgerStep = 200 - phaseDeadline = 20 * time.Second - // restartDeadline bounds the restarted node's return. - restartDeadline = 30 * time.Second - // convergencePoll is the polling interval of every wait here. - convergencePoll = 250 * time.Millisecond + // ledgerStep is how many further ACKed rows each phase of a load test + // waits for (rowLedger.waitForMore). + ledgerStep = 30 + // writeRetryDeadline bounds how long a writer retries one statement + // through its node before it moves on to the next. + writeRetryDeadline = time.Second ) -// rowLedger records every ACKed row as table/id -> value, and reports each -// one to the harness's stall watchdog as progress. +// target is one database.table a writer writes. +type target struct{ database, table string } + +// rowLedger records every ACKed row as "database.table/id" -> value. A row +// whose last write failed with an unknown outcome keeps only its presence +// checked (uncertain). type rowLedger struct { - harness *ClusterHarness - mu sync.Mutex - rows map[string]string - dups []string + mu sync.Mutex + rows map[string]string + uncertain map[string]bool + own map[string][]string + dups []string } -func newRowLedger(harness *ClusterHarness) *rowLedger { - return &rowLedger{harness: harness, rows: map[string]string{}} +func newRowLedger() *rowLedger { + return &rowLedger{rows: map[string]string{}, uncertain: map[string]bool{}, own: map[string][]string{}} } -// size returns the number of ACKed rows recorded. -func (l *rowLedger) size() int { +func ledgerKey(database, table string, id int64) string { + return fmt.Sprintf("%s.%s/%d", database, table, id) +} + +// inserted records a row writer inserted and was ACKed. The same key twice +// (an id generated twice) is recorded as a duplicate. +func (l *rowLedger) inserted(writer, key, v string) { l.mu.Lock() defer l.mu.Unlock() - return len(l.rows) + if prev, dup := l.rows[key]; dup { + l.dups = append(l.dups, fmt.Sprintf("%s: %s and %s", key, prev, v)) + } + l.rows[key] = v + l.own[writer] = append(l.own[writer], key) } -// waitForMore waits until n more rows than now are ACKed, or fails. -func (l *rowLedger) waitForMore(t *testing.T, n int, phase string) { - t.Helper() - want := l.size() + n - deadline := time.Now().Add(phaseDeadline) - for l.size() < want { - if time.Now().After(deadline) { - t.Fatalf("%s: only %d of %d ACKed rows within %s", phase, l.size(), want, phaseDeadline) - } - time.Sleep(convergencePoll) +// updated records the outcome of an UPDATE of key to v: ACKed sets the +// value; a failure leaves the row's value unknown. +func (l *rowLedger) updated(key, v string, acked bool) { + l.mu.Lock() + defer l.mu.Unlock() + if acked { + l.rows[key] = v + delete(l.uncertain, key) + return } + l.uncertain[key] = true } -func (l *rowLedger) record(table string, id int64, v string) { - l.harness.Progress() +// size is the number of ACKed rows. +func (l *rowLedger) size() int { + l.mu.Lock() + defer l.mu.Unlock() + return len(l.rows) +} + +// pick returns one of writer's own ACKed keys, spread by i. +func (l *rowLedger) pick(writer string, i int) (string, bool) { l.mu.Lock() defer l.mu.Unlock() - k := fmt.Sprintf("%s/%d", table, id) - if prev, ok := l.rows[k]; ok { - l.dups = append(l.dups, fmt.Sprintf("%s: %s and %s", k, prev, v)) + keys := l.own[writer] + if len(keys) == 0 { + return "", false } - l.rows[k] = v + return keys[(i*7919)%len(keys)], true } -// missingOn returns the ledger rows node does not hold with their ACKed value. -func (l *rowLedger) missingOn(t *testing.T, harness *ClusterHarness, node int, tables []string) []string { - t.Helper() - conn := openTimedNodeDatabase(t, harness, node, "marmot") +// waitForMore waits until n more rows than now are ACKed. +func (l *rowLedger) waitForMore(c *cluster, n int, phase string) { + c.t.Helper() + want := l.size() + n + waitFor(c.t, fmt.Sprintf("%s: %d ACKed rows", phase, want), replicationDeadline, func() (bool, string) { + c.progress() + return l.size() >= want, fmt.Sprintf("%d ACKed", l.size()) + }) +} + +// mismatchesOn lists every ACKed row node id does not hold with its ACKed +// value (only its presence, for an uncertain row). +func (l *rowLedger) mismatchesOn(c *cluster, id int, targets []target) []string { got := map[string]string{} - for _, table := range tables { - a, err := queryAnswer(conn, "SELECT id, v FROM "+table) + for _, tg := range targets { + rows, err := c.rows(id, tg.database, "SELECT id, v FROM "+tg.table) if err != nil { - return []string{fmt.Sprintf("node %d %s unreadable: %v", node, table, err)} + return []string{fmt.Sprintf("node %d %s.%s: %v", id, tg.database, tg.table, err)} } - for _, r := range a.rows { - got[table+"/"+r[0]] = r[1] + for _, r := range rows { + idText, v, _ := strings.Cut(r, "|") + got[tg.database+"."+tg.table+"/"+idText] = v } } l.mu.Lock() defer l.mu.Unlock() - var missing []string - for k, v := range l.rows { - if got[k] != v { - missing = append(missing, fmt.Sprintf("node %d %s want %q got %q", node, k, v, got[k])) + var bad []string + for key, v := range l.rows { + gotV, present := got[key] + if !present || (!l.uncertain[key] && gotV != v) { + bad = append(bad, fmt.Sprintf("node %d %s: want %q, got %q (present %v)", id, key, v, gotV, present)) } } - sort.Strings(missing) - return missing + slices.Sort(bad) + return bad } -// waitLedgerConverged polls until every node holds every ACKed row or -// convergenceDeadline passes, and fails with the rows still missing. -func waitLedgerConverged(t *testing.T, harness *ClusterHarness, ledger *rowLedger, tables []string) { - t.Helper() - start := time.Now() - for { - var missing []string - for node := 1; node <= numNodes; node++ { - missing = append(missing, ledger.missingOn(t, harness, node, tables)...) - } - if len(missing) == 0 { - t.Logf("converged: %d ACKed rows on every node %s after load stopped", len(ledger.rows), time.Since(start).Round(time.Millisecond)) - return - } - if time.Since(start) > convergenceDeadline { - if len(missing) > 20 { - missing = append(missing[:20], fmt.Sprintf("... %d more", len(missing)-20)) - } - t.Fatalf("ACKed rows still missing %s after load stopped:\n%s", convergenceDeadline, strings.Join(missing, "\n")) - } - time.Sleep(convergencePoll) - } -} - -// runLedgerWriters inserts into tables round-robin from every node in nodes until -// stop closes, recording each ACKed row. A node may be down part of the time; -// its failed statements are simply not ACKed. -func runLedgerWriters(t *testing.T, harness *ClusterHarness, ledger *rowLedger, nodes []int, tables []string, stop <-chan struct{}) *sync.WaitGroup { +// runWriters starts one writer per node in nodes: it inserts into targets +// round-robin and, with updates, makes every third statement an UPDATE of +// one of its own ACKed rows. A statement that fails (its node down, a quorum +// out of reach) is retried for up to writeRetryDeadline and then given up. +// The returned stop ends every writer and waits for them. +func runWriters(c *cluster, l *rowLedger, nodes []int, targets []target, updates bool) (stop func()) { + done := make(chan struct{}) var wg sync.WaitGroup for _, n := range nodes { wg.Add(1) go func(n int) { defer wg.Done() - conn := openTimedNodeDatabase(t, harness, n, "marmot") + writer := fmt.Sprintf("n%d", n) for i := 0; ; i++ { select { - case <-stop: + case <-done: return default: } - table := tables[i%len(tables)] - v := fmt.Sprintf("n%d#%d", n, i) - res, err := execTimed(conn, "INSERT INTO "+table+" (v) VALUES (?)", v) - if err != nil { - time.Sleep(convergencePoll) + tg, v := targets[i%len(targets)], fmt.Sprintf("n%d#%d", n, i) + if key, ok := l.pick(writer, i); updates && i%3 == 2 && ok { + database, rest, _ := strings.Cut(key, ".") + table, id, _ := strings.Cut(rest, "/") + _, acked, _ := writeRetrying(c, n, database, done, "UPDATE "+table+" SET v = ? WHERE id = "+id, v) + l.updated(key, v, acked) continue } - if id, err := res.LastInsertId(); err == nil { - ledger.record(table, id, v) + if id, acked, _ := writeRetrying(c, n, tg.database, done, "INSERT INTO "+tg.table+" (v) VALUES (?)", v); acked { + l.inserted(writer, ledgerKey(tg.database, tg.table, id), v) } } }(n) } - return &wg + return func() { + close(done) + wg.Wait() + } +} + +// writeRetrying runs q through node n, retrying a failure with a real +// client's jittered backoff (retryBackoff) until it succeeds, +// writeRetryDeadline passes or done closes. It reports the insert id, whether +// the statement was ACKed, and the last error when it was not. +func writeRetrying(c *cluster, n int, database string, done <-chan struct{}, q string, args ...any) (int64, bool, string) { + var id int64 + var acked bool + lastErr, _ := retryBackoff(writeRetryDeadline, func() (bool, string) { + res, err := c.exec(n, database, q, args...) + if err == nil { + acked = true + id, _ = res.LastInsertId() + return true, "" + } + select { + case <-done: + return true, "" + default: + return false, err.Error() + } + }) + return id, acked, lastErr } // TestClaimsUnderDDLConvergesOnEveryNode: narrow inserts on all three nodes // while DDL commits from every node. A node that declines a PREPARE (its -// schema version is briefly behind) misses those transactions; anti-entropy -// must deliver every ACKed row to every node within convergenceDeadline of -// the load stopping. On 7a66596 30-50 rows per table stayed missing on every -// node forever. +// schema version briefly behind) misses those transactions; anti-entropy +// must deliver every ACKed row to every node. On 7a66596 30-50 rows per +// table stayed missing on every node forever. func TestClaimsUnderDDLConvergesOnEveryNode(t *testing.T) { - harness := startNarrowCluster(t, "s1", "CREATE TABLE s1 (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - defer harness.Cleanup() - c1 := openTimedNodeDatabase(t, harness, 1, "marmot") - if _, err := execTimed(c1, "CREATE TABLE s2 (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)"); err != nil { - t.Fatal(err) - } - waitForTableIn(t, harness, "marmot", "s2", 30*time.Second) - harness.Phase("start") - - tables := []string{"s1", "s2"} - ledger := newRowLedger(harness) - stop := make(chan struct{}) - wg := runLedgerWriters(t, harness, ledger, []int{1, 2, 3}, tables, stop) - - deadline := time.Now().Add(loadWindow) - for i := 0; time.Now().Before(deadline); i++ { + c := startNarrowCluster(t, "s1", "CREATE TABLE s1 (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + c.createTable(1, "marmot", "s2", "CREATE TABLE s2 (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + targets := []target{{"marmot", "s1"}, {"marmot", "s2"}} + l := newRowLedger() + stop := runWriters(c, l, []int{1, 2, 3}, targets, false) + const ddlRounds = 12 + committed := 0 + for i := 0; i < ddlRounds; i++ { creator, alterer := i%3+1, (i+1)%3+1 - cc := openTimedNodeDatabase(t, harness, creator, "marmot") - if _, err := execTimed(cc, fmt.Sprintf("CREATE TABLE d%d (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)", i)); err != nil { - t.Logf("create d%d on node %d: %v", i, creator, err) - continue + // A DDL may be refused while claims hold its table's schema (1213, + // retryable); only the rows' convergence is under test. + if _, err := c.exec(creator, "marmot", fmt.Sprintf("CREATE TABLE d%d (id SMALLINT AUTO_INCREMENT PRIMARY KEY, v TEXT)", i)); err == nil { + committed++ } if i > 0 { - ca := openTimedNodeDatabase(t, harness, alterer, "marmot") - if _, err := execTimed(ca, fmt.Sprintf("ALTER TABLE d%d ADD COLUMN c%d INT", i-1, i)); err != nil { - t.Logf("alter d%d on node %d: %v", i-1, alterer, err) + if _, err := c.exec(alterer, "marmot", fmt.Sprintf("ALTER TABLE d%d ADD COLUMN c%d INT", i-1, i)); err == nil { + committed++ } } + l.waitForMore(c, 5, fmt.Sprintf("DDL round %d", i)) } - close(stop) - wg.Wait() - harness.Phase("load-stopped") - - if len(ledger.dups) > 0 { - t.Fatalf("duplicate ids: %v", ledger.dups) + stop() + if committed == 0 { + t.Fatal("no DDL committed during the load") } - if len(ledger.rows) == 0 { - t.Fatal("no row was ACKed") - } - waitLedgerConverged(t, harness, ledger, tables) - harness.Phase("converged") + t.Logf("%d DDL committed, %d rows ACKed", committed, l.size()) + l.waitConverged(c, targets) } -// TestKilledNodeConvergesAfterRestart: a node SIGKILLed under write load -// and restarted converges to every row the cluster ACKed, whether it -// coordinated the row, missed it while down, or held it only in its own log. +// TestKilledNodeConvergesAfterRestart: a node SIGKILLed under write load and +// restarted converges to every row the cluster ACKed, whether it coordinated +// the row, missed it while down, or held it only in its own log. func TestKilledNodeConvergesAfterRestart(t *testing.T) { - harness := startNarrowCluster(t, "kr", "CREATE TABLE kr (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") - defer harness.Cleanup() - - tables := []string{"kr"} - ledger := newRowLedger(harness) - stop := make(chan struct{}) - wg := runLedgerWriters(t, harness, ledger, []int{1, 2, 3}, tables, stop) - harness.Phase("start") - - ledger.waitForMore(t, rowLedgerStep, "before the kill") - harness.Phase("load") - if err := harness.KillNode(3); err != nil { - t.Fatalf("kill node 3: %v", err) - } - harness.Phase("node3-killed") - ledger.waitForMore(t, rowLedgerStep, "while node 3 is down") - if err := harness.StartNode(3); err != nil { - t.Fatalf("restart node 3: %v", err) + c := startNarrowCluster(t, "kr", "CREATE TABLE kr (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + targets := []target{{"marmot", "kr"}} + l := newRowLedger() + stop := runWriters(c, l, []int{1, 2, 3}, targets, false) + l.waitForMore(c, ledgerStep, "before the kill") + c.kill(3) + l.waitForMore(c, ledgerStep, "while node 3 is down") + c.restart(3) + l.waitForMore(c, ledgerStep, "after node 3 returned") + stop() + l.waitConverged(c, targets) +} + +// convergenceDeadline bounds how long after the load stops every node must +// hold every ACKed row: a few anti-entropy rounds. +const convergenceDeadline = 5 * time.Second + +// waitConverged waits until every node holds every ACKed row with its ACKed +// value and all nodes hold the same rows in every target, and fails on any +// id generated twice. +func (l *rowLedger) waitConverged(c *cluster, targets []target) { + c.t.Helper() + if len(l.dups) > 0 { + c.t.Fatalf("duplicate ids: %v", l.dups) } - if err := harness.WaitForAlive(3, restartDeadline); err != nil { - t.Fatalf("node 3 did not come back: %v", err) + if l.size() == 0 { + c.t.Fatal("no row was ACKed") } - harness.Phase("node3-alive") - ledger.waitForMore(t, rowLedgerStep, "after node 3 returned") - close(stop) - wg.Wait() - harness.Phase("load-stopped") - - if len(ledger.dups) > 0 { - t.Fatalf("duplicate ids: %v", ledger.dups) + waitFor(c.t, fmt.Sprintf("%d ACKed rows on every node", l.size()), convergenceDeadline, func() (bool, string) { + var bad []string + for _, id := range c.all() { + bad = append(bad, l.mismatchesOn(c, id, targets)...) + } + if len(bad) > 0 { + return false, fmt.Sprintf("%d mismatches, first: %s", len(bad), strings.Join(bad[:min(len(bad), 5)], "; ")) + } + return true, "" + }) + for _, tg := range targets { + c.waitSameRows(tg.database, "SELECT id, v FROM "+tg.table+" ORDER BY id", c.all()...) } - waitLedgerConverged(t, harness, ledger, tables) - harness.Phase("converged") } From 718e334aefc221a67bec6cf91b8e6ecc3cec7d0d Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Tue, 29 Sep 2026 19:26:06 -0500 Subject: [PATCH 11/17] fix(autoinc): a narrow insert waits out the startup vote hold instead of failing at once A client that wrote to a narrow AUTO_INCREMENT table through a node that had just started, before it released its claim votes, got 1205 "lock wait timeout" at once: the node's own prepare declined the claim and the claim path gave up without waiting. An application started together with the cluster failed on its first insert. - A held node's own prepare decline is now typed (ErrLocalVotesHeld). The claim waits for the release, signalled through common.Broadcast, bounded by transaction.lock_wait_timeout_seconds, and returns 1205 only when that expires. - A peer turning ALIVE triggers an immediate merge round, so the hold ends as soon as enough members answer instead of on the next interval. - A narrow insert inside a transaction that already holds a pinned writer still fails fast, as before. Waiting there deadlocks with a joiner's snapshot checkpoint, which needs that writer. An insert that is the first write of its transaction, the common ORM shape, holds no writer and waits. - The in-process test harness now injects ids as a node does. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- common/broadcast.go | 30 +++ common/broadcast_test.go | 82 +++++++ coordinator/autoinc_claim.go | 80 ++++++- coordinator/autoinc_claim_test.go | 72 +++++++ coordinator/errors.go | 7 + coordinator/handler.go | 55 +++-- coordinator/write_coordinator.go | 38 +++- db/autoinc_vote_hold.go | 31 ++- db/database_manager.go | 2 + db/replication_engine.go | 7 +- docs/src/pages/compatibility.mdx | 7 +- grpc/autoinc_merge.go | 14 +- grpc/autoinc_merge_test.go | 52 +++++ grpc/node_registry.go | 11 + grpc/node_registry_test.go | 32 +++ id/range_allocator.go | 7 + marmot.go | 2 + test/autoinc_claim_inprocess_test.go | 1 + test/autoinc_hold_wait_cluster_test.go | 128 +++++++++++ test/autoinc_hold_wait_inprocess_test.go | 264 +++++++++++++++++++++++ test/autoinc_membership_cluster_test.go | 15 +- test/autoinc_narrow_cluster_test.go | 7 +- test/main_test.go | 5 +- 23 files changed, 911 insertions(+), 38 deletions(-) create mode 100644 common/broadcast.go create mode 100644 common/broadcast_test.go create mode 100644 test/autoinc_hold_wait_cluster_test.go create mode 100644 test/autoinc_hold_wait_inprocess_test.go diff --git a/common/broadcast.go b/common/broadcast.go new file mode 100644 index 00000000..37218d7a --- /dev/null +++ b/common/broadcast.go @@ -0,0 +1,30 @@ +package common + +import "sync/atomic" + +// Broadcast wakes every waiter at once, without locks: Next returns a channel +// the next Notify closes. A waiter takes the channel before it checks the +// condition it waits for, so a Notify that follows the check always wakes +// it. The zero value is ready to use. +type Broadcast struct { + ch atomic.Pointer[chan struct{}] +} + +// Next returns the channel the next Notify closes. +func (b *Broadcast) Next() <-chan struct{} { + for { + if ch := b.ch.Load(); ch != nil { + return *ch + } + fresh := make(chan struct{}) + b.ch.CompareAndSwap(nil, &fresh) + } +} + +// Notify wakes every waiter on the current channel and arms a new one. +func (b *Broadcast) Notify() { + fresh := make(chan struct{}) + if old := b.ch.Swap(&fresh); old != nil { + close(*old) + } +} diff --git a/common/broadcast_test.go b/common/broadcast_test.go new file mode 100644 index 00000000..c185a107 --- /dev/null +++ b/common/broadcast_test.go @@ -0,0 +1,82 @@ +package common + +import ( + "sync" + "testing" +) + +func closed(ch <-chan struct{}) bool { + select { + case <-ch: + return true + default: + return false + } +} + +// TestBroadcast_NotifyClosesEarlierNextOnly pins the wake-up contract on the +// zero value: a channel taken before a Notify is closed by it, and one taken +// after it stays open until the next Notify. +// +// Mutation: Notify closing the fresh channel instead of the old one fails +// "before" here and TestBroadcast_NotifyWithoutWaiters. +func TestBroadcast_NotifyClosesEarlierNextOnly(t *testing.T) { + var b Broadcast + before := b.Next() + if closed(before) { + t.Fatal("a channel was closed before any Notify") + } + b.Notify() + if !closed(before) { + t.Fatal("before: Notify did not close a channel taken before it") + } + after := b.Next() + if closed(after) { + t.Fatal("after: a channel taken after the Notify is already closed") + } + b.Notify() + if !closed(after) { + t.Fatal("the second Notify did not close the channel taken after the first") + } +} + +// TestBroadcast_NotifyWithoutWaiters pins that Notify on the zero value, with +// no Next taken, neither panics nor pre-closes the next channel. +func TestBroadcast_NotifyWithoutWaiters(t *testing.T) { + var b Broadcast + b.Notify() + b.Notify() + if closed(b.Next()) { + t.Fatal("a channel taken after every Notify is closed") + } +} + +// TestBroadcast_ConcurrentNextAndNotify runs Next and Notify from many +// goroutines, for -race, and checks that no waiter misses the final Notify. +func TestBroadcast_ConcurrentNextAndNotify(t *testing.T) { + var b Broadcast + const workers, rounds = 8, 1000 + var wg sync.WaitGroup + for range workers { + wg.Add(2) + go func() { + defer wg.Done() + for range rounds { + _ = b.Next() + } + }() + go func() { + defer wg.Done() + for range rounds { + b.Notify() + } + }() + } + wg.Wait() + + last := b.Next() + b.Notify() + if !closed(last) { + t.Fatal("a channel taken after the concurrent phase missed the final Notify") + } +} diff --git a/coordinator/autoinc_claim.go b/coordinator/autoinc_claim.go index b27a9312..ea28f417 100644 --- a/coordinator/autoinc_claim.go +++ b/coordinator/autoinc_claim.go @@ -84,6 +84,16 @@ const ( // fix by itself, so it ends the attempt loop immediately rather than spending // the full budget. // +// A round this node itself declined because its claim votes are held +// (ErrLocalVotesHeld) cannot commit whatever its peers answer: the +// coordinator must participate. ClaimRange waits for the release +// (WriteCoordinator.voteHold) and then retries, without spending an attempt; +// every such wait in one call shares a single deadline of +// transaction.lock_wait_timeout_seconds. ctx bounds it too, so a caller that +// promises the lock wait gives ctx at least the lock wait on top of the +// claim's own rounds (the handler's narrow allocator does). FailFastClaimer +// never waits. +// // The two ways a claim can end are kept apart structurally, because a client // must retry one and must never retry the other: // - exhaustion - size found no room above the base, or a participant @@ -99,9 +109,37 @@ const ( // builds a statement and drives it through WriteTransaction, exactly like // any other 2PC write. func (wc *WriteCoordinator) ClaimRange(ctx context.Context, database, table string, prevBase uint64, size id.RangeSizer) (newBase uint64, granted uint64, err error) { + return wc.claimRange(ctx, database, table, prevBase, size, true) +} + +// FailFastClaimer returns a claimer that claims exactly as ClaimRange does, +// except that it never waits out this node's held votes: a round this node +// declines for them ends the claim at once with 1205. A claimant holding a +// pinned SQLite writer - an explicit transaction that already wrote - claims +// through it: the release it would wait for can need a joining peer's +// snapshot of this node, whose checkpoint waits on that very writer. +func (wc *WriteCoordinator) FailFastClaimer() id.Claimer { + return failFastClaimer{wc: wc} +} + +// failFastClaimer is WriteCoordinator.FailFastClaimer. +type failFastClaimer struct { + wc *WriteCoordinator +} + +// ClaimRange is WriteCoordinator.ClaimRange without the hold wait. +func (c failFastClaimer) ClaimRange(ctx context.Context, database, table string, prevBase uint64, size id.RangeSizer) (uint64, uint64, error) { + return c.wc.claimRange(ctx, database, table, prevBase, size, false) +} + +// claimRange is ClaimRange, waiting out this node's held votes only when +// waitForRelease is set. +func (wc *WriteCoordinator) claimRange(ctx context.Context, database, table string, prevBase uint64, size id.RangeSizer, waitForRelease bool) (newBase uint64, granted uint64, err error) { var lastErr error declinedRounds := 0 - for attempt := 0; attempt < maxClaimAttempts; attempt++ { + holdWait := votesHeldWait{ctx: ctx} + defer holdWait.stop() + for attempt := 0; attempt < maxClaimAttempts; { claimSize, sizeErr := size(prevBase) if sizeErr != nil { return 0, 0, fmt.Errorf("auto-increment claim for %s.%s: %w", database, table, sizeErr) @@ -149,6 +187,17 @@ func (wc *WriteCoordinator) ClaimRange(ctx context.Context, database, table stri } lastErr = attemptErr + // A round this node declined for its held votes is not an attempt: it + // waits for the release, and only the lock wait bounds that. + if waitForRelease && wc.voteHold != nil && errors.Is(attemptErr, ErrLocalVotesHeld) { + if waitErr := holdWait.wait(wc.voteHold); waitErr != nil { + lastErr = fmt.Errorf("%w (waiting for the release: %v)", attemptErr, waitErr) + break + } + continue + } + attempt++ + if exhaustedByParticipant(attemptErr) { return 0, 0, fmt.Errorf("auto-increment claim for %s.%s: %w (%v)", database, table, id.ErrRangeExhausted, attemptErr) } @@ -156,7 +205,7 @@ func (wc *WriteCoordinator) ClaimRange(ctx context.Context, database, table stri prevBase = base continue } - if !declinedByParticipant(attemptErr) || attempt == maxClaimAttempts-1 { + if !declinedByParticipant(attemptErr) || attempt == maxClaimAttempts { break } if err := sleepClaimBackoff(ctx, declinedRounds); err != nil { @@ -220,6 +269,33 @@ func (wc *WriteCoordinator) claimSchemaVersion(database string) (uint64, error) return wc.schemaVersion(database) } +// votesHeldWait bounds every wait one ClaimRange call spends on this node's +// held votes by a single lock-wait deadline, set when the first wait starts: +// the claim is a statement waiting for a lock the release frees, so it waits +// exactly as long as MySQL waits for a row lock. +type votesHeldWait struct { + ctx context.Context + cancel context.CancelFunc +} + +// wait blocks until hold is released or the deadline passes. +func (w *votesHeldWait) wait(hold VoteHold) error { + if w.cancel == nil { + w.ctx, w.cancel = context.WithTimeout(w.ctx, lockWaitTimeout()) + } + if err := w.ctx.Err(); err != nil { + return err + } + return hold.WaitAutoIncVotesReleased(w.ctx) +} + +// stop releases the deadline's timer. +func (w *votesHeldWait) stop() { + if w.cancel != nil { + w.cancel() + } +} + // sleepClaimBackoff waits the k-th jittered claim backoff, or until ctx ends. func sleepClaimBackoff(ctx context.Context, k int) error { ceiling := claimBackoffBase << k diff --git a/coordinator/autoinc_claim_test.go b/coordinator/autoinc_claim_test.go index d37a9810..1178ddbb 100644 --- a/coordinator/autoinc_claim_test.go +++ b/coordinator/autoinc_claim_test.go @@ -386,6 +386,78 @@ func TestClaimRange_QuorumNotAchievedReturns1205WithoutRetrying(t *testing.T) { } } +// countingVoteHold is a VoteHold that is always released and counts waits. +type countingVoteHold struct { + mu sync.Mutex + waits int +} + +func (h *countingVoteHold) WaitAutoIncVotesReleased(context.Context) error { + h.mu.Lock() + defer h.mu.Unlock() + h.waits++ + return nil +} + +// TestClaimRange_WaitsOutLocalVotesHeld pins the startup shape of a new +// cluster: every node declines while its votes are held, so the round fails +// quorum with this node's decline marked VotesHeld. ClaimRange waits for the +// release after each such round and claims once the node votes, and a held +// round is not an attempt: more held rounds than maxClaimAttempts still end +// in a grant. +// +// Mutations: breaking out on the decline (HEAD's behaviour) returns 1205; +// counting held rounds as attempts returns 1205 after maxClaimAttempts. +func TestClaimRange_WaitsOutLocalVotesHeld(t *testing.T) { + InitTestTelemetry() + + fake := newClaimStepReplicator() + heldRounds := maxClaimAttempts + 2 + for range heldRounds { + fake.push(1, &ReplicationResponse{Error: "votes held", VotesHeld: true}) + fake.push(2, &ReplicationResponse{Error: "votes held"}) + fake.push(3, &ReplicationResponse{Error: "votes held"}) + } + hold := &countingVoteHold{} + wc := NewWriteCoordinator(1, newMockNodeProvider([]uint64{1, 2, 3}), fake, fake, 200*time.Millisecond, hlc.NewClock(1)) + wc.SetVoteHold(hold) + + newBase, granted, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, fixedClaimSize(50)) + if err != nil { + t.Fatalf("ClaimRange: %v", err) + } + if newBase != 100 || granted != 50 { + t.Errorf("claim = (%d, +%d), want (100, +50)", newBase, granted) + } + if hold.waits != heldRounds { + t.Errorf("waits for the release = %d, want %d (one per held round)", hold.waits, heldRounds) + } +} + +// TestClaimRange_RemoteVotesHeldIsNotALocalWait pins that only this node's +// own held votes make ClaimRange wait: a peer's held decline carries no +// VotesHeld flag across the wire and stays an ordinary decline. +// +// Mutation: classify any declined round as held. The claim waits on the +// hold instead of backing off, and waits becomes non-zero. +func TestClaimRange_RemoteVotesHeldIsNotALocalWait(t *testing.T) { + InitTestTelemetry() + + fake := newClaimStepReplicator() + fake.push(2, &ReplicationResponse{Error: "votes held"}) + fake.push(3, &ReplicationResponse{Error: "votes held"}) + hold := &countingVoteHold{} + wc := NewWriteCoordinator(1, newMockNodeProvider([]uint64{1, 2, 3}), fake, fake, 200*time.Millisecond, hlc.NewClock(1)) + wc.SetVoteHold(hold) + + if _, _, err := wc.ClaimRange(context.Background(), "testdb", "orders", 100, fixedClaimSize(50)); err != nil { + t.Fatalf("ClaimRange: %v", err) + } + if hold.waits != 0 { + t.Errorf("waits for the release = %d, want 0: only this node's own hold is waited on", hold.waits) + } +} + // fixedClaimSize is a RangeSizer asking for the same size above any base: // these tests pin the claim protocol, not the allocator's sizing policy. func fixedClaimSize(size uint64) id.RangeSizer { diff --git a/coordinator/errors.go b/coordinator/errors.go index abb2f613..e92ca200 100644 --- a/coordinator/errors.go +++ b/coordinator/errors.go @@ -8,6 +8,13 @@ import ( "github.com/maxpert/marmot/protocol/query/transform" ) +// ErrLocalVotesHeld reports an AUTO_INCREMENT range claim this node declined +// at PREPARE because its own claim votes are held until it merges claim bases +// from its peers. The coordinator must participate in every quorum, so no +// claim it coordinates can commit before the release; ClaimRange waits for it, +// up to transaction.lock_wait_timeout_seconds, and FailFastClaimer does not. +var ErrLocalVotesHeld = errors.New("this node's auto-increment claim votes are held until it merges claim bases from its peers") + // PrepareConflictError represents a write-write conflict detected during the prepare phase type PrepareConflictError struct { NodeID uint64 diff --git a/coordinator/handler.go b/coordinator/handler.go index 388ed7eb..e93d4162 100644 --- a/coordinator/handler.go +++ b/coordinator/handler.go @@ -162,6 +162,12 @@ func getDDLValidationTimeout() time.Duration { // uses for hookDB sessions), applied here for its naturally longer eager // duration instead of a new speculative config value. func pinnedSessionTimeout() time.Duration { + return lockWaitTimeout() +} + +// lockWaitTimeout returns transaction.lock_wait_timeout_seconds, or MySQL's +// innodb_lock_wait_timeout default (50s). +func lockWaitTimeout() time.Duration { if cfg.Config != nil && cfg.Config.Transaction.LockWaitTimeoutSeconds > 0 { return time.Duration(cfg.Config.Transaction.LockWaitTimeoutSeconds) * time.Second } @@ -224,7 +230,10 @@ type CoordinatorHandler struct { // narrowIDs mints ids for AUTO_INCREMENT columns declared narrower than // BIGINT, from ranges this node claims through writeCoord. Nil when the // handler has no write coordinator to claim through. - narrowIDs *id.RangeAllocator + narrowIDs *id.RangeAllocator + // narrowIDsFailFast shares narrowIDs' ranges but never waits out a vote + // hold; statements inside a transaction that pinned a writer use it. + narrowIDsFailFast *id.RangeAllocator recentTxnIDs sync.Map // txn_id -> conn_id for duplicate detection publisherRegistry PublisherRegistry publisherMu sync.RWMutex @@ -251,31 +260,41 @@ type CoordinatorHandler struct { func NewCoordinatorHandler(nodeID uint64, writeCoord *WriteCoordinator, readCoord *ReadCoordinator, clock *hlc.Clock, dbManager DatabaseManager, ddlLockMgr *DDLLockManager, schemaVersionMgr SchemaVersionManager, nodeRegistry NodeRegistry) *CoordinatorHandler { // The narrow allocator claims through this handler's own write // coordinator, so it is built here, beside it, rather than in the shared - // query pipeline. A claim gets the same bound as any other write. - var narrowIDs *id.RangeAllocator + // query pipeline. A claim gets the bound of any other write plus the lock + // wait: a claim on a node whose votes are held waits for their release up + // to the lock wait (ClaimRange), and then still needs its own rounds. + var narrowIDs, narrowIDsFailFast *id.RangeAllocator if writeCoord != nil { - narrowIDs = id.NewRangeAllocator(writeCoord, getWriteTimeout()) + narrowIDs = id.NewRangeAllocator(writeCoord, getWriteTimeout()+lockWaitTimeout()) + narrowIDsFailFast = narrowIDs.Through(writeCoord.FailFastClaimer()) } return &CoordinatorHandler{ - narrowIDs: narrowIDs, - nodeID: nodeID, - writeCoord: writeCoord, - readCoord: readCoord, - clock: clock, - dbManager: dbManager, - ddlLockMgr: ddlLockMgr, - schemaVersionMgr: schemaVersionMgr, - nodeRegistry: nodeRegistry, - metadata: handlers.NewMetadataHandler(dbManager, SystemDatabaseName), + narrowIDs: narrowIDs, + narrowIDsFailFast: narrowIDsFailFast, + nodeID: nodeID, + writeCoord: writeCoord, + readCoord: readCoord, + clock: clock, + dbManager: dbManager, + ddlLockMgr: ddlLockMgr, + schemaVersionMgr: schemaVersionMgr, + nodeRegistry: nodeRegistry, + metadata: handlers.NewMetadataHandler(dbManager, SystemDatabaseName), } } -// narrowAllocator returns the narrow allocator as the interface the parser -// takes, keeping a nil allocator a nil interface. -func (h *CoordinatorHandler) narrowAllocator() rules.NarrowAllocator { +// narrowAllocator returns the narrow allocator for a statement on session as +// the interface the parser takes, keeping a nil allocator a nil interface. +// A session whose explicit transaction has pinned a SQLite writer gets the +// fail-fast one (WriteCoordinator.FailFastClaimer): it must not wait for a +// vote release while holding a writer the release can depend on. +func (h *CoordinatorHandler) narrowAllocator(session *protocol.ConnectionSession) rules.NarrowAllocator { if h.narrowIDs == nil { return nil } + if pinned := h.lookupPinnedState(session.ConnID); pinned != nil && !pinned.isEmpty() { + return h.narrowIDsFailFast + } return h.narrowIDs } @@ -373,7 +392,7 @@ func (h *CoordinatorHandler) HandleQuery(session *protocol.ConnectionSession, sq // Parse with options based on session state stmt := protocol.ParseStatementWithOptions(sql, protocol.ParseOptions{ SchemaLookup: schemaLookup, - NarrowIDs: h.narrowAllocator(), + NarrowIDs: h.narrowAllocator(session), BoundParams: params, SchemaProvider: schemaProvider, SkipTranspilation: !session.TranspilationEnabled, diff --git a/coordinator/write_coordinator.go b/coordinator/write_coordinator.go index 0fe88046..5477a08b 100644 --- a/coordinator/write_coordinator.go +++ b/coordinator/write_coordinator.go @@ -35,6 +35,17 @@ type WriteCoordinator struct { // with a base kept for an incarnation this node has already replaced. // Nil stamps claims with 0, which no participant checks. schemaVersion func(database string) (uint64, error) + // voteHold is this node's claim vote hold, which a claim declined with + // ErrLocalVotesHeld waits out. Nil fails such a claim at once. + voteHold VoteHold +} + +// VoteHold is this node's AUTO_INCREMENT claim vote hold, as a claimant +// coordinating on this node waits it out. +type VoteHold interface { + // WaitAutoIncVotesReleased returns once this node's claim votes are not + // held, or with ctx's error once ctx ends first. + WaitAutoIncVotesReleased(ctx context.Context) error } // SetSchemaVersionSource sets where claims read this node's schema version @@ -43,6 +54,12 @@ func (wc *WriteCoordinator) SetSchemaVersionSource(source func(database string) wc.schemaVersion = source } +// SetVoteHold sets the vote hold claims wait out (WriteCoordinator.voteHold). +// Call it before the coordinator is used. +func (wc *WriteCoordinator) SetVoteHold(hold VoteHold) { + wc.voteHold = hold +} + // Replicator sends replication requests to remote nodes type Replicator interface { ReplicateTransaction(ctx context.Context, nodeID uint64, req *ReplicationRequest) (*ReplicationResponse, error) @@ -141,6 +158,10 @@ type ReplicationResponse struct { // (protocol.ErrAutoIncClaimNotApplicable). Set only by the local // participant. ClaimNotApplicable bool + // VotesHeld reports a PREPARE of an AUTO_INCREMENT range claim declined + // because the participant's claim votes are held. Set only by the local + // participant. + VotesHeld bool } // NewWriteCoordinator creates a new write coordinator for full database replication @@ -417,6 +438,10 @@ func (wc *WriteCoordinator) runPreparePhase(ctx context.Context, txn *Transactio return nil, &CoordinatorNotParticipatedError{TxnID: txn.ID, Err: localPrepareErr} } + if errors.Is(prepErr, ErrLocalVotesHeld) { + return nil, &CoordinatorNotParticipatedError{TxnID: txn.ID, Err: prepErr} + } + if prepErr != nil { // Write-write conflict detected - abort and signal client to retry telemetry.WriteConflictsTotal.With("intent", "slow").Inc() @@ -791,7 +816,8 @@ type response struct { // any - the caller decides whether it matters, since a remote rejection alone must not // veto a transaction that otherwise reaches quorum), the number of participants that // answered but declined without a verdict (for example a lost race for a row lock), -// and a conflict error if any node reports a conflict. +// and a conflict error if any node reports a conflict - or ErrLocalVotesHeld when this +// node declined a claim because its own votes are held. func (wc *WriteCoordinator) executePreparePhase(ctx context.Context, txn *Transaction, prepReq *ReplicationRequest, otherNodes []uint64, skipLocalReplication bool) (map[uint64]*ReplicationResponse, *RemotePrepareRejectedError, int, error) { @@ -855,6 +881,7 @@ func (wc *WriteCoordinator) executePreparePhase(ctx context.Context, txn *Transa // participant that is furthest ahead. var maxAutoIDStoredBase uint64 declined := 0 + localVotesHeld := false for i := 0; i < totalNodes; i++ { select { @@ -897,6 +924,9 @@ func (wc *WriteCoordinator) executePreparePhase(ctx context.Context, txn *Transa if !r.resp.Rejected { declined++ } + if r.nodeID == wc.nodeID && r.resp.VotesHeld { + localVotesHeld = true + } if r.resp.Rejected { reason := r.resp.Error if r.nodeID == wc.nodeID { @@ -941,6 +971,12 @@ func (wc *WriteCoordinator) executePreparePhase(ctx context.Context, txn *Transa return prepResponses, remoteRejection, declined, localErr } + // This node's own vote is held: nothing any peer answered can let the + // claim commit before the release, so that is the round's answer. + if localVotesHeld { + return prepResponses, remoteRejection, declined, ErrLocalVotesHeld + } + // If any conflict was detected, return the error if conflictErr != nil { return prepResponses, remoteRejection, declined, conflictErr diff --git a/db/autoinc_vote_hold.go b/db/autoinc_vote_hold.go index a3d484fa..6a70aaf1 100644 --- a/db/autoinc_vote_hold.go +++ b/db/autoinc_vote_hold.go @@ -1,6 +1,7 @@ package db import ( + "context" "database/sql" "fmt" "math" @@ -221,9 +222,35 @@ func (dm *DatabaseManager) AutoIncBases() ([]AutoIncBase, bool, error) { } // MergeAutoIncBasesAndReleaseVotes raises this node's merged floors to at -// least bases and releases its vote hold, atomically. +// least bases and releases its vote hold, atomically, then wakes every +// WaitAutoIncVotesReleased. func (dm *DatabaseManager) MergeAutoIncBasesAndReleaseVotes(bases []AutoIncBase) error { - return NewAutoIncClaimStore(dm.systemDB).MergeAndReleaseVotes(bases) + if err := NewAutoIncClaimStore(dm.systemDB).MergeAndReleaseVotes(bases); err != nil { + return err + } + dm.voteRelease.Notify() + return nil +} + +// WaitAutoIncVotesReleased returns once this node's claim votes are not held, +// or with ctx's error once ctx ends first. A claimant coordinating on this +// node waits here: while its own votes are held no claim it coordinates can +// commit. +func (dm *DatabaseManager) WaitAutoIncVotesReleased(ctx context.Context) error { + for { + // Taken before reading the hold, so a release committed after the + // read closes this very channel. + released := dm.voteRelease.Next() + held, err := dm.AutoIncVotesHeld() + if err != nil || !held { + return err + } + select { + case <-released: + case <-ctx.Done(): + return ctx.Err() + } + } } // RaiseAutoIncBases raises this node's merged floors to at least bases, in diff --git a/db/database_manager.go b/db/database_manager.go index b1614d50..fc0dc834 100644 --- a/db/database_manager.go +++ b/db/database_manager.go @@ -15,6 +15,7 @@ import ( "sync/atomic" "time" + "github.com/maxpert/marmot/common" "github.com/maxpert/marmot/coordinator" "github.com/maxpert/marmot/hlc" "github.com/maxpert/marmot/protocol/query/transform" @@ -59,6 +60,7 @@ type DatabaseManager struct { membershipView atomic.Pointer[func() int] // this node's view of total cluster membership gcMembershipFunc atomic.Pointer[func() []uint64] // current member node ids (self included, not REMOVED); source for each database's GC safe position tombstoneMemberCount atomic.Pointer[func() int] // members for the tombstone horizon (SetTombstoneMemberCountFunc) + voteRelease common.Broadcast // wakes WaitAutoIncVotesReleased on every vote release } // SetClusterMembership installs view as the source of this node's view of diff --git a/db/replication_engine.go b/db/replication_engine.go index 9ddbaa9c..937a7f51 100644 --- a/db/replication_engine.go +++ b/db/replication_engine.go @@ -54,6 +54,10 @@ type PrepareResult struct { // the coordinator, local or remote, and without the code every // deterministic refusal reaches the client as 1105 HY000. ErrorCode uint16 + // VotesHeld reports an AUTO_INCREMENT range claim declined because this + // node's claim votes are held (AutoIncHoldTable): a claimant coordinating + // on this node waits for the release instead of failing the claim. + VotesHeld bool } // CommitRequest contains parameters for the commit phase @@ -477,7 +481,7 @@ func (re *ReplicationEngine) prepareAutoIncClaim( // it or a majority granted, until it has merged bases from a // majority (AutoIncHoldTable). A missing ACK leaves the claim to the // rest of the cluster, and to this node once the merge completes. - return &PrepareResult{Success: false, + return &PrepareResult{Success: false, VotesHeld: true, Error: fmt.Sprintf("auto-increment claim for %s declined: this node's votes are held until it merges claim bases from a majority", claim.Table)} } @@ -1024,6 +1028,7 @@ func (pr *PrepareResult) ToCoordinatorResponse() *coordinator.ReplicationRespons Rejected: pr.Rejected, ErrorCode: pr.ErrorCode, AutoIDStoredBase: pr.AutoIDStoredBase, + VotesHeld: pr.VotesHeld, } } diff --git a/docs/src/pages/compatibility.mdx b/docs/src/pages/compatibility.mdx index a42f1444..ed25d67f 100644 --- a/docs/src/pages/compatibility.mdx +++ b/docs/src/pages/compatibility.mdx @@ -132,8 +132,11 @@ Membership Changes and Narrow AUTO_INCREMENT Ids in [Operations](/operations)). - **Single node:** set `cluster.standalone = true` with `seed_nodes` empty. The node claims ranges without peers. - **Cluster:** a node whose system database is new holds its claim votes until it has merged claim bases from - enough cluster members; until its log says `AUTO_INCREMENT claim votes released after merging claim bases`, narrow inserts can return the - retryable 1205. Start applications that insert at boot after that line. + enough cluster members, and logs `AUTO_INCREMENT claim votes released after merging claim bases` when it does. + A narrow insert through that node during the hold waits for the release, up to + `transaction.lock_wait_timeout_seconds`, and returns the retryable 1205 only if the hold outlasts it. Inside an + explicit transaction that has already written a row, the insert does not wait: it returns 1205 at once, because + the release can depend on peers joining from a snapshot that waits for that transaction. - Ids increase within one node's range, not across nodes: two nodes inserting into the same table issue ids from different ranges. Ids a range did not issue are skipped for good, for example after a restart. - `LAST_INSERT_ID()` reports the generated id as usual. diff --git a/grpc/autoinc_merge.go b/grpc/autoinc_merge.go index b96120e1..fcfce44b 100644 --- a/grpc/autoinc_merge.go +++ b/grpc/autoinc_merge.go @@ -229,8 +229,10 @@ func runAutoIncBaseMerge(ctx context.Context, nodeID uint64, store autoIncBaseSt } } -// holdUntilMerged runs merge rounds every conf.MergeInterval until this -// node's votes are released. It reports false when ctx ended first. +// holdUntilMerged runs merge rounds until this node's votes are released: +// every conf.MergeInterval, and at once whenever a peer turns ALIVE - a new +// answer is what a held node waits for, and narrow inserts wait on the +// release. It reports false when ctx ended first. func holdUntilMerged(ctx context.Context, nodeID uint64, store autoIncBaseStore, registry autoIncMembership, conf AutoIncMergeConfig, fetch fetchAutoIncBases) bool { if held, err := store.AutoIncVotesHeld(); err == nil && !held { return true @@ -240,6 +242,9 @@ func holdUntilMerged(ctx context.Context, nodeID uint64, store autoIncBaseStore, release := standaloneRelease{configured: conf.Standalone} warned := false for { + // Taken before the round, so a peer that turns ALIVE during it + // triggers another. + aliveChanged := registry.AliveChanged() membership := registry.Count() released, err := mergeAutoIncBasesOnce(ctx, store, alivePeers(registry, nodeID), membership, release.allowed(membership), fetch) if err != nil { @@ -253,7 +258,8 @@ func holdUntilMerged(ctx context.Context, nodeID uint64, store autoIncBaseStore, warned = true log.Warn().Int("membership", membership).Bool("standalone", conf.Standalone). Msg("AUTO_INCREMENT claim votes held until claim bases are merged from enough cluster members; " + - "narrow AUTO_INCREMENT inserts may return 1205 until then. A single-node deployment sets " + + "narrow AUTO_INCREMENT inserts wait for the release, up to transaction.lock_wait_timeout_seconds, " + + "and return 1205 after it (at once inside a transaction that already wrote). A single-node deployment sets " + "cluster.standalone = true; an operator can force a release with " + "POST /admin/cluster/autoinc/release-votes?accept_risk=" + ForceReleaseRiskToken) } @@ -261,6 +267,7 @@ func holdUntilMerged(ctx context.Context, nodeID uint64, store autoIncBaseStore, case <-ctx.Done(): return false case <-ticker.C: + case <-aliveChanged: } } } @@ -324,6 +331,7 @@ type autoIncMembership interface { Count() int MemberIDs() []uint64 GetAlive() []*NodeState + AliveChanged() <-chan struct{} } // syncAutoIncBasesOnce asks every alive peer - not a quorum, every one, held diff --git a/grpc/autoinc_merge_test.go b/grpc/autoinc_merge_test.go index 4ceb302d..6bc7f2c7 100644 --- a/grpc/autoinc_merge_test.go +++ b/grpc/autoinc_merge_test.go @@ -4,10 +4,12 @@ import ( "context" "errors" "sync" + "sync/atomic" "testing" "time" "github.com/maxpert/marmot/cfg" + "github.com/maxpert/marmot/common" "github.com/maxpert/marmot/db" "github.com/maxpert/marmot/hlc" "github.com/stretchr/testify/require" @@ -227,10 +229,18 @@ func TestForceReleaseAutoIncVotesReleasesWithoutASafeMerge(t *testing.T) { type fakeMembership struct { members []uint64 down map[uint64]bool + // alive signals a peer turning ALIVE; nil never does. + alive *common.Broadcast } func (m fakeMembership) Count() int { return len(m.members) } func (m fakeMembership) MemberIDs() []uint64 { return m.members } +func (m fakeMembership) AliveChanged() <-chan struct{} { + if m.alive == nil { + return nil + } + return m.alive.Next() +} func (m fakeMembership) GetAlive() []*NodeState { var alive []*NodeState for _, id := range m.members { @@ -326,6 +336,48 @@ func TestReleasedNodeSyncsAtOnce(t *testing.T) { require.False(t, held) } +// TestHeldNodeMergesWhenPeerTurnsAlive pins that a held node retries its +// merge the moment a peer turns ALIVE, not a merge interval later: narrow +// inserts on a held node wait for the release. +// +// Mutation: drop the AliveChanged case from holdUntilMerged's wait. The node +// stays held until the hour-long interval, and "not released on ALIVE" fires. +func TestHeldNodeMergesWhenPeerTurnsAlive(t *testing.T) { + var reachable atomic.Bool + answers := peerAnswers(map[uint64]*AutoIncBasesResponse{2: {VotesHeld: true}, 3: {VotesHeld: true}}) + fetch := func(ctx context.Context, peer uint64) (*AutoIncBasesResponse, error) { + if !reachable.Load() { + return nil, errors.New("unreachable") + } + return answers(ctx, peer) + } + store := &fakeBaseStore{held: true} + alive := &common.Broadcast{} + conf := AutoIncMergeConfig{MergeInterval: time.Hour, BaseSyncInterval: time.Hour} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + runAutoIncBaseMerge(ctx, 1, store, fakeMembership{members: []uint64{1, 2, 3}, alive: alive}, conf, fetch) + close(done) + }() + defer func() { + cancel() + <-done + }() + + // The first round finds no peer; only an ALIVE can start the next. + require.Never(t, func() bool { + held, _ := store.AutoIncVotesHeld() + return !held + }, 100*time.Millisecond, 10*time.Millisecond, "released with no peer answering") + reachable.Store(true) + alive.Notify() + require.Eventually(t, func() bool { + held, _ := store.AutoIncVotesHeld() + return !held + }, time.Second, 10*time.Millisecond, "not released on ALIVE") +} + // TestStandaloneNeverReleasesAloneAfterSeeingPeers pins R3c-11: a node // configured standalone releases alone only while it has never seen another // member. One that did is a member of a cluster, whatever its configuration diff --git a/grpc/node_registry.go b/grpc/node_registry.go index 6c45c1bd..42995487 100644 --- a/grpc/node_registry.go +++ b/grpc/node_registry.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/maxpert/marmot/common" "github.com/maxpert/marmot/telemetry" "github.com/rs/zerolog/log" ) @@ -56,6 +57,9 @@ type NodeRegistry struct { onNodeDeadFunc func(*NodeState) // Callback when node transitions to DEAD onNodeLeavingFunc func() // Callback when local node marked LEAVING via remote decommission callbackMu sync.RWMutex + // aliveChanged is notified, after onNodeAliveFunc, whenever a node turns + // or is discovered ALIVE (AliveChanged). + aliveChanged common.Broadcast // store persists membership so a restarted node does not compute a quorum // from a membership of one. nil disables persistence. @@ -290,6 +294,13 @@ func (nr *NodeRegistry) fireOnNodeAlive(node *NodeState) { if callback != nil { callback(node) } + nr.aliveChanged.Notify() +} + +// AliveChanged returns a channel closed the next time a node turns or is +// discovered ALIVE, once the ALIVE callback has run for it. +func (nr *NodeRegistry) AliveChanged() <-chan struct{} { + return nr.aliveChanged.Next() } // handleSelfUpdateLocked implements SWIM refutation for local node diff --git a/grpc/node_registry_test.go b/grpc/node_registry_test.go index 5b277924..fbeceb6f 100644 --- a/grpc/node_registry_test.go +++ b/grpc/node_registry_test.go @@ -1152,6 +1152,38 @@ func TestLeavingThenCrashesSuspect(t *testing.T) { } } +// TestNodeRegistry_AliveChangedFiresOnAlive pins that AliveChanged wakes its +// waiters when a peer is discovered ALIVE or turns ALIVE, and not otherwise. +// +// Mutation: drop the Notify from fireOnNodeAlive. "discovered ALIVE" fires. +func TestNodeRegistry_AliveChangedFiresOnAlive(t *testing.T) { + nr := NewNodeRegistry(1, "localhost:8081") + closed := func(ch <-chan struct{}) bool { + select { + case <-ch: + return true + default: + return false + } + } + + ch := nr.AliveChanged() + nr.Update(&NodeState{NodeId: 2, Address: "localhost:8082", Status: NodeStatus_ALIVE, Incarnation: 1}) + if !closed(ch) { + t.Fatal("discovered ALIVE did not signal AliveChanged") + } + + ch = nr.AliveChanged() + nr.Update(&NodeState{NodeId: 3, Address: "localhost:8083", Status: NodeStatus_SUSPECT, Incarnation: 1}) + if closed(ch) { + t.Fatal("a peer discovered SUSPECT signalled AliveChanged") + } + nr.Update(&NodeState{NodeId: 3, Address: "localhost:8083", Status: NodeStatus_ALIVE, Incarnation: 2}) + if !closed(ch) { + t.Fatal("a peer turning ALIVE did not signal AliveChanged") + } +} + // TestNodeRegistry_DiscoveringAnAliveNodeConnectsToIt pins that the ALIVE // callback, which opens this node's connection to a // peer, fires when gossip first tells us of a peer that is already ALIVE, not diff --git a/id/range_allocator.go b/id/range_allocator.go index 5708fd50..a56d7208 100644 --- a/id/range_allocator.go +++ b/id/range_allocator.go @@ -130,6 +130,13 @@ func NewRangeAllocator(claimer Claimer, timeout time.Duration) *RangeAllocator { } } +// Through returns an allocator that shares a's ranges and timeout but claims +// through claimer. A table's cursor is one for both, so neither ever issues +// an id the other did. +func (a *RangeAllocator) Through(claimer Claimer) *RangeAllocator { + return &RangeAllocator{claimer: claimer, timeout: a.timeout, cursors: a.cursors} +} + func (a *RangeAllocator) cursorFor(database, table string) *cursor { c, _ := a.cursors.LoadOrCompute(tableKey{database: database, table: table}, func() *cursor { return &cursor{ramp: rampInitial} diff --git a/marmot.go b/marmot.go index 43932f13..8de9b71c 100644 --- a/marmot.go +++ b/marmot.go @@ -460,6 +460,8 @@ func main() { // AUTO_INCREMENT claims carry this node's schema version, so a voter // that lags a DDL declines them as it declines DML. writeCoordinator.SetSchemaVersionSource(schemaVersionMgr.GetSchemaVersion) + // A claim this node declines for its held votes waits for their release. + writeCoordinator.SetVoteHold(dbMgr) // Initialize LocalReader for ReadCoordinator localReader := db.NewLocalReader(dbMgr) diff --git a/test/autoinc_claim_inprocess_test.go b/test/autoinc_claim_inprocess_test.go index 9e8a60af..ca5e699b 100644 --- a/test/autoinc_claim_inprocess_test.go +++ b/test/autoinc_claim_inprocess_test.go @@ -229,6 +229,7 @@ func (c *inprocCluster) buildNode(id uint64, dataDir string) *inprocNode { dm.SetClusterMembership(c.provider.GetTotalMembershipSize) lr := db.NewLocalReplicator(id, dm, clock) wc := coordinator.NewWriteCoordinator(id, c.provider, c.fanout, lr, inprocCoordinatorTimeout, clock) + wc.SetVoteHold(dm) c.fanout.register(id, lr) return &inprocNode{id: id, dataDir: dataDir, clock: clock, dm: dm, lr: lr, wc: wc} } diff --git a/test/autoinc_hold_wait_cluster_test.go b/test/autoinc_hold_wait_cluster_test.go new file mode 100644 index 00000000..cfe0c5b9 --- /dev/null +++ b/test/autoinc_hold_wait_cluster_test.go @@ -0,0 +1,128 @@ +package test + +import ( + "database/sql" + "fmt" + "strings" + "testing" + "time" + + "github.com/maxpert/marmot/protocol/mysqlcode" +) + +// TestNarrowAutoInc_HeldInsertInTransactionDoesNotStallJoin: on a held node +// alone, an explicit transaction writes a row - pinning the user database's +// SQLite writer - and then inserts into a narrow table while its peers start. +// The release needs their answers, they join from this node's snapshot, and +// the snapshot's checkpoint waits on the pinned writer; so the insert answers +// 1205 at once instead of waiting for a release it blocks. After the client +// rolls back, the peers join, the node releases, and a narrow insert +// succeeds. +// +// The merge interval is a minute, so only peers turning ALIVE trigger the +// release; the lock wait is 20s, far above the fail-fast bound. +// +// Mutation: wait for the release inside a pinned transaction. The insert +// blocks until the claim's deadline and "waited for the release" fires. +func TestNarrowAutoInc_HeldInsertInTransactionDoesNotStallJoin(t *testing.T) { + c := newCluster(t, func(cfg *clusterConfig) { cfg.autoIncMergeMS = 60000; cfg.lockWaitTimeoutSecs = 20 }) + c.startNode(1) + c.waitMySQL(1) + c.mustExec(1, "marmot", "CREATE TABLE plain (k INT PRIMARY KEY, v TEXT)") + c.mustExec(1, "marmot", "CREATE TABLE narrow (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + if held, err := c.votesHeld(1); err != nil || !held { + t.Fatalf("node 1 alone must hold its votes: held=%v err=%v", held, err) + } + + // A read timeout above the lock wait, so a wait shows as its duration + // rather than a dropped connection. + pool, err := sql.Open("mysql", strings.ReplaceAll(c.dsn(1, "marmot"), clientTimeout.String(), "30s")) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + tx, err := pool.Begin() + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec("INSERT INTO plain (k, v) VALUES (1, 'pin')"); err != nil { + t.Fatal(err) + } + c.startNode(2) + c.startNode(3) + start := time.Now() + _, err = tx.Exec("INSERT INTO narrow (v) VALUES ('in-txn')") + elapsed := time.Since(start) + if mysqlCode(err) != mysqlcode.ErrCodeLockTimeout { + t.Fatalf("narrow insert in a pinned transaction on a held node returned %v, want 1205", err) + } + if elapsed > time.Second { + t.Fatalf("narrow insert in a pinned transaction waited for the release: 1205 after %s", elapsed) + } + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + c.phase("in-transaction insert failed fast") + + if _, err := pool.Exec("INSERT INTO narrow (v) VALUES ('after')"); err != nil { + t.Fatalf("autocommit narrow insert after the rollback: %v", err) + } + if held, err := c.votesHeld(1); err != nil || held { + t.Fatalf("node 1 still held after a granted claim: held=%v err=%v", held, err) + } + c.phase("released and inserted") +} + +// TestNarrowAutoInc_InsertWaitsOutStartupVoteHold: a client that writes to a +// narrow AUTO_INCREMENT table the moment a new cluster's first node answers +// MySQL - LLDAP creating its first group - gets its row, not 1205. Every +// node of a new cluster starts with its claim votes held (db.AutoIncHoldTable) +// and releases them only after merging claim bases from its peers; a narrow +// insert waits for that release, up to the lock wait. +// +// The merge interval is the production default, so the hold outlasts the +// insert's arrival unless a peer turning ALIVE triggers the merge. +// +// Mutation: ClaimRange not waiting on ErrLocalVotesHeld fails the first +// insert with 1205. +func TestNarrowAutoInc_InsertWaitsOutStartupVoteHold(t *testing.T) { + c := newCluster(t, func(cfg *clusterConfig) { cfg.autoIncMergeMS = 2000 }) + c.startNode(1) + waitFor(t, "seed node 1 ALIVE", readyDeadline, func() (bool, string) { + statuses, err := c.statuses(1) + if err != nil { + return false, err.Error() + } + return statuses[1] == "ALIVE", fmt.Sprint(statuses) + }) + c.startNode(2) + c.startNode(3) + c.waitMySQL(1) + + const table = "startup_groups" + c.mustExec(1, "marmot", "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + held, err := c.votesHeld(1) + if err != nil { + t.Fatal(err) + } + t.Logf("node 1 votes held before the first insert: %v", held) + ids := map[int64]int{} + record := func(nodeID int) { + res := c.mustExec(nodeID, "marmot", "INSERT INTO "+table+" (v) VALUES (?)", fmt.Sprintf("n%d", nodeID)) + got, err := res.LastInsertId() + if err != nil { + t.Fatal(err) + } + if prev, dup := ids[got]; dup { + t.Fatalf("id %d issued by node %d and node %d", got, prev, nodeID) + } + ids[got] = nodeID + } + record(1) + c.phase("first insert") + + c.waitTable("marmot", table, 2, 3) + record(2) + record(3) + c.phase("inserts on every node") +} diff --git a/test/autoinc_hold_wait_inprocess_test.go b/test/autoinc_hold_wait_inprocess_test.go new file mode 100644 index 00000000..fbb2378d --- /dev/null +++ b/test/autoinc_hold_wait_inprocess_test.go @@ -0,0 +1,264 @@ +//go:build sqlite_preupdate_hook +// +build sqlite_preupdate_hook + +package test + +import ( + "errors" + "testing" + "time" + + "github.com/maxpert/marmot/cfg" + "github.com/maxpert/marmot/coordinator" + "github.com/maxpert/marmot/db" + "github.com/maxpert/marmot/id" + "github.com/maxpert/marmot/protocol" + "github.com/stretchr/testify/require" +) + +// holdWaitTableWidthMax is the ceiling of markedTableDDL's 32-bit signed column. +const holdWaitTableWidthMax = 1<<31 - 1 + +// setupHoldWaitCluster builds a released 3-node in-process cluster with +// database.table created through node 1 and seeded on every node. +func setupHoldWaitCluster(t *testing.T, database, table string) *inprocCluster { + t.Helper() + c := newInprocCluster(t, []uint64{1, 2, 3}) + for _, nodeID := range []uint64{1, 2, 3} { + require.NoError(t, c.nodes[nodeID].dm.CreateDatabase(database)) + } + driveDDL(t, c.nodes[1], database, table, sprintfDDL(table)) + for _, nodeID := range []uint64{1, 2, 3} { + waitForBase(t, c.nodes[nodeID], database, table, 0) + } + return c +} + +// holdVotes holds node's claim votes, as a node that initialised its system +// database does until it merges claim bases (db.AutoIncHoldTable). +func holdVotes(t *testing.T, node *inprocNode) { + t.Helper() + _, err := node.dm.GetSystemDatabase().GetWriteDB().Exec( + "INSERT INTO "+db.AutoIncHoldTable+" (id, since) VALUES (1, ?)", time.Now().UnixNano()) + require.NoError(t, err) +} + +// setLockWait sets transaction.lock_wait_timeout_seconds for one test. +func setLockWait(t *testing.T, seconds int) { + prev := cfg.Config.Transaction.LockWaitTimeoutSeconds + cfg.Config.Transaction.LockWaitTimeoutSeconds = seconds + t.Cleanup(func() { cfg.Config.Transaction.LockWaitTimeoutSeconds = prev }) +} + +// allocateResult is one narrow id allocation's outcome. +type allocateResult struct { + id uint64 + err error +} + +// allocateAsync allocates one id for database.table through allocator. +func allocateAsync(allocator *id.RangeAllocator, database, table string) <-chan allocateResult { + done := make(chan allocateResult, 1) + go func() { + next, err := allocator.Allocate(database, table, holdWaitTableWidthMax, 1) + done <- allocateResult{id: next, err: err} + }() + return done +} + +// isLockWaitTimeout reports whether err is MySQL 1205. +func isLockWaitTimeout(err error) bool { + var mysqlErr *protocol.MySQLError + return errors.As(err, &mysqlErr) && mysqlErr.Code == protocol.ErrCodeLockTimeout +} + +// TestClaimRange_HeldCoordinatorWaitsForRelease: a narrow id allocated on a +// node whose claim votes are held waits for the release, then claims a fresh +// range - the LLDAP startup failure, where the other two nodes had released +// and node 1 answered 1205 at once. +// +// Mutations: ClaimRange not waiting on ErrLocalVotesHeld returns 1205 within +// milliseconds, before the release; a release that does not wake waiters +// (no Notify in MergeAutoIncBasesAndReleaseVotes) misses the 3s guard. +func TestClaimRange_HeldCoordinatorWaitsForRelease(t *testing.T) { + setLockWait(t, 5) + const database, table = "holdwait", "items" + c := setupHoldWaitCluster(t, database, table) + + first, err := id.NewRangeAllocator(c.nodes[2].wc, inprocCoordinatorTimeout).Allocate(database, table, holdWaitTableWidthMax, 1) + require.NoError(t, err) + + holdVotes(t, c.nodes[1]) + const releaseAfter = 300 * time.Millisecond + start := time.Now() + done := allocateAsync(id.NewRangeAllocator(c.nodes[1].wc, inprocCoordinatorTimeout), database, table) + + select { + case res := <-done: + t.Fatalf("allocation on a held node returned before the release after %s: id %d, err %v", time.Since(start), res.id, res.err) + case <-time.After(releaseAfter): + } + require.NoError(t, c.nodes[1].dm.MergeAutoIncBasesAndReleaseVotes(nil)) + + select { + case res := <-done: + require.NoError(t, res.err, "a held node's allocation must wait out the hold, not return 1205") + require.NotZero(t, res.id) + require.NotEqual(t, first, res.id, "the id must come from a fresh range") + require.Greater(t, res.id, first, "node 1's range must lie above node 2's") + case <-time.After(3 * time.Second): + t.Fatal("allocation did not complete within 3s of the release") + } +} + +// setWriteTimeout sets replication.write_timeout_ms for one test. +func setWriteTimeout(t *testing.T, ms int) { + prev := cfg.Config.Replication.WriteTimeoutMS + cfg.Config.Replication.WriteTimeoutMS = ms + t.Cleanup(func() { cfg.Config.Replication.WriteTimeoutMS = prev }) +} + +// setupHandlerCluster builds a released 3-node in-process cluster and a +// MySQL-facing handler over node 1, as marmot.go wires one, with a narrow +// table and a plain table created through it in database. +func setupHandlerCluster(t *testing.T, database, table string) (*inprocCluster, *coordinator.CoordinatorHandler) { + t.Helper() + c := newInprocCluster(t, []uint64{1, 2, 3}) + for _, nodeID := range []uint64{1, 2, 3} { + require.NoError(t, c.nodes[nodeID].dm.CreateDatabase(database)) + } + node := c.nodes[1] + reader := coordinator.NewReadCoordinator(node.id, c.provider, db.NewLocalReader(node.dm), inprocCoordinatorTimeout) + handler := coordinator.NewCoordinatorHandler(node.id, node.wc, reader, node.clock, node.dm, nil, nil, nil) + session := newSession(100, database) + for _, ddl := range []string{ + "CREATE TABLE " + table + " (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)", + "CREATE TABLE plain (k INT PRIMARY KEY, v TEXT)", + } { + _, err := handler.HandleQuery(session, ddl, nil) + require.NoError(t, err) + } + for _, nodeID := range []uint64{1, 2, 3} { + waitForBase(t, c.nodes[nodeID], database, table, 0) + } + return c, handler +} + +// newSession is a client connection to database. +func newSession(connID uint64, database string) *protocol.ConnectionSession { + return &protocol.ConnectionSession{ConnID: connID, CurrentDatabase: database, TranspilationEnabled: true} +} + +// timedQuery runs q through handler and reports how long it took. +func timedQuery(handler *coordinator.CoordinatorHandler, session *protocol.ConnectionSession, q string) (time.Duration, error) { + start := time.Now() + _, err := handler.HandleQuery(session, q, nil) + return time.Since(start), err +} + +// TestNarrowInsert_HeldWaitIsBoundedByTheLockWait: a narrow insert on a node +// whose votes are never released waits the lock wait - the MySQL-facing +// promise - even when the write timeout is shorter, then answers 1205. +// +// Mutation: bound the claim by the write timeout alone (the allocator's +// budget without the lock wait). The insert answers 1205 after 0.5s. +func TestNarrowInsert_HeldWaitIsBoundedByTheLockWait(t *testing.T) { + setLockWait(t, 2) + setWriteTimeout(t, 500) + const database, table = "holdlockwait", "items" + c, handler := setupHandlerCluster(t, database, table) + + holdVotes(t, c.nodes[1]) + elapsed, err := timedQuery(handler, newSession(1, database), "INSERT INTO "+table+" (v) VALUES ('held')") + require.True(t, isLockWaitTimeout(err), "want 1205, got %v", err) + require.GreaterOrEqual(t, elapsed, 1800*time.Millisecond, "the hold wait ended before the lock wait") + require.Less(t, elapsed, 4*time.Second, "the hold wait outlasted the lock wait") +} + +// TestNarrowInsert_HeldInsidePinnedTransactionFailsFast: inside an explicit +// transaction that already wrote a row - so its connection holds the user +// database's SQLite writer - a narrow insert on a held node answers 1205 at +// once instead of waiting. The release it would wait for needs the joining +// peers' snapshot, whose checkpoint waits on that very writer. +// +// Mutation: wait regardless of a pinned session. The insert waits out the +// 5s lock wait. +func TestNarrowInsert_HeldInsidePinnedTransactionFailsFast(t *testing.T) { + setLockWait(t, 5) + const database, table = "holdpinned", "items" + c, handler := setupHandlerCluster(t, database, table) + session := newSession(1, database) + + holdVotes(t, c.nodes[1]) + _, err := handler.HandleQuery(session, "BEGIN", nil) + require.NoError(t, err) + _, err = handler.HandleQuery(session, "INSERT INTO plain (k, v) VALUES (1, 'pin')", nil) + require.NoError(t, err) + elapsed, err := timedQuery(handler, session, "INSERT INTO "+table+" (v) VALUES ('in-txn')") + require.True(t, isLockWaitTimeout(err), "want 1205, got %v", err) + require.Less(t, elapsed, time.Second, "a claim inside a pinned transaction waited for the release") + _, err = handler.HandleQuery(session, "ROLLBACK", nil) + require.NoError(t, err) +} + +// TestNarrowInsert_FirstWriteOfTransactionWaitsForRelease: LLDAP creates each +// startup group in its own transaction whose first write is the narrow +// INSERT. No writer is pinned yet when it claims, so it waits for the release +// and succeeds - the LLDAP startup failure stays fixed. +// +// Mutation: fail fast in every explicit transaction. The INSERT answers 1205 +// before the release. +func TestNarrowInsert_FirstWriteOfTransactionWaitsForRelease(t *testing.T) { + setLockWait(t, 5) + const database, table = "holdfirstwrite", "items" + c, handler := setupHandlerCluster(t, database, table) + session := newSession(1, database) + + holdVotes(t, c.nodes[1]) + _, err := handler.HandleQuery(session, "BEGIN", nil) + require.NoError(t, err) + done := make(chan error, 1) + go func() { + _, err := handler.HandleQuery(session, "INSERT INTO "+table+" (v) VALUES ('lldap_admin')", nil) + done <- err + }() + select { + case err := <-done: + t.Fatalf("the first write of a transaction returned before the release: %v", err) + case <-time.After(300 * time.Millisecond): + } + require.NoError(t, c.nodes[1].dm.MergeAutoIncBasesAndReleaseVotes(nil)) + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(3 * time.Second): + t.Fatal("the insert did not complete within 3s of the release") + } + _, err = handler.HandleQuery(session, "COMMIT", nil) + require.NoError(t, err) +} + +// TestClaimRange_HeldCoordinatorWaitIsBounded: a node whose votes are never +// released answers a narrow allocation with 1205 once the lock wait expires, +// not at once and not never. +// +// Mutation: an unbounded wait (no lock-wait deadline) never returns within +// the 3s guard; no wait at all returns before the lock wait. +func TestClaimRange_HeldCoordinatorWaitIsBounded(t *testing.T) { + setLockWait(t, 1) + const database, table = "holdbound", "items" + c := setupHoldWaitCluster(t, database, table) + + holdVotes(t, c.nodes[1]) + start := time.Now() + done := allocateAsync(id.NewRangeAllocator(c.nodes[1].wc, inprocCoordinatorTimeout), database, table) + + select { + case res := <-done: + elapsed := time.Since(start) + require.True(t, isLockWaitTimeout(res.err), "want 1205, got id %d, err %v", res.id, res.err) + require.GreaterOrEqual(t, elapsed, 900*time.Millisecond, "1205 came before the lock wait expired") + case <-time.After(3 * time.Second): + t.Fatal("a never-released hold was waited on past the lock wait") + } +} diff --git a/test/autoinc_membership_cluster_test.go b/test/autoinc_membership_cluster_test.go index e4759029..56eb361f 100644 --- a/test/autoinc_membership_cluster_test.go +++ b/test/autoinc_membership_cluster_test.go @@ -89,16 +89,21 @@ func (c *cluster) waitCompleteSync(id int, restarted ...uint64) autoIncSyncAnswe // TestNarrowAutoInc_HeldNodeReleasesAfterItsSeedReturns: a wiped node whose // only seed is down - so its join fails and its catch-up cannot use the seed // - is held while it cannot reach enough members: the sync reports it held -// and incomplete, and a narrow insert through it returns 1205. It releases -// once its seed is back, which needs it to connect to the returned seed it -// learned of as ALIVE from another member. +// and incomplete, and a narrow insert through it waits out the lock wait and +// returns 1205. It releases once its seed is back, which needs it to connect +// to the returned seed it learned of as ALIVE from another member. +// +// The lock wait is 1s, inside the client's read timeout, so the insert's +// bounded wait ends in 1205 rather than a dropped connection. // // Mutations: never hold (both hold sites) - "was not held while its seed was // down" fires; drop the ALIVE callback on discovery (node_registry.go) - the -// cluster never becomes ready after the seed returns. +// cluster never becomes ready after the seed returns; an unbounded hold wait +// - the insert never answers 1205. func TestNarrowAutoInc_HeldNodeReleasesAfterItsSeedReturns(t *testing.T) { const table = "seedback" - c := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)") + c := startNarrowCluster(t, table, "CREATE TABLE "+table+" (id INT AUTO_INCREMENT PRIMARY KEY, v TEXT)", + func(cfg *clusterConfig) { cfg.lockWaitTimeoutSecs = 1 }) ledger := newIDLedger(t) insertOnEvery(c, ledger, table, "before", 3, 1, 2, 3) c.wipe(3) diff --git a/test/autoinc_narrow_cluster_test.go b/test/autoinc_narrow_cluster_test.go index 6a8c7f4f..1e0828a5 100644 --- a/test/autoinc_narrow_cluster_test.go +++ b/test/autoinc_narrow_cluster_test.go @@ -26,10 +26,11 @@ const ( claimRetryDeadline = 3 * time.Second ) -// startNarrowCluster starts a cluster and creates table with ddl on it. -func startNarrowCluster(t *testing.T, table, ddl string) *cluster { +// startNarrowCluster starts a cluster, configured by opts, and creates table +// with ddl on it. +func startNarrowCluster(t *testing.T, table, ddl string, opts ...func(*clusterConfig)) *cluster { t.Helper() - c := newCluster(t) + c := newCluster(t, opts...) c.start() c.createTable(1, "marmot", table, ddl) return c diff --git a/test/main_test.go b/test/main_test.go index fe15f7d5..a8da97f4 100644 --- a/test/main_test.go +++ b/test/main_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/maxpert/marmot/id" "github.com/maxpert/marmot/protocol" ) @@ -39,7 +40,9 @@ func TestMain(m *testing.M) { fmt.Fprintln(os.Stderr, "skipping cluster integration tests; set MARMOT_RUN_CLUSTER_INTEGRATION_TESTS=1 to run") os.Exit(0) } - if err := protocol.InitializePipeline(10000, nil); err != nil { + // A real generator enables id injection, as in a node process, for the + // in-process tests that drive statements through a CoordinatorHandler. + if err := protocol.InitializePipeline(10000, id.NewCompactGenerator(1)); err != nil { fmt.Fprintln(os.Stderr, "initialize query pipeline:", err) os.Exit(1) } From 276e96f46015840addcfba9660054c7e98a813b9 Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Tue, 29 Sep 2026 19:33:43 -0500 Subject: [PATCH 12/17] test: describe narrow AUTO_INCREMENT tests by behaviour, not by one application Test names, fixtures and comments said "LLDAP shape" where they test general behaviour: a narrow insert as the first write of an ORM-style transaction, and a client that starts while a node still holds its claim votes. They now say so. Statements and assertions are unchanged. The LLDAP page no longer says to wait for the vote release before starting: inserts wait for it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- coordinator/narrow_autoinc_insert_test.go | 10 +++++----- db/autoinc_hookdb_test.go | 8 ++++---- db/transaction.go | 7 ++++--- db/transaction_nondml_atomic_test.go | 2 +- docs/src/pages/lldap.mdx | 5 +++-- .../query/rules/autoincrement_id_narrow_test.go | 12 ++++++------ test/autoinc_hold_wait_cluster_test.go | 2 +- test/autoinc_hold_wait_inprocess_test.go | 15 ++++++++------- test/autoinc_narrow_cluster_test.go | 13 +++++++------ 9 files changed, 39 insertions(+), 35 deletions(-) diff --git a/coordinator/narrow_autoinc_insert_test.go b/coordinator/narrow_autoinc_insert_test.go index 28fd101f..c79b17b8 100644 --- a/coordinator/narrow_autoinc_insert_test.go +++ b/coordinator/narrow_autoinc_insert_test.go @@ -54,15 +54,15 @@ func TestNarrowAutoIncrementLastInsertIDAutocommit(t *testing.T) { require.Equal(t, ids[0], res.LastInsertId, "LAST_INSERT_ID must be the statement's first id") } -// TestNarrowAutoIncrementLLDAPTransactionShape pins D5 on the pinned -// explicit-transaction path, in the shape LLDAP boots with: one transaction -// inserting into two narrow tables, both of which need their first claim. +// TestNarrowAutoIncrementFirstWriteInTransactionShape pins D5 on the pinned +// explicit-transaction path, in the shape an ORM-style application boots +// with: one transaction inserting into two narrow tables, both of which need their first claim. // It must complete, every id must fit 32 bits, and each statement must report // its own first id - not the transaction's first insert. // // Mutation: report the transaction's first captured insert instead of the // statement's. The groups INSERT then reports the users id and this fires. -func TestNarrowAutoIncrementLLDAPTransactionShape(t *testing.T) { +func TestNarrowAutoIncrementFirstWriteInTransactionShape(t *testing.T) { s := setupNoopDML(t) for _, ddl := range []string{ "CREATE TABLE users (user_id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)", @@ -77,7 +77,7 @@ func TestNarrowAutoIncrementLLDAPTransactionShape(t *testing.T) { usersRes, err := s.handler.HandleQuery(s.session, "INSERT INTO users (name) VALUES ('admin')", nil) require.NoError(t, err) groupsRes, err := s.handler.HandleQuery(s.session, - "INSERT INTO `groups` (display_name) VALUES (?), (?)", []interface{}{"lldap_admin", "lldap_password_manager"}) + "INSERT INTO `groups` (display_name) VALUES (?), (?)", []interface{}{"app_admin", "app_editor"}) require.NoError(t, err) _, err = s.handler.HandleQuery(s.session, "COMMIT", nil) require.NoError(t, err) diff --git a/db/autoinc_hookdb_test.go b/db/autoinc_hookdb_test.go index c0f6b04f..c2d3654c 100644 --- a/db/autoinc_hookdb_test.go +++ b/db/autoinc_hookdb_test.go @@ -14,11 +14,11 @@ import ( "github.com/stretchr/testify/require" ) -// TestClaimCompletesWhilePinnedSessionOpen is the LLDAP-shape property, and it -// is the reason the claim row lives in the system database rather than in the -// user database it is about. +// TestClaimCompletesWhilePinnedSessionOpen is the ORM-transaction property, +// and it is the reason the claim row lives in the system database rather than +// in the user database it is about. // -// The sea-orm/LLDAP shape is: BEGIN; INSERT INTO users ...; INSERT INTO +// The ORM-style transaction shape is: BEGIN; INSERT INTO users ...; INSERT INTO // groups ...; COMMIT. Marmot serves that shape with a pinned session // (db/db_integration.go BeginPinnedSession), which takes hookDB's single // connection (hookDB.SetMaxOpenConns(1)) and opens a SQLite transaction on it diff --git a/db/transaction.go b/db/transaction.go index a4ed8cd0..f4847478 100644 --- a/db/transaction.go +++ b/db/transaction.go @@ -840,9 +840,10 @@ func classifyNonDMLIntents(intents []*WriteIntentRecord) []nonDMLIntentKind { // // A claim-only commit writes no marker because it cannot take the user // database's writer: ReplicationEngine.Commit calls this after -// AutoIncClaimStore.ApplyClaims for every claim-carrying txn, including the -// LLDAP shape where a pinned session already holds the user database's one -// SQLite writer (_txlock=immediate) for the whole surrounding transaction. +// AutoIncClaimStore.ApplyClaims for every claim-carrying txn, including an +// ORM-style explicit transaction where a pinned session already holds the +// user database's one SQLite writer (_txlock=immediate) for the whole +// surrounding transaction. // __marmot_applied_txn lives in that same user database file, so writing it // needs that same writer - which would self-deadlock the claim behind the // pinned session that is waiting on it, defeating the entire reason diff --git a/db/transaction_nondml_atomic_test.go b/db/transaction_nondml_atomic_test.go index 55e2edb2..ac754a2a 100644 --- a/db/transaction_nondml_atomic_test.go +++ b/db/transaction_nondml_atomic_test.go @@ -58,7 +58,7 @@ func TestApplyNonDMLIntents_AtomicAcrossDDLFailure(t *testing.T) { // is filtered out entirely) must NOT open a SQLite tx or write a marker. // Writing one here would need the user database's single writer // (_txlock=immediate) even when a concurrent pinned session already holds -// it for the whole surrounding LLDAP-shaped transaction - see +// it for the whole surrounding explicit transaction - see // TestClaimCompletesWhilePinnedSessionOpen (db/autoinc_hookdb_test.go), // which this behavior exists to keep passing. func TestApplyNonDMLIntents_ClaimOnlyWritesNoMarkerAndNoSQLiteTx(t *testing.T) { diff --git a/docs/src/pages/lldap.mdx b/docs/src/pages/lldap.mdx index 4172abbb..163d8b23 100644 --- a/docs/src/pages/lldap.mdx +++ b/docs/src/pages/lldap.mdx @@ -26,8 +26,9 @@ LLDAP's migrations run through Marmot and replicate like any other DDL. Its `gro [Compatibility](/compatibility)): - A single node needs `cluster.standalone = true`. -- On a new cluster, start LLDAP after the node it connects to logs `AUTO_INCREMENT claim votes released after merging claim bases`. Each node - issues group ids from its own range, so groups created through different nodes get ids from different ranges. +- On a cluster, LLDAP can start together with the nodes: while a node still holds its claim votes, the first + group insert waits for the release (see [Compatibility](/compatibility)). Each node issues group ids from its own + range, so groups created through different nodes get ids from different ranges. ## Verified Scenarios diff --git a/protocol/query/rules/autoincrement_id_narrow_test.go b/protocol/query/rules/autoincrement_id_narrow_test.go index af0591af..c8884e2e 100644 --- a/protocol/query/rules/autoincrement_id_narrow_test.go +++ b/protocol/query/rules/autoincrement_id_narrow_test.go @@ -51,10 +51,10 @@ func (f *fakeNarrow) Admit(database, table string, widthMax, explicit uint64) er // with it third, a TINYINT one, and a narrow INTEGER PRIMARY KEY not declared // AUTO_INCREMENT. var narrowSchemas = map[string]transform.SchemaInfo{ - "alias": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 0, AutoIncrementWidth: 32, Database: "lldap"}, - "groups": {AutoIncrementColumn: "group_id", AutoIncrementOrdinal: 0, AutoIncrementWidth: 32, AutoIncrementExplicit: true, Database: "lldap"}, - "late": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 2, AutoIncrementWidth: 32, AutoIncrementExplicit: true, Database: "lldap"}, - "tiny": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 0, AutoIncrementWidth: 8, AutoIncrementExplicit: true, Database: "lldap"}, + "alias": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 0, AutoIncrementWidth: 32, Database: "app"}, + "groups": {AutoIncrementColumn: "group_id", AutoIncrementOrdinal: 0, AutoIncrementWidth: 32, AutoIncrementExplicit: true, Database: "app"}, + "late": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 2, AutoIncrementWidth: 32, AutoIncrementExplicit: true, Database: "app"}, + "tiny": {AutoIncrementColumn: "id", AutoIncrementOrdinal: 0, AutoIncrementWidth: 8, AutoIncrementExplicit: true, Database: "app"}, } func applyNarrow(t *testing.T, sql string, narrow NarrowAllocator) (string, bool, error) { @@ -103,8 +103,8 @@ func TestNarrowInjectionShapes(t *testing.T) { if fmt.Sprint(narrow.allocs) != fmt.Sprint(tc.wantAllocs) { t.Fatalf("Allocate calls %v, want %v: a statement takes its ids in one call", narrow.allocs, tc.wantAllocs) } - if narrow.database != "lldap" || narrow.lastWidth != 2147483647 { - t.Fatalf("allocated for %q with ceiling %d, want lldap / 2147483647", narrow.database, narrow.lastWidth) + if narrow.database != "app" || narrow.lastWidth != 2147483647 { + t.Fatalf("allocated for %q with ceiling %d, want app / 2147483647", narrow.database, narrow.lastWidth) } }) } diff --git a/test/autoinc_hold_wait_cluster_test.go b/test/autoinc_hold_wait_cluster_test.go index cfe0c5b9..1aee9b17 100644 --- a/test/autoinc_hold_wait_cluster_test.go +++ b/test/autoinc_hold_wait_cluster_test.go @@ -75,7 +75,7 @@ func TestNarrowAutoInc_HeldInsertInTransactionDoesNotStallJoin(t *testing.T) { // TestNarrowAutoInc_InsertWaitsOutStartupVoteHold: a client that writes to a // narrow AUTO_INCREMENT table the moment a new cluster's first node answers -// MySQL - LLDAP creating its first group - gets its row, not 1205. Every +// MySQL - an application creating its first row - gets it, not 1205. Every // node of a new cluster starts with its claim votes held (db.AutoIncHoldTable) // and releases them only after merging claim bases from its peers; a narrow // insert waits for that release, up to the lock wait. diff --git a/test/autoinc_hold_wait_inprocess_test.go b/test/autoinc_hold_wait_inprocess_test.go index fbb2378d..0a86f856 100644 --- a/test/autoinc_hold_wait_inprocess_test.go +++ b/test/autoinc_hold_wait_inprocess_test.go @@ -74,8 +74,8 @@ func isLockWaitTimeout(err error) bool { // TestClaimRange_HeldCoordinatorWaitsForRelease: a narrow id allocated on a // node whose claim votes are held waits for the release, then claims a fresh -// range - the LLDAP startup failure, where the other two nodes had released -// and node 1 answered 1205 at once. +// range - the startup failure where the other two nodes had released and +// node 1 answered 1205 at once. // // Mutations: ClaimRange not waiting on ErrLocalVotesHeld returns 1205 within // milliseconds, before the release; a release that does not wake waiters @@ -201,10 +201,11 @@ func TestNarrowInsert_HeldInsidePinnedTransactionFailsFast(t *testing.T) { require.NoError(t, err) } -// TestNarrowInsert_FirstWriteOfTransactionWaitsForRelease: LLDAP creates each -// startup group in its own transaction whose first write is the narrow -// INSERT. No writer is pinned yet when it claims, so it waits for the release -// and succeeds - the LLDAP startup failure stays fixed. +// TestNarrowInsert_FirstWriteOfTransactionWaitsForRelease: an ORM-style client +// that starts while a node still holds its claim votes creates each startup +// row in its own transaction whose first write is the narrow INSERT. No +// writer is pinned yet when it claims, so it waits for the release and +// succeeds instead of answering 1205. // // Mutation: fail fast in every explicit transaction. The INSERT answers 1205 // before the release. @@ -219,7 +220,7 @@ func TestNarrowInsert_FirstWriteOfTransactionWaitsForRelease(t *testing.T) { require.NoError(t, err) done := make(chan error, 1) go func() { - _, err := handler.HandleQuery(session, "INSERT INTO "+table+" (v) VALUES ('lldap_admin')", nil) + _, err := handler.HandleQuery(session, "INSERT INTO "+table+" (v) VALUES ('app_admin')", nil) done <- err }() select { diff --git a/test/autoinc_narrow_cluster_test.go b/test/autoinc_narrow_cluster_test.go index 1e0828a5..7405107e 100644 --- a/test/autoinc_narrow_cluster_test.go +++ b/test/autoinc_narrow_cluster_test.go @@ -149,10 +149,11 @@ func TestNarrowAutoInc_TinyIntExhaustionIsDupEntry(t *testing.T) { c.waitRows("marmot", idStats(table, "id"), []string{"127|127|127"}, 1, 2, 3) } -// TestNarrowAutoInc_LLDAPTransactionShape: LLDAP boots with one transaction -// inserting into two narrow tables that both need their first claim. It -// commits, and each INSERT's LAST_INSERT_ID is its own first id. -func TestNarrowAutoInc_LLDAPTransactionShape(t *testing.T) { +// TestNarrowAutoInc_FirstWriteInTransactionShape: an ORM-style application +// boots with one transaction inserting into two narrow tables that both need +// their first claim. It commits, and each INSERT's LAST_INSERT_ID is its own +// first id. +func TestNarrowAutoInc_FirstWriteInTransactionShape(t *testing.T) { c := startNarrowCluster(t, "users", "CREATE TABLE users (user_id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)") c.createTable(1, "marmot", "groups", "CREATE TABLE `groups` (group_id INT AUTO_INCREMENT PRIMARY KEY, display_name TEXT)") @@ -164,7 +165,7 @@ func TestNarrowAutoInc_LLDAPTransactionShape(t *testing.T) { if err != nil { t.Fatalf("INSERT users: %v", err) } - groupsRes, err := tx.Exec("INSERT INTO `groups` (display_name) VALUES (?), (?)", "lldap_admin", "lldap_password_manager") + groupsRes, err := tx.Exec("INSERT INTO `groups` (display_name) VALUES (?), (?)", "app_admin", "app_editor") if err != nil { t.Fatalf("INSERT groups: %v", err) } @@ -178,7 +179,7 @@ func TestNarrowAutoInc_LLDAPTransactionShape(t *testing.T) { } c.waitRows("marmot", "SELECT user_id, name FROM users ORDER BY user_id", []string{fmt.Sprintf("%d|admin", userID)}, 1, 2, 3) c.waitRows("marmot", "SELECT group_id, display_name FROM `groups` ORDER BY group_id", - []string{fmt.Sprintf("%d|lldap_admin", groupID), fmt.Sprintf("%d|lldap_password_manager", groupID+1)}, 1, 2, 3) + []string{fmt.Sprintf("%d|app_admin", groupID), fmt.Sprintf("%d|app_editor", groupID+1)}, 1, 2, 3) } // TestNarrowAutoInc_UnmarkedTableKeepsWideIDs: a BIGINT AUTO_INCREMENT table From e5be2321f4032e21f6048c87868855de0c041750 Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Tue, 29 Sep 2026 20:33:31 -0500 Subject: [PATCH 13/17] fix(gc): gc_max_retention_hours = 0 means no maximum, not "delete the whole log" The config validation and the tombstone horizon treat a zero max retention as unlimited, but CleanupOldTransactionRecords computed maxCutoff = now, so every GC pass deleted every committed entry whatever the members had consumed. Lagging peers then fell back to snapshots instead of pulling the log. The same behaviour exists on master. With max <= 0 the max-retention branch never fires; only the safe position plus min retention delete entries. Tests that relied on 0 meaning "delete now" use an explicit 1ms. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- db/database_manager.go | 2 +- db/gc_log_test.go | 51 +++++++++++++++++++++++++++++++++++++---- db/meta_store.go | 1 + db/meta_store_pebble.go | 8 ++++++- db/transaction.go | 3 ++- 5 files changed, 57 insertions(+), 8 deletions(-) diff --git a/db/database_manager.go b/db/database_manager.go index fc0dc834..a957dfa0 100644 --- a/db/database_manager.go +++ b/db/database_manager.go @@ -361,7 +361,7 @@ func (dm *DatabaseManager) wireGCCoordination(mdb *ReplicatedDatabase, dbName st // (db/meta_store.go's ConsumedPositions). A // member with no recorded position counts as the zero position, so GC // deletes nothing for that database until every member has reported at - // least once (bounded by gcMaxRetention regardless). A member no longer + // least once (bounded by gcMaxRetention when it is set). A member no longer // current has its consumed position deleted so it stops pinning GC. // // A member down for longer than gc_max_retention_hours finds its diff --git a/db/gc_log_test.go b/db/gc_log_test.go index 793f7164..326a9c37 100644 --- a/db/gc_log_test.go +++ b/db/gc_log_test.go @@ -224,10 +224,11 @@ func TestGCDeletesOlderThanMaxRetentionRegardlessOfSafe(t *testing.T) { commitTestTxn(t, store, 3201, 1) time.Sleep(2 * time.Millisecond) - // maxRetention=0 forces unconditional deletion; minRetention huge so - // that branch alone would not fire; safe is zero (would refuse via the - // min-retention branch, but must not block the max-retention branch). - n, err := store.CleanupOldTransactionRecords(time.Hour, 0, LogPosition{}) + // maxRetention=1ms (the entry is 2ms old) forces unconditional deletion; + // minRetention huge so that branch alone would not fire; safe is zero + // (would refuse via the min-retention branch, but must not block the + // max-retention branch). + n, err := store.CleanupOldTransactionRecords(time.Hour, time.Millisecond, LogPosition{}) if err != nil { t.Fatalf("CleanupOldTransactionRecords: %v", err) } @@ -236,6 +237,41 @@ func TestGCDeletesOlderThanMaxRetentionRegardlessOfSafe(t *testing.T) { } } +// TestGCZeroMaxRetentionMeansNoForcedDeletion pins that maxRetention 0 is +// "no maximum": nothing is deleted past the safe position, however old, and +// consumed entries are still deleted once older than min retention. +func TestGCZeroMaxRetentionMeansNoForcedDeletion(t *testing.T) { + store, cleanup := createTestPebbleMetaStore(t) + defer cleanup() + + posA := commitTestTxn(t, store, 3251, 1) + commitTestTxn(t, store, 3252, 1) + posC := commitTestTxn(t, store, 3253, 1) + time.Sleep(2 * time.Millisecond) + + if n, err := store.CleanupOldTransactionRecords(0, 0, LogPosition{}); err != nil || n != 0 { + t.Fatalf("nothing consumed: deleted = %d, err = %v; want 0", n, err) + } + if n, err := store.CleanupOldTransactionRecords(time.Hour, 0, posC); err != nil || n != 0 { + t.Fatalf("all consumed but younger than min retention: deleted = %d, err = %v; want 0", n, err) + } + if n, err := store.CleanupOldTransactionRecords(0, 0, posA); err != nil || n != 1 { + t.Fatalf("safe covers A only: deleted = %d, err = %v; want 1", n, err) + } + for _, id := range []uint64{3252, 3253} { + if rec, _ := store.GetTransaction(id); rec == nil { + t.Fatalf("txn %d is above safe and must survive", id) + } + } + if through, err := store.TruncatedThrough(); err != nil || through != posA { + t.Fatalf("TruncatedThrough = %v, err = %v; want %v", through, err, posA) + } + + if n, err := store.CleanupOldTransactionRecords(0, 0, posC); err != nil || n != 2 { + t.Fatalf("safe covers C: deleted = %d, err = %v; want 2", n, err) + } +} + // TestGCStopsAtFirstNonDeletableEntryPrefixOnly enforces the no-holes // invariant: GC only ever deletes a prefix of the log in position order, so // TruncatedThrough always means "everything at or below this is gone." @@ -350,7 +386,12 @@ func TestGCSkipsANonCommittedSeqEntryUntilMaxRetention(t *testing.T) { t.Fatalf("TruncatedThrough = %v, want %v", through, posC) } - n, err = store.CleanupOldTransactionRecords(0, 0, posC) + // With no maximum, nothing ever forces the stray entry out. + if n, err := store.CleanupOldTransactionRecords(0, 0, posC); err != nil || n != 0 { + t.Fatalf("max retention 0: deleted = %d, err = %v; want 0", n, err) + } + + n, err = store.CleanupOldTransactionRecords(0, time.Millisecond, posC) if err != nil { t.Fatalf("CleanupOldTransactionRecords: %v", err) } diff --git a/db/meta_store.go b/db/meta_store.go index 9639c904..9ac5647b 100644 --- a/db/meta_store.go +++ b/db/meta_store.go @@ -200,6 +200,7 @@ type MetaStore interface { // CleanupOldTransactionRecords deletes committed log entries whose // position is <= safe and whose CommittedAt is older than minRetention, // or whose CommittedAt is older than maxRetention regardless of safe. + // A maxRetention <= 0 means no maximum: nothing above safe is deleted. // It only ever deletes a prefix of the log in position order (see the // implementation's doc comment) and advances TruncatedThrough to the // highest position it deleted. diff --git a/db/meta_store_pebble.go b/db/meta_store_pebble.go index a20c634d..13917a27 100644 --- a/db/meta_store_pebble.go +++ b/db/meta_store_pebble.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "errors" "fmt" + "math" "os" "path/filepath" "sort" @@ -2412,7 +2413,12 @@ func (s *PebbleMetaStore) CleanupOldTransactionRecords(minRetention, maxRetentio now := time.Now() minCutoff := now.Add(-minRetention).UnixNano() - maxCutoff := now.Add(-maxRetention).UnixNano() + // maxRetention <= 0 is "no maximum": no timestamp is below MinInt64, so + // nothing is deleted past safe and a non-committed entry is never forced. + maxCutoff := int64(math.MinInt64) + if maxRetention > 0 { + maxCutoff = now.Add(-maxRetention).UnixNano() + } type deletable struct { pos LogPosition diff --git a/db/transaction.go b/db/transaction.go index f4847478..31bd7c8c 100644 --- a/db/transaction.go +++ b/db/transaction.go @@ -1360,7 +1360,8 @@ func (tm *TransactionManager) cleanupStaleTransactions() (int, error) { // cleanupOldTransactionRecords deletes committed log entries at or below // the GC-safe position (see GCSafePositionFunc) once they are older than -// gcMinRetention, or unconditionally once they are older than gcMaxRetention. +// gcMinRetention, or unconditionally once they are older than gcMaxRetention +// (never, when gcMaxRetention is 0). func (tm *TransactionManager) cleanupOldTransactionRecords() (int, error) { // Get callback under lock tm.mu.RLock() From ea2bae8dcfd4389be1d043cabd6437196e5a899b Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Tue, 29 Sep 2026 20:33:31 -0500 Subject: [PATCH 14/17] docs: describe the log-pull anti-entropy, row versions and narrow ids README and the docs still described delta sync as the catch-up mechanism. They now describe the per-peer commit-log pull, applied-txn markers, snapshot fallback, GC by consumed position, the single commit timestamp and per-row versions, persisted membership, and narrow AUTO_INCREMENT range claims beside the BIGINT compact/extended ids. - reference.mdx: what the legacy delta_sync_* keys still control, the two autoinc interval keys, [metastore] strict_prepare_sync, and gc_max_retention_hours = 0 meaning no maximum. - metrics.mdx: the log-pull metrics; the delta sync counter never existed. - replication.mdx also corrects older errors: a nonexistent /intent/ key, node states and transitions, gossip timeouts, and the TxnID, LWW and HLC examples. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- README.md | 166 ++++++----- docs/src/pages/compatibility.mdx | 4 +- docs/src/pages/metrics.mdx | 10 +- docs/src/pages/reference.mdx | 54 ++-- docs/src/pages/replication.mdx | 492 ++++++++++++++++--------------- 5 files changed, 392 insertions(+), 334 deletions(-) diff --git a/README.md b/README.md index cc7f0975..4c2f50ba 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ Marmot v2 uses a fundamentally different architecture from other SQLite replicat **How It Works:** 1. **Write Coordination**: 2PC (Two-Phase Commit) with configurable consistency (ONE, QUORUM, ALL) -2. **Conflict Resolution**: Last-Write-Wins (LWW) with HLC timestamps +2. **Conflict Resolution**: Per-row Last-Write-Wins (LWW) on the transaction's commit HLC timestamp 3. **Cluster Membership**: SWIM-style gossip with failure detection 4. **Data Replication**: Full database replication - all nodes receive all data 5. **DDL Replication**: Cluster-wide schema changes with automatic idempotency @@ -209,12 +209,16 @@ Marmot v2 supports **distributed DDL (Data Definition Language) replication** wi → DROP TABLE IF EXISTS users ``` -3. **Schema Version Tracking**: Each database maintains a schema version counter +3. **Schema Version Tracking**: Each database maintains a schema version counter (`__marmot_schema_version`) - Incremented on every DDL operation - Exchanged via gossip protocol for drift detection - - Used by delta sync to validate transaction applicability + - A joining node is not promoted until its schema versions match or exceed its alive peers'; anti-entropy restores a + snapshot from the peer with the highest schema version -4. **Quorum-Based Replication**: DDL replicates like DML through the same 2PC mechanism +4. **Rolling Upgrades**: DDL and `CREATE`/`DROP DATABASE` return the retryable error 1213 while any cluster member does + not serve the commit-log pull protocol (an older release). Upgrade every node, then retry. + +5. **Quorum-Based Replication**: DDL replicates like DML through the same 2PC mechanism - No special master node needed - Works with existing consistency levels (QUORUM, ALL, etc.) @@ -253,7 +257,7 @@ Marmot v2 uses **Change Data Capture (CDC)** for replication instead of SQL stat - **Performance**: Binary format is more efficient than SQL text - **Reliability**: No issues with SQL syntax variations between MySQL and SQLite - **Lower Write Amplification**: Row payloads are not duplicated into Pebble intent keys; Pebble tracks transaction metadata and row locks while the CDC segment log stores DML bytes -- **Atomic CDC Publication**: DML bytes become visible through `/cdc_manifest/{txnID}` only after their segment ranges are known; recovery validates and truncates segment tails at record boundaries +- **Atomic CDC Publication**: DML bytes become visible through `/cdc/manifest/{txnID}` only after their segment ranges are known; recovery validates and truncates segment tails at record boundaries ### Row Key Extraction @@ -746,52 +750,59 @@ Marmot handles various failure and recovery scenarios automatically: |----------|----------| | **Minority partition** | Writes **fail** - cannot achieve quorum | | **Majority partition** | Writes **succeed** - quorum achieved | -| **Partition heals** | Delta sync + LWW merges divergent data | +| **Partition heals** | Anti-entropy log pull + per-row LWW merges divergent data | **How it works:** 1. During partition, only the majority side can commit writes (quorum enforcement) -2. When partition heals, nodes exchange transaction logs via `StreamChanges` RPC -3. Conflicts resolved using Last-Writer-Wins (LWW) with HLC timestamps -4. Higher node ID breaks ties for simultaneous writes +2. When partition heals, each node pulls the commit logs of its alive peers (see Anti-Entropy below) +3. Each row keeps the version of its last change; an older change never overwrites a newer one (per-row LWW) +4. For equal HLC timestamps, the higher node ID wins ### Node Failure & Recovery | Scenario | Recovery Method | |----------|-----------------| -| **Brief outage** | Delta sync - replay missed transactions | -| **Extended outage** | Snapshot transfer + delta sync | +| **Brief outage** | Log pull - fetch the transactions it missed from every alive peer | +| **Outage longer than log retention** | Snapshot transfer, then log pull from the snapshot onward | | **New node joining** | Full snapshot from existing node | +A node's membership is persisted. A restarted node is not a quorum by itself, and a gracefully stopped node rejoins +on restart. A node marked REMOVED stays out until an operator allows it back (see Cluster Membership Management). + **Anti-Entropy Background Process:** -Marmot v2 includes an automatic anti-entropy system that continuously monitors and repairs replication lag across the cluster: - -1. **Lag Detection**: Every 30 seconds (configurable), each node queries peers for their replication state -2. **Smart Recovery Decision**: - - **Delta Sync** if lag < 10,000 transactions AND < 1 hour: Streams missed transactions incrementally - - **Snapshot Transfer** if lag exceeds thresholds: Full database file transfer for efficiency -3. **Gap Detection**: Detects when transaction logs have been GC'd and automatically falls back to snapshot -4. **Multi-Database Support**: Tracks and syncs each database independently -5. **GC Coordination**: Garbage collection respects peer replication state - logs aren't deleted until all peers have applied them - -**Delta Sync Process:** -1. Lagging node queries `last_applied_txn_id` for each peer/database -2. Requests transactions since that ID via `StreamChanges` RPC -3. **Gap Detection**: Checks if first received txn_id has a large gap from requested ID - - If gap > delta_sync_threshold_txns, indicates missing (GC'd) transactions - - Automatically falls back to snapshot transfer to prevent data loss -4. Applies changes using LWW conflict resolution -5. Updates replication state tracking (per-database) -6. Progress logged every 100 transactions - -**GC Coordination with Anti-Entropy:** -- Transaction logs are retained with a two-tier policy: - - **Min retention** (2 hours): Must be >= delta sync threshold, respects peer lag - - **Max retention** (24 hours): Force delete after this time to prevent unbounded growth -- Config validation enforces: `gc_min >= delta_threshold` and `gc_max >= 2x delta_threshold` -- Each database tracks replication progress per peer -- GC queries minimum applied txn_id across all peers before cleanup -- **Gap detection** prevents data loss if GC runs while nodes are offline +Every `anti_entropy_interval_seconds` (default 30), each node pulls every alive peer's local commit log, for every +database: + +1. **Durable cursors**: Each (peer, database) pair has its own cursor, persisted after every page, so a restart resumes + where it stopped. The node lists the peer's committed log entries after the cursor (`ListCommittedLog` RPC) and + fetches the transactions it lacks (`FetchTransactions` RPC). +2. **Exactly once**: A replayed transaction writes an applied marker (`__marmot_applied_txn`) in the same SQLite + transaction, so a transaction received from several peers applies once. +3. **Unfinished local transactions**: A transaction left prepared locally that a peer committed is committed locally + with the peer's commit timestamp. An abandoned local begin is discarded and the transaction replayed. A prepare + whose rows were lost is discarded and replayed from the peer. +4. **Snapshot fallback**: If a peer has truncated its log past this node's cursor, the node restores the database from a + snapshot, re-applies its own local log, and resumes pulling. +5. **Promotion**: A joining node turns ALIVE only once every database has caught up with its alive peers. + +**Log Garbage Collection:** +- A log entry is deleted once every current member has consumed it and it is older than `gc_min_retention_hours` +- An entry older than `gc_max_retention_hours` is deleted regardless; a member that was down longer restores by snapshot +- A departed member's consumed position is dropped, so it no longer holds back GC +- DELETE tombstones in the row-version table are purged in chunks, once they are older than the member count times + `gc_max_retention_hours`, and never when `gc_max_retention_hours` is 0 + +**Commit Timestamps and Per-Row Versions:** +- The coordinator merges the participants' PREPARE clocks and picks one commit HLC timestamp, sent on COMMIT and + stored in the log. Every node commits with it and merges it into its clock; on startup the clock is seeded past the + highest recorded commit. +- Every applied row change stamps `__marmot_row_version` (table, key, timestamp, node, tombstone flag) in the same + SQLite transaction. An incoming change applies only if its version is at least the stored one. +- A DELETE leaves a tombstone, so an older INSERT or UPDATE cannot bring the row back. A primary-key change stamps + both the old and the new key. `LOAD DATA` stamps every row. +- DDL keeps versions consistent: dropping a table removes its versions, renaming re-keys them, and changing a table's + primary key clears them. ### Consistency Guarantees @@ -802,8 +813,9 @@ Marmot v2 includes an automatic anti-entropy system that continuously monitors a | `ALL` | Returns after all nodes ACK (slow, most durable) | **Conflict Resolution:** -- All conflicts resolved via LWW using HLC timestamps -- No data loss - later write always wins deterministically +- Concurrent writes to the same row conflict at PREPARE (write intents); only one commits +- Replicated and replayed changes resolve per row via LWW on the commit HLC timestamp, so every node converges on the + same row regardless of the order changes arrive - Tie-breaker: higher node ID wins for equal timestamps ## Limitations @@ -840,7 +852,8 @@ TiDB's answer? "Use strings." But that breaks ORMs, existing application code, a ### Marmot's Solution: Compact ID Mode -Marmot offers **two ID generation modes** to solve this: +For `BIGINT AUTO_INCREMENT` columns, Marmot offers **two ID generation modes** to solve this (narrower columns are +covered in [Narrow AUTO_INCREMENT Columns](#narrow-auto_increment-columns)): | Mode | Bits | Range | Use Case | |------|------|-------|----------| @@ -869,32 +882,48 @@ JSON.parse('{"id": 4503599627370496}'); // {id: 4503599627370496} - Correct! ### How Auto-Increment Works -> **Note:** Marmot automatically converts `INT AUTO_INCREMENT` to `BIGINT` to support distributed ID generation. - -1. **DDL Transformation**: When you create a table with `AUTO_INCREMENT`: +1. **DDL Transformation**: An integer `AUTO_INCREMENT` column is stored as SQLite `INTEGER`. A column declared narrower + than `BIGINT` keeps its declared width (and `UNSIGNED`) in a comment marker in the stored schema. ```sql - CREATE TABLE users (id INT AUTO_INCREMENT PRIMARY KEY, name VARCHAR(100)) + CREATE TABLE users (id BIGINT AUTO_INCREMENT PRIMARY KEY, name VARCHAR(100)) -- Becomes internally: - CREATE TABLE users (id BIGINT PRIMARY KEY, name TEXT) + CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT) ``` -2. **DML ID Injection**: When inserting with `0` or `NULL` for an auto-increment column: +2. **DML ID Injection**: When inserting with `0` or `NULL` for a `BIGINT` auto-increment column: ```sql INSERT INTO users (id, name) VALUES (0, 'alice') -- Becomes internally (compact mode): INSERT INTO users (id, name) VALUES (4503599627370496, 'alice') ``` -3. **Explicit IDs Preserved**: If you provide an explicit non-zero ID, it is used as-is. +3. **Explicit IDs Preserved**: If you provide an explicit non-zero ID in a `BIGINT` column, it is used as-is. **Schema-Based Detection:** Marmot automatically detects auto-increment columns by querying SQLite schema directly: -- Single-column `INTEGER PRIMARY KEY` (SQLite rowid alias) +- Single-column `INTEGER PRIMARY KEY` (SQLite rowid alias), with the declared width read from its marker - Single-column `BIGINT PRIMARY KEY` (Marmot's transformed columns) No registration required - columns are detected from schema at runtime, works across restarts, and works with existing databases. +### Narrow AUTO_INCREMENT Columns + +`TINYINT`, `SMALLINT`, `MEDIUMINT` and `INT` `AUTO_INCREMENT` columns get ids that fit their declared width. Each node +takes ids from ranges of the column it claims through a quorum two-phase commit, so no two nodes issue the same id +while cluster membership is stable. + +- On a node that is still merging claim bases from its peers after startup, an insert waits up to + `transaction.lock_wait_timeout_seconds` and then returns the retryable 1205; inside a transaction that has already + written a row, it returns 1205 at once. +- An explicit id outside the declared width returns 1264; an id another node may issue returns 1235; a column whose + ids are used up returns 1062. +- Ids are not reset by `DROP TABLE`/`CREATE TABLE` or `DROP DATABASE`/`CREATE DATABASE`. + +The full rules are in [Compatibility](https://maxpert.github.io/marmot/compatibility#narrow-auto_increment-columns); +adding or removing nodes follows the procedure in +[Operations](https://maxpert.github.io/marmot/operations#membership-changes-and-narrow-auto_increment-ids). + ## Configuration Marmot v2 uses a TOML configuration file (default: `config.toml`). All settings have sensible defaults. @@ -915,6 +944,9 @@ conflict_window_seconds = 10 # Conflict resolution window lock_wait_timeout_seconds = 50 # Lock wait timeout (MySQL: innodb_lock_wait_timeout) ``` +An insert into a narrow `AUTO_INCREMENT` column also waits up to `lock_wait_timeout_seconds` while its node is still +merging claim bases after startup (see [Narrow AUTO_INCREMENT Columns](#narrow-auto_increment-columns)). + **Note**: Transaction log garbage collection is managed by the replication configuration to coordinate with anti-entropy. See `replication.gc_min_retention_hours` and `replication.gc_max_retention_hours`. ### Connection Pool @@ -957,6 +989,8 @@ gossip_interval_ms = 1000 # Gossip interval gossip_fanout = 3 # Number of peers to gossip to suspect_timeout_ms = 5000 # Suspect timeout dead_timeout_ms = 10000 # Dead timeout +autoinc_merge_interval_ms = 2000 # Narrow AUTO_INCREMENT: retry merging claim bases while votes are held +autoinc_base_sync_interval_ms = 10000 # Narrow AUTO_INCREMENT: raise claim bases to every alive peer's ``` ### Security @@ -1089,34 +1123,32 @@ default_read_consistency = "LOCAL_ONE" # Read consistency level write_timeout_ms = 5000 # Write operation timeout read_timeout_ms = 2000 # Read operation timeout -# Anti-Entropy: Background healing for eventual consistency -# - Detects and repairs divergence between replicas -# - Uses delta sync for small lags, snapshot for large lags -# - Includes gap detection to prevent incomplete data after GC -enable_anti_entropy = true # Enable automatic catch-up for lagging nodes -anti_entropy_interval_seconds = 30 # How often to check for lag (default: 30s) +# Anti-Entropy: each node pulls every alive peer's commit log (snapshot when a peer's log no longer covers it) +enable_anti_entropy = true # Enable the background log pull +anti_entropy_interval_seconds = 30 # How often each node pulls its peers' logs (default: 30s) gc_interval_seconds = 60 # GC interval (MUST be >= anti_entropy_interval) -delta_sync_threshold_transactions = 10000 # Delta sync if lag < 10K txns -delta_sync_threshold_seconds = 3600 # Snapshot if lag > 1 hour +delta_sync_threshold_transactions = 10000 # Read replicas only: snapshot instead of catch-up above this txn gap +delta_sync_threshold_seconds = 3600 # Only used by the GC retention checks below # Garbage Collection: Reclaim disk space by deleting old transaction records -# - gc_interval must be >= anti_entropy_interval (validated at startup) -# - gc_min must be >= delta_sync_threshold (validated at startup) -# - gc_max should be >= 2x delta_sync_threshold (recommended) +# - A log entry is deleted once every member consumed it and it is older than gc_min, +# or once it is older than gc_max regardless # - Set gc_max = 0 for unlimited retention -gc_min_retention_hours = 2 # Keep at least 2 hours (>= 1 hour delta threshold) -gc_max_retention_hours = 24 # Force delete after 24 hours +gc_min_retention_hours = 2 +gc_max_retention_hours = 24 ``` **Anti-Entropy Tuning:** - **Small clusters (2-3 nodes)**: Use default settings (30s AE, 60s GC) - **Large clusters (5+ nodes)**: Consider increasing AE interval to 60-120s and GC to 2x that value -- **High write throughput**: Increase `delta_sync_threshold_transactions` to 50000+ -- **Long-running clusters**: Keep `gc_max_retention_hours` at 24+ to handle extended outages +- **Long-running clusters**: Keep `gc_max_retention_hours` at 24+ so a node down for a while catches up from the log + instead of a snapshot -**GC Configuration Rules (Validated at Startup):** +**GC Configuration Rules (validated at startup when `enable_anti_entropy` is true):** +- `gc_interval_seconds` must be >= `anti_entropy_interval_seconds` - `gc_min_retention_hours` must be >= `delta_sync_threshold_seconds` (in hours) -- `gc_max_retention_hours` should be >= 2x `delta_sync_threshold_seconds` +- `gc_max_retention_hours` must be >= 2x `delta_sync_threshold_seconds` (in hours) and > `gc_min_retention_hours`, + unless it is 0 (unlimited) - Violating these rules will cause startup failure with helpful error messages ### Query Pipeline diff --git a/docs/src/pages/compatibility.mdx b/docs/src/pages/compatibility.mdx index ed25d67f..58d7482c 100644 --- a/docs/src/pages/compatibility.mdx +++ b/docs/src/pages/compatibility.mdx @@ -112,7 +112,7 @@ Marmot supports key MySQL functions for application compatibility: ### Auto-Increment ID Generation -Marmot generates unique distributed IDs for `AUTO_INCREMENT` columns. Two modes are available: +Marmot generates unique distributed IDs for `BIGINT AUTO_INCREMENT` columns. Two modes are available: | Mode | Bits | Max Value | Use Case | |------|------|-----------|----------| @@ -121,7 +121,7 @@ Marmot generates unique distributed IDs for `AUTO_INCREMENT` columns. Two modes Configure via `[mysql].auto_id_mode` in config. The compact mode uses: 41 bits timestamp + 6 bits node ID + 6 bits sequence. -These modes apply to `BIGINT AUTO_INCREMENT` columns. +These modes apply only to `BIGINT AUTO_INCREMENT` columns. Narrower columns use range claims instead (below). ### Narrow AUTO_INCREMENT Columns diff --git a/docs/src/pages/metrics.mdx b/docs/src/pages/metrics.mdx index 193d8589..67281d3a 100644 --- a/docs/src/pages/metrics.mdx +++ b/docs/src/pages/metrics.mdx @@ -64,10 +64,14 @@ curl http://localhost:8080/metrics | Metric | Type | Labels | Description | |--------|------|--------|-------------| | `marmot_v2_antientropy_rounds_total` | Counter | - | Total anti-entropy rounds executed | -| `marmot_v2_antientropy_syncs_total` | Counter | `type`, `result` | Anti-entropy syncs by type (delta, snapshot) and result | +| `marmot_v2_antientropy_syncs_total` | Counter | `type`, `result` | Snapshot restores by result (`success`, `failed`); `type` is always `snapshot` | | `marmot_v2_antientropy_duration_seconds` | Histogram | - | Anti-entropy round duration in seconds | -| `marmot_v2_replication_lag_txns` | Gauge | `peer` | Transaction lag behind peer | -| `marmot_v2_delta_sync_txns_total` | Counter | - | Total transactions applied via delta sync | +| `marmot_v2_log_pull_txns_applied_total` | Counter | - | Transactions applied by pulling peers' local commit logs | +| `marmot_v2_log_pull_pair_results_total` | Counter | `outcome` | Per-(peer, database) pulls by outcome: `caught_up`, `behind`, `needs_snapshot`, `unimplemented`, `error` | +| `marmot_v2_anti_entropy_stuck_txns` | Gauge | `database` | Pulled transactions no longer retried every round after repeated replay failures | +| `marmot_v2_replication_lag_txns` | Gauge | `peer` | Registered but not currently updated | + +There is no delta sync counter: anti-entropy is a log pull, counted by the `log_pull_*` metrics. ## Histogram Buckets diff --git a/docs/src/pages/reference.mdx b/docs/src/pages/reference.mdx index 6eda0932..564101c6 100644 --- a/docs/src/pages/reference.mdx +++ b/docs/src/pages/reference.mdx @@ -82,8 +82,12 @@ gossip_interval_ms = 1000 # Gossip interval gossip_fanout = 3 # Number of peers to gossip to suspect_timeout_ms = 5000 # Suspect timeout dead_timeout_ms = 10000 # Dead timeout +autoinc_merge_interval_ms = 2000 # Retry interval for merging narrow AUTO_INCREMENT claim bases while votes are held +autoinc_base_sync_interval_ms = 10000 # Interval for raising claim bases to every alive peer's ``` +The two `autoinc_*` intervals drive narrow (`TINYINT`/`SMALLINT`/`MEDIUMINT`/`INT`) `AUTO_INCREMENT` id ranges. `0` selects the default and negative values fail validation. `autoinc_base_sync_interval_ms` bounds how long a claim committed before a membership change can stay unknown to a new quorum. See [Compatibility](/compatibility) and [Operations](/operations). + ## Replication ```toml @@ -93,36 +97,46 @@ default_read_consistency = "LOCAL_ONE" # Read consistency level write_timeout_ms = 5000 # Write operation timeout read_timeout_ms = 2000 # Read operation timeout -# Anti-Entropy: Background healing for eventual consistency -# - Detects and repairs divergence between replicas -# - Uses delta sync for small lags, snapshot for large lags -# - Includes gap detection to prevent incomplete data after GC -enable_anti_entropy = true # Enable automatic catch-up for lagging nodes -anti_entropy_interval_seconds = 30 # How often to check for lag (default: 30s) +# Anti-Entropy: each round pulls every alive peer's local commit log from a +# durable per-(peer, database) cursor; falls back to a snapshot when a peer's +# log is truncated past that cursor +enable_anti_entropy = true # Enable anti-entropy log pull +anti_entropy_interval_seconds = 30 # Interval between rounds (default: 30s) gc_interval_seconds = 60 # GC interval (MUST be >= anti_entropy_interval) -delta_sync_threshold_transactions = 10000 # Delta sync if lag < 10K txns -delta_sync_threshold_seconds = 3600 # Snapshot if lag > 1 hour - -# Garbage Collection: Reclaim disk space by deleting old transaction records -# - gc_interval must be >= anti_entropy_interval (validated at startup) -# - gc_min must be >= delta_sync_threshold (validated at startup) -# - gc_max should be >= 2x delta_sync_threshold (recommended) -# - Set gc_max = 0 for unlimited retention -gc_min_retention_hours = 2 # Keep at least 2 hours (>= 1 hour delta threshold) -gc_max_retention_hours = 24 # Force delete after 24 hours +delta_sync_threshold_transactions = 10000 # Read replica only: bootstrap snapshot if gap > this +delta_sync_threshold_seconds = 3600 # Only used by the GC validation rules below + +# Garbage Collection: deletes committed log entries once every current member +# has consumed them and they are older than gc_min, or unconditionally once +# they are older than gc_max +gc_min_retention_hours = 2 # Minimum age before a consumed entry is deleted +gc_max_retention_hours = 24 # Entries older than this are deleted regardless; 0 = no maximum ``` **Anti-Entropy Tuning:** - **Small clusters (2-3 nodes)**: Use default settings (30s AE, 60s GC) - **Large clusters (5+ nodes)**: Consider increasing AE interval to 60-120s and GC to 2x that value -- **High write throughput**: Increase `delta_sync_threshold_transactions` to 50000+ -- **Long-running clusters**: Keep `gc_max_retention_hours` at 24+ to handle extended outages +- **Long-running clusters**: Keep `gc_max_retention_hours` at 24+ to handle extended outages. A node whose cursor falls behind a peer's truncated log restores that database by snapshot +- `gc_max_retention_hours = 0` sets no maximum: an entry is kept until every current member has consumed it, and row-version tombstones are never purged + +**Legacy `delta_sync_*` keys:** cluster nodes no longer run a delta sync. `delta_sync_threshold_transactions` is read only by a read replica at bootstrap: a database more than this many transactions behind its source is restored by snapshot, a smaller gap is streamed with `StreamChanges`. `delta_sync_threshold_seconds` has no runtime effect; it only feeds the GC rules below. -**GC Configuration Rules (Validated at Startup):** +**GC Configuration Rules (Validated at Startup when `enable_anti_entropy` is true):** - `gc_min_retention_hours` must be >= `delta_sync_threshold_seconds` (in hours) -- `gc_max_retention_hours` should be >= 2x `delta_sync_threshold_seconds` +- `gc_max_retention_hours` must be >= 2x `delta_sync_threshold_seconds` (in hours), unless it is `0` +- `gc_min_retention_hours` must be < `gc_max_retention_hours` (when it is not `0`) +- `gc_interval_seconds` must be >= `anti_entropy_interval_seconds` - Violating these rules will cause startup failure with helpful error messages +## Metastore + +```toml +[metastore] +strict_prepare_sync = false # Wait for CDC segment fsync before acknowledging PREPARE +``` + +`strict_prepare_sync = true` (or `MARMOT_CDC_PREPARE_SYNC=strict`) strengthens local crash durability at the cost of write latency. The default groups segment syncs asynchronously. + ## Query Pipeline ```toml diff --git a/docs/src/pages/replication.mdx b/docs/src/pages/replication.mdx index c704ea2c..103cefb8 100644 --- a/docs/src/pages/replication.mdx +++ b/docs/src/pages/replication.mdx @@ -109,7 +109,7 @@ This keeps `LOAD DATA LOCAL INFILE` deterministic and durable across cluster mem **Step 1: Node 1 (Coordinator) executes locally** ``` -Time: T1 (TxnID = 0x0001A2B3C4D50001) +Time: T1 (TxnID = 0x...0001) Physical: 1736000000000ms | NodeID: 1 | Logical: 1 Node 1 executes UPDATE on hookDB connection @@ -121,10 +121,10 @@ Node 1 executes UPDATE on hookDB connection **Step 2: Coordinator creates write intent** ``` -MetaStore Write on Node 1: - Key: /intent/users/users:1 +Row lock on Node 1 (in memory): + Key: users/users:1 Value: { - TxnID: 0x0001A2B3C4D50001, + TxnID: 0x...0001, NodeID: 1, StartTS: {Wall: 1736000000000000000, Logical: 1}, IntentType: UPDATE, @@ -138,7 +138,7 @@ MetaStore Write on Node 1: gRPC ReplicateTransaction (parallel to both): { Phase: PREPARE, - TxnID: 0x0001A2B3C4D50001, + TxnID: 0x...0001, NodeID: 1, Database: "mydb", Statements: [{ @@ -163,13 +163,13 @@ gRPC ReplicateTransaction (parallel to both): ``` Node 2: 1. Check Cuckoo filter for "users:1" → MISS (no conflict) - 2. Create write intent: /intent/users/users:1 + 2. Acquire row lock users/users:1 (in memory) 3. Append encoded row to local cdcseg segment and publish transaction manifest 4. Return: {Status: SUCCESS, PrepareTS: ...} Node 3: 1. Check Cuckoo filter for "users:1" → MISS (no conflict) - 2. Create write intent: /intent/users/users:1 + 2. Acquire row lock users/users:1 (in memory) 3. Append encoded row to local cdcseg segment and publish transaction manifest 4. Return: {Status: SUCCESS, PrepareTS: ...} ``` @@ -182,7 +182,7 @@ Node 1 received 2 SUCCESS responses (quorum = 2 for 3-node cluster) gRPC ReplicateTransaction to Node 2, Node 3: { Phase: COMMIT, - TxnID: 0x0001A2B3C4D50001, + TxnID: 0x...0001, CommitTS: {Wall: 1736000000005000000, Logical: 0} } ``` @@ -317,10 +317,10 @@ source may apply that source's order. When Node 1 executes `UPDATE users SET balance = 75 WHERE id = 1`: -**DML payload stored in `cdcseg` and referenced by `/cdc_manifest/{txnID}`**: +**DML payload stored in `cdcseg` and referenced by `/cdc/manifest/{txnID}`**: ``` -Manifest: /cdc_manifest/0x0001A2B3C4D50001 +Manifest: /cdc/manifest/0x...0001 Segment range: cdcseg-00000042.log offset=8192 length=148 EncodedCapturedRow (msgpack): @@ -421,13 +421,11 @@ IntentKey Format: "order_items:c:MTAw:NDI=" Write intents are **distributed locks** stored in MetaStore: ``` -Key: /intent/{tableName}/{intentKey} +Row lock: (table, intentKey) → txnID, held in memory (RowLockStore); non-DML intents are also persisted under /intent_txn/{txnID}/{table}/{intentKey} Value: WriteIntentRecord { TxnID, NodeID, StartTS, IntentType, Statement, DataSnapshot, MarkedForCleanup } - -Index: /intent_txn/{txnID}/{tableName}/{intentKey} // For cleanup ``` ### Cuckoo Filter Fast Path @@ -493,15 +491,15 @@ Time T2: Node 2 starts UPDATE users SET balance = 30 WHERE id = 2 ``` Time T1: Node 1 starts UPDATE users SET balance = 75 WHERE id = 1 - TxnID: 0x0001A2B3C4D50001 - → Creates intent: /intent/users/users:1 → {TxnID: 0x0001A2B3C4D50001} + TxnID: 0x...0001 + → Creates intent: row lock users/users:1 → {TxnID: 0x...0001} → Adds to Cuckoo filter: hash("users:1") Time T2: Node 2 starts UPDATE users SET balance = 50 WHERE id = 1 - TxnID: 0x0001A2B3C4D60001 + TxnID: 0x...0002 → Cuckoo filter check for "users:1" → HIT - → Lookup Pebble: /intent/users/users:1 - → Found intent with different TxnID (0x0001A2B3C4D50001) + → Look up row lock: users/users:1 + → Found intent with different TxnID (0x...0001) → Check if overwritable: - MarkedForCleanup = false - Status = PENDING (not COMMITTED/ABORTED) @@ -514,7 +512,7 @@ Time T2: Node 2 starts UPDATE users SET balance = 50 WHERE id = 1 ``` Time T1: Node 1 starts UPDATE users SET balance = 75 WHERE id = 1 - → Creates intent: /intent/users/users:1 + → Creates intent: row lock users/users:1 → Node 1 crashes before COMMIT Time T2: (10 seconds later, heartbeat timeout) @@ -524,7 +522,7 @@ Time T2: (10 seconds later, heartbeat timeout) Time T3: Node 3 starts UPDATE users SET balance = 50 WHERE id = 1 → Cuckoo filter check → HIT - → Lookup Pebble → Found intent + → Look up row lock → Found intent → Check: MarkedForCleanup = true → Overwrite allowed → Creates new intent → Transaction succeeds @@ -534,11 +532,11 @@ Time T3: Node 3 starts UPDATE users SET balance = 50 WHERE id = 1 ``` Time T1: Node 1 PREPARE phase creates intent for "users:1" - TxnID: 0x0001A2B3C4D50001 + TxnID: 0x...0001 Time T2: Network hiccup, Node 1 retries PREPARE → Cuckoo filter check → HIT - → Lookup Pebble → Found intent with SAME TxnID + → Look up row lock → Found intent with SAME TxnID → Update in-place (idempotent) → No conflict ``` @@ -555,8 +553,8 @@ COMMIT; ``` Time T1: Node 1 creates intents for both rows: - → /intent/users/users:1 → TxnID: 0x...0001 - → /intent/users/users:2 → TxnID: 0x...0001 + → row lock users/users:1 → TxnID: 0x...0001 + → row lock users/users:2 → TxnID: 0x...0001 Time T2: Node 2 tries UPDATE users SET name = 'Robert' WHERE id = 2 → Cuckoo filter check for "users:2" → HIT @@ -591,19 +589,24 @@ data_dir/ | Prefix | Purpose | |--------|---------| -| `/txn/{txnID}` | Transaction records (status, timestamps, statements) | +| `/txn/{txnID}` | Immutable transaction record (id, node, start ts, database) | +| `/txn_status/{txnID}` | Transaction status | +| `/txn_commit/{txnID}` | Commit record | +| `/txn_idx/txnid/{txnID}` | Transaction index by id (used for streaming) | +| `/txn_prepared/{txnID}` | Prepared payload | | `/txn_idx/pend/{txnID}` | Pending transaction index | -| `/txn_idx/seq/{seqNum}/{txnID}` | Sequence-ordered index (gap detection) | -| `/intent/{table}/{intentKey}` | Write intents | -| `/intent_txn/{txnID}/{table}/{intentKey}` | Intent index by transaction | -| `/cdc_manifest/{txnID}` | Manifest pointing to the transaction's CDC segment-log range | -| `/repl/{peerID}/{dbName}` | Replication state per peer | -| `/schema/{dbName}` | Schema version counter | -| `/seq/{nodeID}` | Sequence generator (pre-allocated batches) | +| `/txn_idx/seq/{seqNum}/{txnID}` | Local commit log, in position order | +| `/intent_txn/{txnID}/{table}/{intentKey}` | Non-DML intents by transaction (DML row locks are in memory) | +| `/cdc/manifest/{txnID}` | Manifest pointing to the transaction's CDC segment-log range | +| `/log_seq/current` | Lease end of the local commit log's sequence | +| `/log_pull_cursor/{peerID}` | This node's pull cursor into a peer's log | +| `/log_consumed_pos/{peerID}` | How far a peer has consumed this node's log (for GC) | +| `/log_truncated_through` | Highest log position GC has deleted through | +| `/log_reapply_pending` | Local-log re-apply after a snapshot restore is pending | ### CDC Segment Store -The CDC segment store is the local append-only payload store for DML row changes. Pebble keeps the transaction metadata, row-lock indexes, and `/cdc_manifest/{txnID}` pointer; the actual encoded row payloads live in `cdcseg/seg-*.log`. +The CDC segment store is the local append-only payload store for DML row changes. Pebble keeps the transaction metadata, row-lock indexes, and `/cdc/manifest/{txnID}` pointer; the actual encoded row payloads live in `cdcseg/seg-*.log`. Each row record is: @@ -619,7 +622,7 @@ CDC segment publication is manifest-based: 1. Encoded row records are appended to the current segment file. 2. The in-memory transaction manifest tracks row count, first/last row sequence, CRC, and one or more `{segmentID, offset, length}` chunks. -3. `SealCapturedRows` publishes `/cdc_manifest/{txnID}` in Pebble after the segment ranges are known. +3. `SealCapturedRows` publishes `/cdc/manifest/{txnID}` in Pebble after the segment ranges are known. 4. Readers never scan arbitrary segment bytes for committed DML; they read the manifest and then perform bounded `ReadAt` calls over the referenced segment ranges. This gives atomic visibility: a transaction's CDC payload is visible only when its manifest is visible. A crash before manifest publication leaves unreferenced segment bytes, which are ignored and later reclaimed. A crash after manifest publication leaves a manifest that points to specific validated byte ranges. @@ -631,7 +634,7 @@ By default Marmot keeps the hot path aligned with Pebble `NoSync`: CDC segment f For deployments that require a PREPARE acknowledgement to wait for local CDC segment fsync, enable strict prepare sync: ```toml -[meta_store] +[metastore] strict_prepare_sync = true ``` @@ -641,7 +644,7 @@ or set `MARMOT_CDC_PREPARE_SYNC=strict`. Strict mode waits for the segment sync On startup, Marmot validates each segment record in order. If it finds a torn header, short payload, invalid magic/version, oversized payload, or CRC mismatch, it truncates the active tail to the last valid record boundary. -Prepared manifests embedded in segment records can reconstruct prepared CDC state if Pebble metadata was lost after the segment record became durable. For committed transactions, `/cdc_manifest/{txnID}` is the retention root used by streaming replication, catch-up, and delta sync. +Prepared manifests embedded in segment records can reconstruct prepared CDC state if Pebble metadata was lost after the segment record became durable. For committed transactions, `/cdc/manifest/{txnID}` is the retention root used by read-replica streaming, catch-up, and the anti-entropy log pull (`FetchTransactions`). GC deletes segment files only when no current manifest, pending prepare, or active writer can reference them. Current and pending segments are retained; old unreferenced segments are pruned with the transaction metadata retention rules. @@ -651,7 +654,7 @@ GC deletes segment files only when no current manifest, pending prepare, or acti type TransactionRecord struct { TxnID uint64 NodeID uint64 - SeqNum uint64 // For gap-free replication + SeqNum uint64 // Position in this node's local commit log Status TxnStatus // PENDING/COMMITTED/ABORTED StartTSWall int64 // HLC wall time StartTSLogical int32 // HLC logical @@ -673,18 +676,17 @@ After Node 1's `UPDATE users SET balance = 75 WHERE id = 1` completes: **During PREPARE phase (all 3 nodes)**: ``` -/intent/users/users:1 → WriteIntentRecord{TxnID: 0x0001A2B3C4D50001, ...} -/intent_txn/0x0001A2B3C4D50001/users/users:1 → (index entry) -/cdc_manifest/0x0001A2B3C4D50001 → CDCManifest{SegmentID, Offset, Length, RowCount} -/txn/0x0001A2B3C4D50001 → TransactionRecord{Status: PENDING, ...} -/txn_idx/pend/0x0001A2B3C4D50001 → (pending index) +row lock users/users:1 → TxnID 0x...0001 (in memory, not a Pebble key) +/cdc/manifest/0x...0001 → CDCManifest{SegmentID, Offset, Length, RowCount} +/txn/0x...0001 → TransactionRecord{Status: PENDING, ...} +/txn_idx/pend/0x...0001 → (pending index) ``` **After COMMIT phase**: ``` -/txn/0x0001A2B3C4D50001 → TransactionRecord{ - TxnID: 0x0001A2B3C4D50001, +/txn/0x...0001 → TransactionRecord{ + TxnID: 0x...0001, NodeID: 1, SeqNum: 42, Status: COMMITTED, @@ -692,25 +694,25 @@ After Node 1's `UPDATE users SET balance = 75 WHERE id = 1` completes: CommitTSWall: 1736000000005000000, DatabaseName: "mydb" } -/txn_idx/seq/42/0x0001A2B3C4D50001 → (sequence index for gap detection) +/txn_idx/seq/42/0x...0001 → (local commit log entry at seq 42) -- Deleted (cleanup): -/intent/users/users:1 → (removed) -/intent_txn/0x0001A2B3C4D50001/users/users:1 → (removed) -/txn_idx/pend/0x0001A2B3C4D50001 → (removed) +row lock users/users:1 → (released) +/txn_idx/pend/0x...0001 → (removed) --- CDC segment range kept for anti-entropy (GC'd after retention rules allow it): -/cdc_manifest/0x0001A2B3C4D50001 → (kept for delta sync) +-- CDC segment range kept for peers' log pulls (GC'd after retention rules allow it): +/cdc/manifest/0x...0001 → (kept for peers' log pulls) ``` -**Replication State (per peer)**: +**Log Pull State (per peer, in mydb's MetaStore)**: ``` --- On Node 2, tracking what Node 1 has sent: -/repl/node1/mydb → {LastAppliedTxnID: 0x0001A2B3C4D50001, LastSeqNum: 42} +-- On Node 2: its cursor into Node 1's log, and how far Node 1 has consumed Node 2's log +/log_pull_cursor/node1 → (42, 0x...0001) +/log_consumed_pos/node1 → (last position Node 1 reported) --- On Node 3: -/repl/node1/mydb → {LastAppliedTxnID: 0x0001A2B3C4D50001, LastSeqNum: 42} +-- Seq is local: Node 2 logged the same transaction at its own position +/txn_idx/seq/{node2 seq}/0x...0001 ``` --- @@ -758,45 +760,43 @@ On receiving remote timestamp: 1. Take max of: local wall, remote wall, physical now 2. If same wall time: increment logical beyond both local and remote -3. If new millisecond: reset logical to 0 -4. **Overflow protection**: Spin-wait if logical exceeds 65535 +3. If the remote wall is the max: logical = remote logical + 1 +4. If physical time is ahead of both and the millisecond changed: reset logical to 0 +5. **Overflow protection**: Spin-wait if logical exceeds 65535 ### Example: HLC and TxnID Generation **Our UPDATE transaction on Node 1**: ``` -Physical time: 1736000000000 ms since epoch (Jan 5, 2025 ~12:26 UTC) +Physical time: 1736000000000 ms since epoch (Jan 4, 2025 14:13:20 UTC) Node ID: 1 Logical: 1 (first transaction this millisecond) TxnID calculation: - Physical bits (42): 1736000000000 << 22 = 0x0001A2B3C4D50000 + Physical bits (42): 1736000000000 << 22 = 0x650C6A7400000000 NodeID bits (6): 1 << 16 = 0x0000000000010000 Logical bits (16): 1 = 0x0000000000000001 ───────────────────────────────────────────────────────────── - TxnID: 0x0001A2B3C4D50001 + TxnID: 0x650C6A7400010001 Binary layout: |---- 42 bits ----|-- 6 --|-- 16 --| | 1736000000000 | 1 | 1 | ``` +Later examples abbreviate this id as `0x...0001`. + **Conflict resolution using HLC** (Last-Write-Wins): ``` Scenario: Two updates to same row arrive at Node 3 -Update A (from Node 1): - TxnID: 0x0001A2B3C4D50001 - Wall: 1736000000000ns, Logical: 1 - -Update B (from Node 2): - TxnID: 0x0001A2B3C4D60002 - Wall: 1736000000001ns, Logical: 2 - -Comparison: B.Wall > A.Wall → Update B wins -Result: Node 3 applies Update B (higher timestamp) +Update A: commit ts (wall=1736000000000000000, logical=1, node=1) +Update B: commit ts (wall=1736000000001000000, logical=0, node=2) +Node 3 stamps __marmot_row_version with each applied change's commit ts; +a change older than the stored (wall, logical, node) is skipped. +B.wall > A.wall → B wins in either arrival order. Equal wall and logical → higher node wins. ``` **Clock advance on message receive**: @@ -808,9 +808,9 @@ Receives PREPARE from Node 1: Wall=1736000000002, Logical=1 Update logic: localWall (1736000000000) < remoteWall (1736000000002) → Set wall = max(local, remote, physicalNow) = 1736000000002 - → Reset logical = 0 (new millisecond) + → Remote wall is the max: logical = remote logical + 1 = 2 -Node 2 new clock: Wall=1736000000002, Logical=0 +Node 2 new clock: Wall=1736000000002, Logical=2 ``` --- @@ -824,148 +824,147 @@ Repairs divergence caused by: - Node restarts with stale data - Missed transactions during temporary failures -### Sync Strategy Decision +### Local Commit Log -``` -Lag Detection: - localMaxTxnID vs peerMaxTxnID - localTxnCount vs peerTxnCount +Every node keeps a commit log per database. Each committed transaction gets a position `(seq, txnID)` in it, whether this node coordinated it, took part in its 2PC, or pulled it from a peer. Positions are local: the same transaction has a different `seq` on each node. -Decision: - IF lag < DeltaSyncThreshold (default 10K txns) - AND timeLag < DeltaSyncThresholdSeconds (default 1hr) - THEN → Delta Sync (incremental) - ELSE → Snapshot Transfer (full) -``` +Because a pulled transaction is logged too, a node can receive any transaction from any member that holds it, even when no single peer holds everything it lacks. -### Delta Sync Flow +### Log Pull Flow -1. **Query peer's replication state** -2. **Stream transactions** via `StreamChanges` RPC -3. **Gap detection**: If `event.SeqNum - expectedNext > threshold` → snapshot fallback -4. **Apply each transaction** via REPLAY phase (bypasses 2PC) -5. **Update local replication state** +Anti-entropy runs every `anti_entropy_interval_seconds`. Each round: -### Gap Detection Algorithm +1. **Reconcile databases**: merge every ALIVE peer's database registry (`ListDatabaseRegistry`), so a missed CREATE or DROP DATABASE is repaired before any log is pulled. +2. **Pull every ALIVE peer's log** for each database. Each (peer, database) pair has its own deadline, so one slow peer does not hold up the others. -```go -if event.SeqNum > expectedNextTxn { - gap := event.SeqNum - expectedNextTxn - if gap > DeltaSyncThresholdTxns { - // Transactions GC'd - fall back to snapshot - return ErrGapDetected - } -} -expectedNextTxn = event.SeqNum + 1 -``` +One pull for one pair: + +1. **List**: `ListCommittedLog` returns up to 256 entries after the pair's cursor, the peer's truncation point and its schema version. The request also carries how far this node has consumed the peer's log; the peer stores that position for its GC. +2. **Diff**: an entry whose txn id already has a `__marmot_applied_txn` marker is covered and skipped. Every commit writes its marker in the same SQLite transaction as its rows, whether it came through 2PC or a pull. +3. **Fetch**: missing ids are fetched with `FetchTransactions`. The peer fails the call for a missing, uncommitted or incomplete transaction rather than send part of one. +4. **Apply**: each fetched transaction is first written to this node's own log, then applied in one SQLite transaction together with its marker. Rows are stamped with the transaction's commit timestamp in `__marmot_row_version`, and a row a newer version owns is skipped. DDL in the transaction bumps `__marmot_schema_version` in that same SQLite transaction. The commit timestamp is merged into this node's clock. +5. **Advance the cursor**: the pair's cursor moves over the contiguous prefix of covered entries and is persisted after every page. An entry that cannot be applied yet (a row held by a local write intent, say) keeps the cursor before it while the entries after it still apply. + +The marker makes replay idempotent: a crash between logging and applying re-applies the transaction on the next pass, and a transaction reached through two peers applies once. A transaction whose replay fails 5 times in a row is reported STUCK and then retried every 20th pull of its pair. + +### Unfinished Local Transactions + +Every listed entry is committed in the peer's log. When this node holds the same transaction unfinished: + +| Local state | Action | +|-------------|--------| +| Prepared | Committed locally with the peer's commit timestamp | +| Prepared, but its captured rows were lost (crash before sync) | Prepare discarded; the peer's copy is fetched and replayed | +| Begun, and its PREPARE died with an earlier process | Discarded; the peer's copy is fetched and replayed | +| Begun, PREPARE still running | Left for a later pull | + +### Caught Up and Promotion + +A database is caught up when, in the last round, every member's log was reachable and this node's cursor reached each one's stable point. + +A JOINING node turns ALIVE only when, for every database, at least one ALIVE peer was pulled and every ALIVE peer's pull succeeded, reached the end of the log and needed no snapshot, and its schema versions match or exceed every ALIVE peer's. The check covers ALIVE peers only, so a member that stays down does not keep a restarted node JOINING. + +### Log Truncation -### Snapshot Transfer +GC deletes a prefix of the local log: -1. **Get snapshot info**: Files, checksums, txn_id -2. **Stream chunks**: 4MB each with MD5 verification -3. **Atomic restore**: Replace database files -4. **Resume normal replication** from snapshot txn_id +- An entry older than `gc_min_retention_hours` is deleted once every current member has consumed past it. A member that has never reported a position counts as position zero and blocks the deletion. +- An entry older than `gc_max_retention_hours` is deleted regardless. `0` sets no maximum. + +A departed member's consumed position is dropped, so it no longer blocks GC. The truncation point only moves forward. + +### Snapshot Fallback + +When a peer's truncation point is past this node's cursor into its log, the pull applies nothing and reports that a snapshot is needed. The restore: + +1. Durably marks a re-apply of this node's own log as pending. +2. Restores the database from the ALIVE peer with the highest schema version (ties go to the lowest node id): `GetSnapshotInfo`, then `StreamSnapshot` in `stream_chunk_size_kb` chunks, each checked with MD5 and the file with SHA-256. +3. Re-applies every entry of this node's own log that the restored file lacks (the markers say which). A failure leaves the mark set, and the next round retries. +4. On the next pull from each peer, sets that pair's cursor to the peer's truncation point. + +### Startup Catch-Up + +A starting node picks a seed. With no data of its own it restores a full snapshot. With data, it reconciles the database registry and pulls the seed's log for every database, restoring any database whose cursor the seed has truncated past. A node the pull finds behind is marked JOINING until the promotion check passes. + +### Mixed-Version Clusters + +A peer that does not serve `ListCommittedLog` is skipped for the round and counts as not caught up. DDL and CREATE/DROP DATABASE return error 1213 (retryable) cluster-wide while any member does not report the log-pull protocol, on the coordinator and on each participant's PREPARE. + +### Read Replicas + +`StreamChanges` serves read replicas only. A replica starting up compares its max txn id per database with its source's: a gap above `delta_sync_threshold_transactions` takes a snapshot, a smaller one is streamed. Cluster nodes do not use either threshold for catch-up; `delta_sync_threshold_seconds` is only read by config validation, which, with anti-entropy enabled, requires `gc_min_retention_hours` to be at least that long and a `gc_max_retention_hours` above 0 to be at least twice that. ### Example: Anti-Entropy Scenarios Using our 3-node cluster with the `users` table: -**Scenario 1: Node 3 Brief Network Partition (Delta Sync)** +**Scenario 1: Node 3 Brief Network Partition (Log Pull)** ``` Timeline: - T0: All nodes in sync, SeqNum=40 + T0: All nodes in sync T1: Node 3 loses network for 5 minutes - T2: Node 1, Node 2 continue processing: - - SeqNum 41: UPDATE users SET balance=75 WHERE id=1 - - SeqNum 42: INSERT INTO users VALUES (3, 'carol@...', 'Carol', 200) - - SeqNum 43: DELETE FROM users WHERE id=2 + T2: Node 1, Node 2 continue processing (quorum 2 of 3): + - TxnID ...0001: UPDATE users SET balance=75 WHERE id=1 + - TxnID ...0002: INSERT INTO users VALUES (3, 'carol@...', 'Carol', 200) + - TxnID ...0003: DELETE FROM users WHERE id=2 T3: Node 3 reconnects -Anti-Entropy on Node 3: - 1. Query Node 1: "What's your max SeqNum?" → 43 - 2. Local state: LastSeqNum=40 - 3. Lag = 43 - 40 = 3 transactions (< 10K threshold) - 4. Decision: Delta Sync - -Delta Sync Flow: - Node 3 → Node 1: StreamChanges(fromSeqNum=41) - Node 1 streams: - {SeqNum: 41, TxnID: 0x...0001, CDC: [UPDATE users...]} - {SeqNum: 42, TxnID: 0x...0002, CDC: [INSERT users...]} - {SeqNum: 43, TxnID: 0x...0003, CDC: [DELETE users...]} - - Node 3 applies each via REPLAY (bypasses 2PC): - - Execute CDC for SeqNum 41 → balance=75 for id=1 - - Execute CDC for SeqNum 42 → insert id=3 - - Execute CDC for SeqNum 43 → delete id=2 +Anti-Entropy on Node 3, pair (Node 1, mydb): + Node 3 → Node 1: ListCommittedLog(after=cursor, consumed=cursor) + Node 1 returns: [(41, ...0001), (42, ...0002), (43, ...0003)], truncated=(0,0) + Markers: none of the three applied + Node 3 → Node 1: FetchTransactions([...0001, ...0002, ...0003]) + Node 3 applies each with its marker and row versions: + - ...0001 → balance=75 for id=1 + - ...0002 → insert id=3 + - ...0003 → delete id=2, tombstone for users:2 + Cursor for (Node 1, mydb) → (43, ...0003) - Node 3 now in sync: SeqNum=43 +Pair (Node 2, mydb): + Node 2's log lists the same three txns at its own positions + Markers: all three applied → covered, nothing fetched + Cursor for (Node 2, mydb) advances ``` **Scenario 2: New Node Joining (Snapshot Transfer)** ``` Initial state: - Node 1: SeqNum=50000, users table has 10K rows - Node 2: SeqNum=50000 + Node 1, Node 2: users table has 10K rows Node 4: New node, empty database -Node 4 joins cluster: - 1. Gossip announces Node 4 as JOINING - 2. Anti-entropy runs on Node 4 - 3. Query Node 1: "What's your max SeqNum?" → 50000 - 4. Local state: SeqNum=0 - 5. Lag = 50000 (> 10K threshold) - 6. Decision: Snapshot Transfer - -Snapshot Transfer Flow: - Node 4 → Node 1: GetSnapshotInfo(db="mydb") - Node 1 returns: - { - Files: ["mydb.db", "mydb.db-wal"], - TotalSize: 50MB, - Checksums: {...}, - AtTxnID: 0x...C350, - AtSeqNum: 50000 - } - - Node 4 → Node 1: StreamSnapshot(db="mydb") - Node 1 streams 4MB chunks: - Chunk 1: bytes[0..4MB], MD5=abc123 - Chunk 2: bytes[4MB..8MB], MD5=def456 - ... - Chunk 13: bytes[48MB..50MB], MD5=xyz789 - - Node 4 assembles and verifies: - - Write chunks to temp file - - Verify each chunk MD5 - - Atomic rename to mydb.db - - Set replication state: SeqNum=50000 - - Node 4 state: JOINING → ALIVE +Node 4 starts: + 1. Startup catch-up: no local data → full snapshot from a seed + 2. Node 4 → Node 1: GetSnapshotInfo(db="mydb") + Node 4 → Node 1: StreamSnapshot(db="mydb") + chunks of stream_chunk_size_kb, each with an MD5, + file verified with SHA-256 + 3. Anti-entropy rounds pull Node 1's and Node 2's logs; + transactions already in the snapshot have markers and are skipped + 4. Every database promotion-ready, schema versions match + 5. Node 4: JOINING → ALIVE ``` -**Scenario 3: Gap Detection (Partial GC)** +**Scenario 3: Cursor Behind the Truncation Point** ``` Timeline: - T0: Node 3 offline for 25 hours - T1: GC runs on Node 1, Node 2: - - Transactions SeqNum 1-1000 are > 24hrs old - - Force GC deletes them (gc_max_retention=24hr) - T2: Node 3 reconnects, has SeqNum=500 + T0: Node 3 goes offline + T1: Entries on Node 1 pass gc_max_retention_hours (24hr) + GC deletes them even though Node 3 has not consumed them + Node 1 truncation point → (1000, ...) + T2: Node 3 reconnects, its cursor into Node 1's log is (500, ...) -Delta Sync Attempt: - Node 3 → Node 1: StreamChanges(fromSeqNum=501) - Node 1 scans /txn_idx/seq/: - - First available: SeqNum=1001 - - Requested: SeqNum=501 - - Gap detected: 501-1000 are missing (GC'd) +Pull (Node 1, mydb): + ListCommittedLog returns truncated=(1000, ...) + cursor (500) < truncated (1000) → needs snapshot, nothing applied - Node 1 returns: ErrGapDetected - - Node 3 falls back to Snapshot Transfer +Snapshot fallback on Node 3: + 1. Mark re-apply of its own log pending + 2. Restore mydb from the ALIVE peer with the highest schema version + 3. Re-apply entries of Node 3's own log the restored file lacks + 4. Next pull from each peer starts at that peer's truncation point ``` --- @@ -977,9 +976,10 @@ Delta Sync Attempt: | State | Description | Replication | |-------|-------------|-------------| | `ALIVE` | Healthy, responding | Receives writes | -| `SUSPECT` | May be dead, awaiting confirmation | Receives writes | +| `SUSPECT` | May be dead, awaiting confirmation | Excluded | | `DEAD` | Confirmed failed | Excluded | | `JOINING` | Catching up | Gossip only | +| `LEAVING` | Graceful shutdown in progress | Excluded | | `REMOVED` | Admin removed | Excluded | ### State Transitions @@ -989,14 +989,18 @@ New node → JOINING → (catch-up complete) → ALIVE ALIVE → (no response for SuspectTimeout) → SUSPECT SUSPECT → (no response for DeadTimeout) → DEAD SUSPECT → (received message) → ALIVE +SUSPECT/DEAD → (higher incarnation, node back) → ALIVE +ALIVE → (graceful stop) → LEAVING +any → (admin remove) → REMOVED +REMOVED → (admin allow) → DEAD, then rejoins with a higher incarnation ``` ### SWIM Protocol - **Gossip interval**: 1 second - **Fanout**: 3 random peers per round -- **Suspect timeout**: 15 seconds -- **Dead timeout**: 30 seconds +- **Suspect timeout**: 5 seconds (`suspect_timeout_ms`) +- **Dead timeout**: 10 seconds (`dead_timeout_ms`) ### Incarnation Numbers @@ -1103,10 +1107,11 @@ DDL automatically rewritten for safe replay: ### Schema Version Tracking -Each database maintains a version counter: -- Incremented on every DDL +Each database keeps its schema version in its own `__marmot_schema_version` table: +- Bumped in the same SQLite transaction as the DDL, on the node that runs it and on every node that replays it from a log pull - Exchanged via gossip -- Delta sync validates schema version before applying DML +- A PREPARE whose required schema version is above the participant's local version is refused +- Anti-entropy restores a snapshot from the peer with the highest schema version ### Example: DDL Replication for ALTER TABLE @@ -1127,7 +1132,7 @@ T1: Node 1 broadcasts DDL lock acquisition via gossip T2: Node 1 executes locally - ALTER TABLE users ADD COLUMN ... - - Increment schema version: /schema/mydb → 2 + - Bump __marmot_schema_version → 2 (same SQLite transaction) T3: Node 1 broadcasts DDL to replicas (not CDC, raw SQL) gRPC ReplicateDDL: @@ -1194,15 +1199,15 @@ Node 2: |----------|----------| | Minority partition | Writes **fail** (cannot achieve quorum) | | Majority partition | Writes **succeed** | -| Partition heals | Delta sync + LWW merges data | +| Partition heals | Log pull from every ALIVE peer; per-row versions keep the newest image | ### Node Failure & Recovery | Scenario | Recovery Method | |----------|-----------------| -| Brief outage (<1hr) | Delta sync from peers | -| Extended outage | Snapshot transfer | -| New node joining | Full snapshot + delta sync | +| Restart with data | Startup log pull from a seed, then log pull from every ALIVE peer; ALIVE once every database is caught up | +| Down while peers truncated past its cursor (`gc_max_retention_hours`) | Snapshot restore, then re-apply of its own log | +| New node joining | Full snapshot, then log pull | ### Transaction Failure @@ -1212,7 +1217,7 @@ Node 2: | Partial PREPARE success | Abort if quorum not achieved | | COMMIT after remote quorum | Local must succeed (prepared) | | Node crash during PREPARE | Heartbeat timeout → intents cleaned | -| Node crash during COMMIT | Anti-entropy replays from peers | +| Node crash during COMMIT | Log pull: a prepared txn a peer committed is committed locally; if its rows were lost, it is replayed from the peer | ### Example: Recovery Scenarios for Our UPDATE @@ -1231,7 +1236,7 @@ T1: Network partition isolates Node 1 from Node 2, Node 3 - Decision: ABORT Node 1 rollback: - - Delete local intent: /intent/users/users:1 + - Release local row lock users/users:1 - Update txn record: Status=ABORTED - Return error to client: "quorum not achieved" @@ -1253,7 +1258,7 @@ T2 (+10s): Heartbeat timeout on Node 2 - Marks intents for cleanup T3: Cleanup on Node 2, Node 3 - - Delete intent: /intent/users/users:1 + - Release row lock users/users:1 - Update txn: Status=ABORTED - Remove from Cuckoo filter @@ -1276,10 +1281,10 @@ State: T3: Node 3 restarts T4: Anti-entropy on Node 3 - - Local SeqNum = 40 (before crash) - - Query Node 1: SeqNum = 41 - - Delta sync: get SeqNum 41 (our UPDATE) - - Apply CDC via REPLAY + - Node 1's log lists our UPDATE; Node 3 has no marker for it + - Node 3 still holds it prepared → commit it locally with + the commit timestamp from Node 1's log entry + (rows lost in the crash → discard the prepare, fetch and replay) - Node 3 now has balance=75 for id=1 Result: All nodes converged, transaction successful @@ -1324,16 +1329,17 @@ T3: Decision depends on consistency level - Abort transactions without heartbeat for `HeartbeatTimeout` (10s) - Clean orphaned write intents -**Phase 2 (Background, skipped under load)**: -- Delete COMMITTED/ABORTED records older than `GCMinRetention` (2hr) -- Force delete after `GCMaxRetention` (24hr) regardless of peer state +**Phase 2 (Background)**: +- Delete a prefix of the local commit log (below) +- Purge row-version tombstones, in chunks, once older than `gc_max_retention_hours` times the member count ### GC Coordination -- Track per-peer replication state -- Query minimum applied txn_id across peers -- Only GC transactions all peers have applied -- **Gap detection** prevents data loss if GC runs while nodes offline +- Each peer reports, on every `ListCommittedLog` call, how far it has consumed this node's log; the position is stored per peer (`/log_consumed_pos/{peerID}`) +- The safe position is the minimum across current members; a member that never reported counts as zero +- An entry older than `gc_min_retention_hours` is deleted only at or below the safe position +- An entry older than `gc_max_retention_hours` (when above 0) is deleted regardless; a peer whose cursor ends up behind the truncation point restores from a snapshot +- GC stops at the first entry it may not delete, so the log never has holes, and records the truncation point (`/log_truncated_through`) ### Example: Garbage Collection Timeline @@ -1341,7 +1347,7 @@ T3: Decision depends on consistency level ``` T0: UPDATE users SET balance = 75 WHERE id = 1 committed - TxnID: 0x0001A2B3C4D50001, SeqNum: 42 + TxnID: 0x...0001, at seq 42 in Node 1's log All nodes: Status=COMMITTED T0 + 5s: Phase 1 GC runs @@ -1351,20 +1357,21 @@ T0 + 5s: Phase 1 GC runs T0 + 1hr: Phase 2 GC considers deletion - Transaction age: 1 hour (< gc_min_retention=2hr) - - Skipped: too recent + - Stop: too recent -T0 + 2.5hr: Phase 2 GC runs again +T0 + 2.5hr: Phase 2 GC runs again on Node 1 - Transaction age: 2.5 hours (> gc_min_retention=2hr) - - Check peer replication state: - /repl/node2/mydb → LastSeqNum: 42 ✓ - /repl/node3/mydb → LastSeqNum: 42 ✓ - - All peers have applied SeqNum 42 + - Consumed positions of Node 1's log: + /log_consumed_pos/node2 → seq 42 ✓ + /log_consumed_pos/node3 → seq 42 ✓ + - Safe position = min = seq 42 - Safe to delete Cleanup: - DELETE /txn/0x0001A2B3C4D50001 - DELETE /txn_idx/seq/42/0x0001A2B3C4D50001 - DELETE /cdc_manifest/0x0001A2B3C4D50001 + DELETE /txn/0x...0001 + DELETE /txn_idx/seq/42/0x...0001 + DELETE /cdc/manifest/0x...0001 + /log_truncated_through → (42, 0x...0001) Reclaim unreferenced cdcseg ranges/files T0 + 2.5hr: MetaStore state after GC @@ -1376,22 +1383,22 @@ T0 + 2.5hr: MetaStore state after GC **GC blocked by lagging peer**: ``` -T0: Transactions SeqNum 40-50 committed +T0: Transactions at seq 40-50 committed on Node 1 T1: Node 3 goes offline T0 + 3hr: Phase 2 GC on Node 1 - - Check peer states: - /repl/node2/mydb → LastSeqNum: 50 ✓ - /repl/node3/mydb → LastSeqNum: 39 ✗ (offline, stale) - - Minimum across peers: 39 - - Cannot GC SeqNum 40-50 (Node 3 hasn't applied them) - - Skip cleanup to preserve delta sync capability + - Consumed positions: + /log_consumed_pos/node2 → seq 50 ✓ + /log_consumed_pos/node3 → seq 39 ✗ (offline, stale) + - Safe position = min = seq 39 + - Cannot delete seq 40-50 (Node 3 hasn't consumed them) -T0 + 25hr: Force GC kicks in (> gc_max_retention=24hr) - - Transaction SeqNum 40-50 are > 24 hours old - - Force delete regardless of peer state - - Node 3 will need snapshot transfer when it returns +T0 + 25hr: Max retention kicks in (> gc_max_retention=24hr) + - Entries at seq 40-50 are > 24 hours old + - Deleted regardless of consumed positions + - Node 3's cursor is now behind Node 1's truncation point: + it restores from a snapshot when it returns ``` --- @@ -1463,9 +1470,8 @@ This section traces our `UPDATE users SET balance = 75 WHERE id = 1` through the │ Cuckoo filter: Add hash("users:1") │ │ Mutex: Lock shard[hash("users:1") % 256] │ │ MetaStore write: │ -│ /intent/users/users:1 → {TxnID: 0x...0001, Status: PENDING} │ -│ /intent_txn/0x...0001/users/users:1 → (index) │ -│ /cdc_manifest/0x...0001 → {SegmentID, Offset, Length, RowCount} │ +│ row lock users/users:1 → TxnID 0x...0001 (in memory) │ +│ /cdc/manifest/0x...0001 → {SegmentID, Offset, Length, RowCount} │ │ /txn/0x...0001 → {Status: PENDING, SeqNum: 42} │ │ │ │ T3: 2PC PREPARE PHASE (parallel to Node 2, Node 3) │ @@ -1492,7 +1498,7 @@ This section traces our `UPDATE users SET balance = 75 WHERE id = 1` through the │ │ │ Node 1 ──gRPC──→ Node 2: ReplicateTransaction(COMMIT) │ │ │ ↓ │ -│ │ Read encoded row via /cdc_manifest/0x...0001 │ +│ │ Read encoded row via /cdc/manifest/0x...0001 │ │ │ Generate: UPDATE users SET ... WHERE id=1 │ │ │ Execute on SQLite │ │ │ Update /txn/0x...0001 → COMMITTED │ @@ -1521,24 +1527,22 @@ This section traces our `UPDATE users SET balance = 75 WHERE id = 1` through the │ T7: BACKGROUND CLEANUP (async) │ │ ═════════════════════════════ │ │ All nodes: │ -│ - Delete /intent/users/users:1 │ -│ - Delete /intent_txn/0x...0001/users/users:1 │ +│ - Release row lock users/users:1 │ │ - Remove from Cuckoo filter │ │ - Move /txn_idx/pend/0x...0001 → /txn_idx/seq/42/0x...0001 │ │ │ -│ T8: GOSSIP PROPAGATION │ -│ ═══════════════════════ │ -│ Node 1 gossips to random peers: │ -│ {Type: TXN_COMMITTED, TxnID: 0x...0001, SeqNum: 42} │ -│ Replication state updated: │ -│ /repl/node1/mydb → {LastSeqNum: 42} │ +│ T8: LATER ANTI-ENTROPY ROUNDS │ +│ ═════════════════════════════ │ +│ Each node pulls its peers' logs; our txn already has a │ +│ __marmot_applied_txn marker everywhere, so it is skipped │ +│ and each pull cursor moves past it │ │ │ │ T9: FINAL STATE (all nodes) │ │ ════════════════════════════ │ │ SQLite: users.balance = 75 WHERE id = 1 │ │ MetaStore: │ │ /txn/0x...0001 → {Status: COMMITTED, SeqNum: 42} │ -│ /cdc_manifest/0x...0001 → cdcseg range kept for anti-entropy │ +│ /cdc/manifest/0x...0001 → cdcseg range kept for anti-entropy │ │ Cuckoo filter: "users:1" removed │ │ │ │ T10: GC (after 2+ hours) │ @@ -1563,8 +1567,8 @@ read_timeout_ms = 2000 enable_anti_entropy = true anti_entropy_interval_seconds = 30 gc_interval_seconds = 60 -delta_sync_threshold_transactions = 10000 -delta_sync_threshold_seconds = 3600 +delta_sync_threshold_transactions = 10000 # read-replica bootstrap: snapshot above this txn-id gap +delta_sync_threshold_seconds = 3600 # only checked against the gc_*_retention_hours values # Garbage Collection gc_min_retention_hours = 2 @@ -1599,8 +1603,12 @@ dead_timeout_ms = 10000 | `db/intent_filter.go` | Cuckoo filter for conflict detection | | `db/transaction.go` | Transaction lifecycle, GC | | `grpc/replication_handler.go` | Remote 2PC phase handling | -| `grpc/anti_entropy.go` | Background sync coordination | -| `grpc/delta_sync.go` | Incremental catch-up | +| `grpc/anti_entropy.go` | Anti-entropy rounds, snapshot fallback, promotion readiness | +| `grpc/log_puller.go` | Pulls a peer's commit log (cursor, markers, STUCK tracking) | +| `grpc/log_server.go` | Serves `ListCommittedLog` and `FetchTransactions` | +| `grpc/catch_up.go` | Startup catch-up and snapshot restore | +| `db/replay_apply.go` | Applies pulled transactions with their markers | +| `db/row_version.go` | Per-row versions and tombstone purge | | `grpc/gossip.go` | SWIM protocol, membership | | `hlc/clock.go` | Hybrid Logical Clock | | `publisher/` | CDC publishing system | From a02b9e64a045b1d8119847fce041d0cf090d9b05 Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Wed, 30 Sep 2026 07:38:50 -0500 Subject: [PATCH 15/17] fix(cluster): a joiner's promotion to ALIVE wakes a held seed's claim merge A joiner starts ALIVE at incarnation 0 and refutes the seed's JOINING record before it marks itself JOINING, and MarkJoining does not bump the incarnation. Its later promotion (ALIVE at a higher incarnation) therefore read as ALIVE -> ALIVE on the seed, AliveChanged never fired, and a held seed waited a full merge interval before releasing its claim votes. With a long interval a narrow insert stalled; this made TestNarrowAutoInc_HeldInsertInTransactionDoesNotStallJoin flaky. An ALIVE node announcing ALIVE at a higher incarnation now signals AliveChanged (the ALIVE callback still fires only on a status change). The cluster harness sets batch_commit.max_wait_ms = 1: its clients write one statement at a time, so every commit waited out the 10ms flush window. The suite drops from about 130s to about 106s. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- grpc/node_registry.go | 24 +++++++++++++++++++----- grpc/node_registry_test.go | 37 +++++++++++++++++++++++++++++++++++++ test/harness_test.go | 5 +++++ 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/grpc/node_registry.go b/grpc/node_registry.go index 42995487..4e35adc0 100644 --- a/grpc/node_registry.go +++ b/grpc/node_registry.go @@ -58,7 +58,8 @@ type NodeRegistry struct { onNodeLeavingFunc func() // Callback when local node marked LEAVING via remote decommission callbackMu sync.RWMutex // aliveChanged is notified, after onNodeAliveFunc, whenever a node turns - // or is discovered ALIVE (AliveChanged). + // or is discovered ALIVE, and when an ALIVE node announces ALIVE at a + // higher incarnation (AliveChanged). aliveChanged common.Broadcast // store persists membership so a restarted node does not compute a quorum @@ -182,6 +183,9 @@ func (nr *NodeRegistry) Update(node *NodeState) { // Track if node transitions to ALIVE for callback becameAlive := false + // reannouncedAlive is an ALIVE node at a higher incarnation: no status + // change, so no callback, but AliveChanged waiters may now get an answer. + reannouncedAlive := false // Rule 4: Apply SWIM state update rules stateChanged := false @@ -226,9 +230,15 @@ func (nr *NodeRegistry) Update(node *NodeState) { stateChanged = true } - // Check if node became ALIVE - if oldStatus != NodeStatus_ALIVE && node.Status == NodeStatus_ALIVE { - becameAlive = true + // Check if node became ALIVE, or announced ALIVE again: a joiner's + // promotion reads ALIVE -> ALIVE here when its JOINING, set without a + // bump (MarkJoining), never reached this node. + if node.Status == NodeStatus_ALIVE { + if oldStatus != NodeStatus_ALIVE { + becameAlive = true + } else { + reannouncedAlive = true + } } } else if node.Incarnation == existing.Incarnation && nr.shouldEscalate(existing.Status, node.Status) { // Same incarnation: only allow escalation (ALIVE -> SUSPECT -> DEAD) @@ -281,6 +291,8 @@ func (nr *NodeRegistry) Update(node *NodeState) { // Call callback outside lock to avoid deadlock if becameAlive { nr.fireOnNodeAlive(node) + } else if reannouncedAlive { + nr.aliveChanged.Notify() } } @@ -298,7 +310,9 @@ func (nr *NodeRegistry) fireOnNodeAlive(node *NodeState) { } // AliveChanged returns a channel closed the next time a node turns or is -// discovered ALIVE, once the ALIVE callback has run for it. +// discovered ALIVE, once the ALIVE callback has run for it, or an ALIVE node +// announces ALIVE at a higher incarnation (a promotion this node saw no +// JOINING for, or a restart). func (nr *NodeRegistry) AliveChanged() <-chan struct{} { return nr.aliveChanged.Next() } diff --git a/grpc/node_registry_test.go b/grpc/node_registry_test.go index fbeceb6f..901b77e4 100644 --- a/grpc/node_registry_test.go +++ b/grpc/node_registry_test.go @@ -1184,6 +1184,43 @@ func TestNodeRegistry_AliveChangedFiresOnAlive(t *testing.T) { } } +// TestNodeRegistry_AliveChangedFiresOnJoinerPromotion: a joiner that refutes +// the seed's JOINING record before it marks itself JOINING is ALIVE in the +// seed's view throughout its catch-up, because MarkJoining does not bump its +// incarnation. Its promotion must still wake the seed's AliveChanged waiters: +// a held seed merges claim bases only on that signal or its merge interval, +// and a catching-up joiner cannot answer the merge. +// +// Mutation: signal AliveChanged only when a peer's status turns ALIVE. The +// promotion does not signal. +func TestNodeRegistry_AliveChangedFiresOnJoinerPromotion(t *testing.T) { + seed := NewNodeRegistry(1, "localhost:8081") + joiner := NewNodeRegistry(2, "localhost:8082") + gossip := func() { + self, _ := joiner.Get(2) + seed.Update(self) + } + + seed.Add(&NodeState{NodeId: 2, Address: "localhost:8082", Status: NodeStatus_JOINING}) + rumour, _ := seed.Get(2) + joiner.Update(rumour) + gossip() + if v, _ := seed.Get(2); v.Status != NodeStatus_ALIVE { + t.Fatalf("fixture: seed view after the refutation %v, want ALIVE", v.Status) + } + + joiner.MarkJoining(2) + gossip() + ch := seed.AliveChanged() + joiner.MarkAlive(2) + gossip() + select { + case <-ch: + default: + t.Fatal("the joiner's promotion did not signal AliveChanged on the seed") + } +} + // TestNodeRegistry_DiscoveringAnAliveNodeConnectsToIt pins that the ALIVE // callback, which opens this node's connection to a // peer, fires when gossip first tells us of a peer that is already ALIVE, not diff --git a/test/harness_test.go b/test/harness_test.go index 264300ed..414bb2ae 100644 --- a/test/harness_test.go +++ b/test/harness_test.go @@ -317,6 +317,11 @@ delta_sync_threshold_seconds = %d [metastore] strict_prepare_sync = %t +[batch_commit] +# A test client writes one statement at a time: each commit waits out the +# whole flush window. +max_wait_ms = 1 + [mysql] bind_address = "127.0.0.1" port = %d From 91bf5cfb239de5d4dfe87336d65475025b417c5a Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Wed, 30 Sep 2026 07:38:50 -0500 Subject: [PATCH 16/17] feat(metrics): report replication lag per peer from the log pull marmot_v2_replication_lag_txns was registered but never set. It now holds, per peer, the committed transactions of that peer's log past this node's pull cursor, summed over databases, as of the last pull round. Log sequence numbers are not dense (a restart skips a lease), so the value counts entries rather than subtracting positions. - Each round counts the listed entries that stay unapplied. When a round stops at the page cap, the last page asks the peer for the committed entries remaining (optional remaining_committed); the count reads index keys only, honours ctx, and a count failure leaves the value unknown without failing the listing. - The gauge is set only when every database's count for the peer is known, never to a false 0, and a removed peer's series is deleted. - The truncateThrough test helper truncates again, so the snapshot fallback test exercises the fallback. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- db/gc_log_test.go | 44 +++++ db/meta_store.go | 6 + db/meta_store_memory.go | 6 + db/meta_store_pebble.go | 75 ++++++-- docs/src/pages/metrics.mdx | 4 +- grpc/anti_entropy.go | 81 ++++++++- grpc/anti_entropy_regression_test.go | 20 ++- grpc/log_puller.go | 60 +++++-- grpc/log_server.go | 33 +++- grpc/marmot.pb.go | 35 +++- grpc/marmot.proto | 5 + grpc/replication_lag_test.go | 257 +++++++++++++++++++++++++++ telemetry/metrics.go | 7 +- telemetry/telemetry.go | 8 + 14 files changed, 593 insertions(+), 48 deletions(-) create mode 100644 grpc/replication_lag_test.go diff --git a/db/gc_log_test.go b/db/gc_log_test.go index 326a9c37..34cd6936 100644 --- a/db/gc_log_test.go +++ b/db/gc_log_test.go @@ -4,6 +4,8 @@ package db import ( + "context" + "errors" "testing" "time" @@ -104,6 +106,48 @@ func TestListCommittedLogExcludesUncommitted(t *testing.T) { } } +// TestCountCommittedLogMatchesWhatAListingWouldReturn: the count after a +// position, through a stable point, is the number of entries a listing from +// there returns - COMMITTED only, and none past stable. +func TestCountCommittedLogMatchesWhatAListingWouldReturn(t *testing.T) { + store, cleanup := createTestPebbleMetaStore(t) + defer cleanup() + + var positions []LogPosition + for i := uint64(1); i <= 4; i++ { + positions = append(positions, commitTestTxn(t, store, 3000+i, 1)) + } + if err := store.BeginTransaction(3100, 1, hlc.NewClock(1).Now()); err != nil { + t.Fatalf("BeginTransaction: %v", err) + } + stable := store.StableSeq() + + for _, tc := range []struct { + after LogPosition + stable uint64 + want uint64 + }{ + {LogPosition{}, stable, 4}, + {positions[1], stable, 2}, + {positions[3], stable, 0}, + {LogPosition{}, positions[2].Seq, 3}, + } { + got, err := store.CountCommittedLog(context.Background(), tc.after, tc.stable) + if err != nil { + t.Fatalf("CountCommittedLog(%v, %d): %v", tc.after, tc.stable, err) + } + if got != tc.want { + t.Fatalf("CountCommittedLog(%v, %d) = %d, want %d", tc.after, tc.stable, got, tc.want) + } + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := store.CountCommittedLog(ctx, LogPosition{}, stable); !errors.Is(err, context.Canceled) { + t.Fatalf("CountCommittedLog with a cancelled ctx: err = %v, want context.Canceled", err) + } +} + func TestPullCursorAndConsumedPositionAreSeparateAndAsIs(t *testing.T) { store, cleanup := createTestPebbleMetaStore(t) defer cleanup() diff --git a/db/meta_store.go b/db/meta_store.go index 9ac5647b..b9030976 100644 --- a/db/meta_store.go +++ b/db/meta_store.go @@ -1,6 +1,7 @@ package db import ( + "context" "errors" "path/filepath" "strings" @@ -128,6 +129,11 @@ type MetaStore interface { // beyond the returned page. Each entry carries its commit timestamp. ListCommittedLog(after LogPosition, limit int) (entries []CommittedLogEntry, stable uint64, more bool, err error) + // CountCommittedLog returns how many COMMITTED local log entries lie + // strictly after `after` with Seq <= stable. Its cost grows with the + // count; it stops with ctx's error once ctx is done. + CountCommittedLog(ctx context.Context, after LogPosition, stable uint64) (uint64, error) + // GetPullCursor and SetPullCursor persist C[self,peer,d]: this node's // pull position in peerNodeID's log for this database. GetPullCursor(peerNodeID uint64) (LogPosition, error) diff --git a/db/meta_store_memory.go b/db/meta_store_memory.go index aecc275e..4a174281 100644 --- a/db/meta_store_memory.go +++ b/db/meta_store_memory.go @@ -1,6 +1,7 @@ package db import ( + "context" "fmt" "time" @@ -265,6 +266,11 @@ func (m *MemoryMetaStore) ListCommittedLog(after LogPosition, limit int) ([]Comm return m.pebble.ListCommittedLog(after, limit) } +// CountCommittedLog delegates to Pebble. +func (m *MemoryMetaStore) CountCommittedLog(ctx context.Context, after LogPosition, stable uint64) (uint64, error) { + return m.pebble.CountCommittedLog(ctx, after, stable) +} + // GetPullCursor delegates to Pebble. func (m *MemoryMetaStore) GetPullCursor(peerNodeID uint64) (LogPosition, error) { return m.pebble.GetPullCursor(peerNodeID) diff --git a/db/meta_store_pebble.go b/db/meta_store_pebble.go index 13917a27..ea529658 100644 --- a/db/meta_store_pebble.go +++ b/db/meta_store_pebble.go @@ -1,6 +1,7 @@ package db import ( + "context" "encoding/binary" "errors" "fmt" @@ -1758,16 +1759,65 @@ func (s *PebbleMetaStore) ListCommittedLog(after LogPosition, limit int) ([]Comm return nil, stable, false, nil } + var entries []CommittedLogEntry + more := false + err := s.iterateCommittedLog(after, stable, func(pos LogPosition, stamp func() []byte) (bool, error) { + if len(entries) >= limit { + more = true + return false, nil + } + commitTS, err := decodeLogEntryStamp(stamp()) + if err != nil { + return false, err + } + entries = append(entries, CommittedLogEntry{LogPosition: pos, CommitTS: commitTS}) + return true, nil + }) + if err != nil { + return nil, stable, false, err + } + return entries, stable, more, nil +} + +// CountCommittedLog returns how many of this store's local log entries lie +// strictly after `after` with Seq <= stable and are COMMITTED: the entries a +// ListCommittedLog walk from `after` would list against that stable point. +// It reads each such entry's index key, never its value, plus the +// transaction's status (a stray index entry is not COMMITTED), so its cost +// grows with the count. It stops with ctx's error once ctx is done, checked +// every countCtxCheckInterval entries. +func (s *PebbleMetaStore) CountCommittedLog(ctx context.Context, after LogPosition, stable uint64) (uint64, error) { + var n uint64 + err := s.iterateCommittedLog(after, stable, func(LogPosition, func() []byte) (bool, error) { + if n%countCtxCheckInterval == 0 { + if err := ctx.Err(); err != nil { + return false, err + } + } + n++ + return true, nil + }) + return n, err +} + +// countCtxCheckInterval is how many entries CountCommittedLog counts +// between checks of its context. +const countCtxCheckInterval = 1024 + +// iterateCommittedLog calls fn, in position order, for every local log +// entry strictly after `after` with Seq <= stable whose transaction is +// COMMITTED, passing a reader of the entry's stored commit stamp (the index +// value, read only if fn calls it), until fn returns false or an error. +func (s *PebbleMetaStore) iterateCommittedLog(after LogPosition, stable uint64, fn func(pos LogPosition, stamp func() []byte) (bool, error)) error { lower := buildKeyUint64x2(pebblePrefixTxnSeq, after.Seq, after.TxnID) prefix := []byte(pebblePrefixTxnSeq) iter, err := s.db.NewIter(&pebble.IterOptions{LowerBound: prefix, UpperBound: prefixUpperBound(prefix)}) if err != nil { - return nil, stable, false, err + return err } defer iter.Close() - var entries []CommittedLogEntry - more := false + stamp := iter.Value for iter.SeekGE(lower); iter.Valid(); iter.Next() { key := iter.Key() if len(key) < len(pebblePrefixTxnSeq)+16 { @@ -1786,26 +1836,21 @@ func (s *PebbleMetaStore) ListCommittedLog(after LogPosition, limit int) ([]Comm status, statusErr := s.readTxnStatus(pos.TxnID) if statusErr != nil && statusErr != pebble.ErrNotFound { - return nil, stable, false, statusErr + return statusErr } if statusErr == pebble.ErrNotFound || status != TxnStatusCommitted { continue } - if len(entries) >= limit { - more = true - break - } - commitTS, err := decodeLogEntryStamp(iter.Value()) + cont, err := fn(pos, stamp) if err != nil { - return nil, stable, false, err + return err + } + if !cont { + break } - entries = append(entries, CommittedLogEntry{LogPosition: pos, CommitTS: commitTS}) - } - if err := iter.Error(); err != nil { - return nil, stable, false, err } - return entries, stable, more, nil + return iter.Error() } // GetPullCursor returns C[self,peer,d]: this store's pull position in diff --git a/docs/src/pages/metrics.mdx b/docs/src/pages/metrics.mdx index 67281d3a..8b87151b 100644 --- a/docs/src/pages/metrics.mdx +++ b/docs/src/pages/metrics.mdx @@ -69,7 +69,7 @@ curl http://localhost:8080/metrics | `marmot_v2_log_pull_txns_applied_total` | Counter | - | Transactions applied by pulling peers' local commit logs | | `marmot_v2_log_pull_pair_results_total` | Counter | `outcome` | Per-(peer, database) pulls by outcome: `caught_up`, `behind`, `needs_snapshot`, `unimplemented`, `error` | | `marmot_v2_anti_entropy_stuck_txns` | Gauge | `database` | Pulled transactions no longer retried every round after repeated replay failures | -| `marmot_v2_replication_lag_txns` | Gauge | `peer` | Registered but not currently updated | +| `marmot_v2_replication_lag_txns` | Gauge | `peer` | Committed transactions past this node's pull cursor, as of the last pull round, summed over databases. It is not a real-time value. It is an upper bound on what catch-up still has to do: some of them may already be applied through the write path, because entries beyond one round's page limit are counted without being checked. A round that cannot count every entry leaves the value from the previous round: a pull error, a snapshot fallback, or an older peer that stops at the page limit. A peer that is not alive (SUSPECT or DEAD, but not REMOVED) is not pulled, so it keeps its last value until it is pulled again or removed. The series is deleted when the peer is removed from membership | There is no delta sync counter: anti-entropy is a log pull, counted by the `log_pull_*` metrics. @@ -144,6 +144,6 @@ sum(rate(marmot_v2_write_conflicts_total[1m])) by (type) **Replication lag:** ```promql -# Max replication lag across all peers +# Per node, the most transactions past its pull cursor into any one peer's log, as of its last pull round max(marmot_v2_replication_lag_txns) by (node_id) ``` diff --git a/grpc/anti_entropy.go b/grpc/anti_entropy.go index bb877da5..9b5937e2 100644 --- a/grpc/anti_entropy.go +++ b/grpc/anti_entropy.go @@ -3,6 +3,7 @@ package grpc import ( "context" "fmt" + "strconv" "sync" "time" @@ -48,6 +49,13 @@ type AntiEntropyService struct { statusMu sync.RWMutex caughtUp map[string]bool promotionReady map[string]bool + + // lagGauge is telemetry.ReplicationLagTxns; lagPeers holds the peer + // label values it currently exports, so a peer that leaves membership + // has its series deleted. + lagGauge telemetry.GaugeVec + lagMu sync.Mutex + lagPeers map[uint64]struct{} } // SnapshotTransferFunc initiates a snapshot transfer to a peer @@ -95,6 +103,8 @@ func NewAntiEntropyService(config AntiEntropyConfig) *AntiEntropyService { stopCh: make(chan struct{}), caughtUp: make(map[string]bool), promotionReady: make(map[string]bool), + lagGauge: telemetry.ReplicationLagTxns, + lagPeers: make(map[uint64]struct{}), } } @@ -274,6 +284,7 @@ func (ae *AntiEntropyService) performAntiEntropy() { }() members, alive := ae.currentMembers() + ae.deleteDepartedPeerLag(members) if len(alive) == 0 { log.Debug().Msg("Anti-entropy: no alive peers to sync with") return @@ -290,9 +301,11 @@ func (ae *AntiEntropyService) performAntiEntropy() { // is ever pulled. ae.reconcileDatabaseRegistry(alive) + lag := newReplicationLag() for _, dbName := range ae.dbManager.ListDatabases() { - ae.syncDatabase(dbName, members, alive) + ae.syncDatabase(dbName, members, alive, lag) } + ae.publishLag(lag) ae.restoreAwaitingDatabases(alive) @@ -392,15 +405,16 @@ func considerSnapshotSource(best *snapshotSource, candidate snapshotSource) *sna // syncDatabase runs one database's pull round against every alive peer, // each pair under its own deadline, falls back // to a snapshot restore when any member's log no longer covers this node's -// cursor into it, and records the resulting caught-up status. -func (ae *AntiEntropyService) syncDatabase(dbName string, members, alive []*NodeState) { +// cursor into it, and records the resulting caught-up status and each +// pair's unapplied count in lag. +func (ae *AntiEntropyService) syncDatabase(dbName string, members, alive []*NodeState, lag *replicationLag) { // Finish a pending re-apply of this node's own log, left by a // restore that crashed or failed before completing it, before pulling // anything more for dbName. ae.reapplyIfPending(dbName) dbCaughtUp := len(alive) == len(members) - dbPromotionReady, needsSnapshot, source := ae.pullEveryAlivePeer(dbName, alive, &dbCaughtUp) + dbPromotionReady, needsSnapshot, source := ae.pullEveryAlivePeer(dbName, alive, &dbCaughtUp, lag) if needsSnapshot { dbCaughtUp = false @@ -425,7 +439,7 @@ func (ae *AntiEntropyService) syncDatabase(dbName string, members, alive []*Node // dbCaughtUp (an in-out pointer, cleared by any pair that did not fully // succeed - it starts set from len(alive)==len(members)), promotionReady // (see below) and the best snapshot source seen, for the -// needsSnapshot fallback. +// needsSnapshot fallback. Each pair's outcome is also recorded in lag. // // promotionReady is deliberately over alive peers only, never all members // (checkPromotionCriteria): requiring CaughtUp over every member, DEAD ones @@ -434,13 +448,14 @@ func (ae *AntiEntropyService) syncDatabase(dbName string, members, alive []*Node // did not fully succeed; it also requires at least one alive peer was // actually pulled, so a node with no alive peers yet is never // promotion-ready. -func (ae *AntiEntropyService) pullEveryAlivePeer(dbName string, alive []*NodeState, dbCaughtUp *bool) (promotionReady, needsSnapshot bool, source *snapshotSource) { +func (ae *AntiEntropyService) pullEveryAlivePeer(dbName string, alive []*NodeState, dbCaughtUp *bool, lag *replicationLag) (promotionReady, needsSnapshot bool, source *snapshotSource) { promotionReady = len(alive) > 0 for _, peer := range alive { pairCtx, cancel := context.WithTimeout(context.Background(), ae.interval) result, err := ae.logPuller.PullPair(pairCtx, PeerRef{NodeID: peer.NodeId, Address: peer.Address}, dbName) cancel() + lag.record(peer.NodeId, result, err) if err != nil { log.Warn().Err(err).Uint64("peer_node", peer.NodeId).Str("database", dbName). @@ -469,6 +484,60 @@ func (ae *AntiEntropyService) pullEveryAlivePeer(dbName string, alive []*NodeSta return promotionReady, needsSnapshot, source } +// replicationLag sums one round's PairResult.Unapplied per peer over the +// databases pulled. A peer with any pair of unknown count (an error, a +// snapshot fallback, an older peer that cannot report the rest of its log) +// is marked unknown and not published that round: a partial sum would +// understate its lag. +type replicationLag struct { + unapplied map[uint64]uint64 + unknown map[uint64]bool +} + +func newReplicationLag() *replicationLag { + return &replicationLag{unapplied: make(map[uint64]uint64), unknown: make(map[uint64]bool)} +} + +// record adds one pair's outcome for peer. +func (l *replicationLag) record(peer uint64, result PairResult, err error) { + if err != nil || !result.UnappliedKnown { + l.unknown[peer] = true + return + } + l.unapplied[peer] += result.Unapplied +} + +// publishLag sets ReplicationLagTxns for every peer lag counted in full, +// leaving every other peer's series as it was. +func (ae *AntiEntropyService) publishLag(lag *replicationLag) { + ae.lagMu.Lock() + defer ae.lagMu.Unlock() + for peer, n := range lag.unapplied { + if lag.unknown[peer] { + continue + } + ae.lagGauge.With(strconv.FormatUint(peer, 10)).Set(float64(n)) + ae.lagPeers[peer] = struct{}{} + } +} + +// deleteDepartedPeerLag deletes the ReplicationLagTxns series of every peer +// no longer among members. +func (ae *AntiEntropyService) deleteDepartedPeerLag(members []*NodeState) { + current := make(map[uint64]struct{}, len(members)) + for _, m := range members { + current[m.NodeId] = struct{}{} + } + ae.lagMu.Lock() + defer ae.lagMu.Unlock() + for peer := range ae.lagPeers { + if _, ok := current[peer]; !ok { + ae.lagGauge.Delete(strconv.FormatUint(peer, 10)) + delete(ae.lagPeers, peer) + } + } +} + // restoreFromPeer restores dbName from source's snapshot and then runs the // rest of the restore sequence (restoreDatabase). func (ae *AntiEntropyService) restoreFromPeer(ctx context.Context, dbName string, source *NodeState) error { diff --git a/grpc/anti_entropy_regression_test.go b/grpc/anti_entropy_regression_test.go index 6177dd4f..bc8f97dd 100644 --- a/grpc/anti_entropy_regression_test.go +++ b/grpc/anti_entropy_regression_test.go @@ -57,13 +57,24 @@ func newAERegNode(t *testing.T, id uint64, databases ...string) *aeRegNode { // serve starts n over real gRPC and records its listen address. func (n *aeRegNode) serve(t *testing.T) { + t.Helper() + n.serveWrapped(t, nil) +} + +// serveWrapped is serve through wrap, when non-nil (a peer that behaves +// differently on one RPC, say). +func (n *aeRegNode) serveWrapped(t *testing.T, wrap func(*Server) MarmotServiceServer) { t.Helper() listener, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) server := &Server{nodeID: n.id, registry: NewNodeRegistry(n.id, listener.Addr().String())} server.SetDatabaseManager(n.dm) + var service MarmotServiceServer = server + if wrap != nil { + service = wrap(server) + } gs := grpc.NewServer() - RegisterMarmotServiceServer(gs, server) + RegisterMarmotServiceServer(gs, service) go func() { _ = gs.Serve(listener) }() t.Cleanup(gs.Stop) n.addr = listener.Addr().String() @@ -93,8 +104,8 @@ func (n *aeRegNode) seed(t *testing.T, d string, txnID, origin uint64, wall int6 require.True(t, applied) } -// truncateThrough forces d's GC to unconditionally delete every currently -// committed log entry (maxRetention=0), so ListCommittedLog reports a +// truncateThrough forces d's GC to delete every currently committed log +// entry (every position safe, no minimum retention), so ListCommittedLog reports a // non-zero TruncatedThrough to any puller whose cursor is still behind it. // Used to force a PullPair NeedsSnapshot outcome deterministically, without // waiting out a real retention window. @@ -102,8 +113,9 @@ func (n *aeRegNode) truncateThrough(t *testing.T, d string) { t.Helper() mdb, err := n.dm.GetDatabase(d) require.NoError(t, err) - _, err = mdb.GetMetaStore().CleanupOldTransactionRecords(time.Hour, 0, db.LogPosition{}) + deleted, err := mdb.GetMetaStore().CleanupOldTransactionRecords(0, 0, db.LogPosition{Seq: ^uint64(0), TxnID: ^uint64(0)}) require.NoError(t, err) + require.NotZero(t, deleted, "nothing was truncated") } // rows reads table t of database d. diff --git a/grpc/log_puller.go b/grpc/log_puller.go index 01a43e3b..5709e049 100644 --- a/grpc/log_puller.go +++ b/grpc/log_puller.go @@ -89,6 +89,17 @@ type PairResult struct { // Stuck counts STUCK transactions this call encountered for this pair // (see LogPuller.StuckTxns). Stuck int + + // Unapplied counts the COMMITTED entries of the peer's log, through its + // stable point, past this pair's cursor when the call ends: the listed + // entries left uncovered plus every entry past the last one listed, + // which is not checked against applied markers, so it is an upper bound + // on what is still unapplied. It is meaningful only when UnappliedKnown is set: the walk + // reached the peer's stable point, or ended at maxPagesPerPull with the + // peer reporting the rest (LogListResponse.remaining_committed, unset + // from an older peer). Any other outcome leaves it unknown. + Unapplied uint64 + UnappliedKnown bool } // StuckTxn describes one transaction a LogPuller has stopped retrying every @@ -242,6 +253,10 @@ type pullState struct { // the same busy timeout. localCommitFailed bool + // uncovered counts the entries listed so far this call that are not + // covered, for PairResult.Unapplied. + uncovered uint64 + result PairResult } @@ -254,7 +269,7 @@ func (lp *LogPuller) pullPair(ctx context.Context, peer PeerRef, database string if ctxErr := ctx.Err(); ctxErr != nil { return st.result, ctxErr } - done, err := lp.pullPage(ctx, st) + done, err := lp.pullPage(ctx, st, page == maxPagesPerPull-1) if err != nil || done { return st.result, err } @@ -262,6 +277,14 @@ func (lp *LogPuller) pullPair(ctx context.Context, peer PeerRef, database string return st.result, nil } +// setUnapplied records the call's final unapplied count: the listed +// entries left uncovered plus remaining, the peer's count of those past the +// last one listed. +func (st *pullState) setUnapplied(remaining uint64) { + st.result.Unapplied = st.uncovered + remaining + st.result.UnappliedKnown = true +} + // newPullState resolves the local database, the peer's client and the // pair's persisted cursor, and counts this call for the STUCK retry cadence. func (lp *LogPuller) newPullState(peer PeerRef, database string) (*pullState, error) { @@ -293,7 +316,9 @@ func (lp *LogPuller) newPullState(peer PeerRef, database string) (*pullState, er // pullPage lists and applies one page of the peer's log. done reports that // the call is over (caught up, or an outcome recorded in st.result). -func (lp *LogPuller) pullPage(ctx context.Context, st *pullState) (done bool, err error) { +// lastPage asks the peer to count the entries past this page, the only +// part of its log this call does not list. +func (lp *LogPuller) pullPage(ctx context.Context, st *pullState, lastPage bool) (done bool, err error) { resp, err := st.client.ListCommittedLog(ctx, &LogListRequest{ Database: st.database, RequestingNodeId: lp.nodeID, @@ -302,6 +327,7 @@ func (lp *LogPuller) pullPage(ctx context.Context, st *pullState) (done bool, er Limit: uint32(lp.pageSize), ConsumedSeq: st.cursor.Seq, ConsumedTxnId: st.cursor.TxnID, + CountRemaining: lastPage, }) if err != nil { if status.Code(err) == codes.Unimplemented { @@ -334,15 +360,21 @@ func (lp *LogPuller) pullPage(ctx context.Context, st *pullState) (done bool, er } if len(resp.Entries) == 0 { st.result.CaughtUp = st.contiguous && !resp.More + st.setUnapplied(0) return true, nil } - return lp.applyListedPage(ctx, st, resp) + done, err = lp.applyListedPage(ctx, st, resp) + if err == nil && !done && resp.RemainingCommitted != nil { + st.setUnapplied(*resp.RemainingCommitted) + } + return done, err } // applyListedPage applies one non-empty page, advances the cursor over the // covered prefix, and moves listAfter past the page. func (lp *LogPuller) applyListedPage(ctx context.Context, st *pullState, resp *LogListResponse) (done bool, err error) { - coveredPrefixLen, err := lp.applyPage(ctx, st, resp.Entries) + coveredPrefixLen, uncovered, err := lp.applyPage(ctx, st, resp.Entries) + st.uncovered += uint64(uncovered) if st.contiguous && coveredPrefixLen > 0 { last := resp.Entries[coveredPrefixLen-1] st.cursor = db.LogPosition{Seq: last.Seq, TxnID: last.TxnId} @@ -358,6 +390,7 @@ func (lp *LogPuller) applyListedPage(ctx context.Context, st *pullState, resp *L } if !resp.More { st.result.CaughtUp = st.contiguous + st.setUnapplied(0) return true, nil } last := resp.Entries[len(resp.Entries)-1] @@ -375,21 +408,22 @@ func (lp *LogPuller) persistCursor(st *pullState) error { // applyPage classifies and applies one ListCommittedLog page's entries, in // order, and returns the length of the page's contiguous prefix now -// covered. It returns an error only for a failure that is not attributable -// to a specific transaction (checking local state). +// covered and how many of its entries are still not covered. It returns an +// error only for a failure that is not attributable to a specific +// transaction (checking local state). // // Every listed entry is COMMITTED in the peer's log, so a transaction this // node holds PENDING under the same id was decided COMMITTED, and its // local record is resolved here, in the same call (resolveLocalPending), // instead of stopping the walk until the stale-transaction GC ends it. -func (lp *LogPuller) applyPage(ctx context.Context, st *pullState, entries []*LogEntry) (int, error) { +func (lp *LogPuller) applyPage(ctx context.Context, st *pullState, entries []*LogEntry) (prefix, uncovered int, err error) { ids := make([]uint64, len(entries)) for i, e := range entries { ids[i] = e.TxnId } appliedMarkers, err := st.mdb.AppliedTxns(ids) if err != nil { - return 0, fmt.Errorf("check applied markers: %w", err) + return 0, 0, fmt.Errorf("check applied markers: %w", err) } covered := make([]bool, len(entries)) @@ -407,7 +441,7 @@ func (lp *LogPuller) applyPage(ctx context.Context, st *pullState, entries []*Lo } fetch, err := lp.resolveLocalPending(st, e, &covered[i]) if err != nil { - return 0, err + return 0, 0, err } if fetch { fetchIdx[e.TxnId] = i @@ -419,11 +453,15 @@ func (lp *LogPuller) applyPage(ctx context.Context, st *pullState, entries []*Lo lp.fetchAndApply(ctx, st, toFetch, fetchIdx, covered) } - prefix := 0 for prefix < len(covered) && covered[prefix] { prefix++ } - return prefix, nil + for _, c := range covered[prefix:] { + if !c { + uncovered++ + } + } + return prefix, uncovered, nil } // resolveLocalPending resolves this node's own record for e's transaction, diff --git a/grpc/log_server.go b/grpc/log_server.go index bc739b40..ca6b08ea 100644 --- a/grpc/log_server.go +++ b/grpc/log_server.go @@ -91,14 +91,43 @@ func (s *Server) ListCommittedLog(ctx context.Context, req *LogListRequest) (*Lo return nil, err } - return &LogListResponse{ + resp := &LogListResponse{ Entries: pbEntries, StableSeq: stable, TruncatedSeq: truncated.Seq, TruncatedTxnId: truncated.TxnID, More: more, SchemaVersion: mdb.SchemaVersion(), - }, nil + } + if req.CountRemaining { + resp.RemainingCommitted = remainingCommitted(ctx, metaStore, req.Database, after, entries, stable, more) + } + return resp, nil +} + +// remainingCommitted counts the COMMITTED entries through stable after the +// page listed from `after`: none when the page was the last one, and +// otherwise every entry past the page's last, which walks them all. The +// count only feeds a metric, so a count that fails or outlasts ctx returns +// nil (unknown) rather than failing the page. +func remainingCommitted(ctx context.Context, metaStore db.MetaStore, database string, after db.LogPosition, entries []db.CommittedLogEntry, stable uint64, more bool) *uint64 { + var n uint64 + if !more { + return &n + } + if len(entries) > 0 { + after = entries[len(entries)-1].LogPosition + } + n, err := metaStore.CountCommittedLog(ctx, after, stable) + if err != nil { + ev := log.Warn() + if ctx.Err() != nil { + ev = log.Debug() // the requester gave up; nothing is wrong here + } + ev.Err(err).Str("database", database).Msg("count of remaining committed log entries abandoned; left unset") + return nil + } + return &n } // logEntriesToProto converts listed log entries to LogEntry messages, each diff --git a/grpc/marmot.pb.go b/grpc/marmot.pb.go index 60c59704..73ba070f 100644 --- a/grpc/marmot.pb.go +++ b/grpc/marmot.pb.go @@ -1981,6 +1981,7 @@ type LogListRequest struct { Limit uint32 `protobuf:"varint,5,opt,name=limit,proto3" json:"limit,omitempty"` ConsumedSeq uint64 `protobuf:"varint,6,opt,name=consumed_seq,json=consumedSeq,proto3" json:"consumed_seq,omitempty"` // Position (consumed_seq, consumed_txn_id) the requester has applied every entry through ConsumedTxnId uint64 `protobuf:"varint,7,opt,name=consumed_txn_id,json=consumedTxnId,proto3" json:"consumed_txn_id,omitempty"` + CountRemaining bool `protobuf:"varint,8,opt,name=count_remaining,json=countRemaining,proto3" json:"count_remaining,omitempty"` // Ask for remaining_committed in the response unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2064,6 +2065,13 @@ func (x *LogListRequest) GetConsumedTxnId() uint64 { return 0 } +func (x *LogListRequest) GetCountRemaining() bool { + if x != nil { + return x.CountRemaining + } + return false +} + type LogEntry struct { state protoimpl.MessageState `protogen:"open.v1"` Seq uint64 `protobuf:"varint,1,opt,name=seq,proto3" json:"seq,omitempty"` @@ -2133,8 +2141,12 @@ type LogListResponse struct { More bool `protobuf:"varint,5,opt,name=more,proto3" json:"more,omitempty"` // More stable entries follow the last one returned SchemaVersion uint64 `protobuf:"varint,6,opt,name=schema_version,json=schemaVersion,proto3" json:"schema_version,omitempty"` DatabaseAbsent bool `protobuf:"varint,7,opt,name=database_absent,json=databaseAbsent,proto3" json:"database_absent,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // COMMITTED entries after the last one returned (or after the request's + // position, for an empty page), through stable_seq. Set only when the + // request asked (count_remaining); unset from an older peer. + RemainingCommitted *uint64 `protobuf:"varint,8,opt,name=remaining_committed,json=remainingCommitted,proto3,oneof" json:"remaining_committed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *LogListResponse) Reset() { @@ -2216,6 +2228,13 @@ func (x *LogListResponse) GetDatabaseAbsent() bool { return false } +func (x *LogListResponse) GetRemainingCommitted() uint64 { + if x != nil && x.RemainingCommitted != nil { + return *x.RemainingCommitted + } + return 0 +} + type FetchTransactionsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Database string `protobuf:"bytes,1,opt,name=database,proto3" json:"database,omitempty"` @@ -4417,7 +4436,7 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x17required_schema_version\x18\x05 \x01(\x04R\x15requiredSchemaVersion\x12\x17\n" + "\aseq_num\x18\x06 \x01(\x04R\x06seqNum\x12$\n" + "\x0eorigin_node_id\x18\a \x01(\x04R\foriginNodeId\x12\x1b\n" + - "\trow_count\x18\b \x01(\rR\browCount\"\xfa\x01\n" + + "\trow_count\x18\b \x01(\rR\browCount\"\xa3\x02\n" + "\x0eLogListRequest\x12\x1a\n" + "\bdatabase\x18\x01 \x01(\tR\bdatabase\x12,\n" + "\x12requesting_node_id\x18\x02 \x01(\x04R\x10requestingNodeId\x12\x1b\n" + @@ -4426,11 +4445,12 @@ const file_grpc_marmot_proto_rawDesc = "" + "afterTxnId\x12\x14\n" + "\x05limit\x18\x05 \x01(\rR\x05limit\x12!\n" + "\fconsumed_seq\x18\x06 \x01(\x04R\vconsumedSeq\x12&\n" + - "\x0fconsumed_txn_id\x18\a \x01(\x04R\rconsumedTxnId\"n\n" + + "\x0fconsumed_txn_id\x18\a \x01(\x04R\rconsumedTxnId\x12'\n" + + "\x0fcount_remaining\x18\b \x01(\bR\x0ecountRemaining\"n\n" + "\bLogEntry\x12\x10\n" + "\x03seq\x18\x01 \x01(\x04R\x03seq\x12\x15\n" + "\x06txn_id\x18\x02 \x01(\x04R\x05txnId\x129\n" + - "\x10commit_timestamp\x18\x03 \x01(\v2\x0e.marmot.v2.HLCR\x0fcommitTimestamp\"\x92\x02\n" + + "\x10commit_timestamp\x18\x03 \x01(\v2\x0e.marmot.v2.HLCR\x0fcommitTimestamp\"\xe0\x02\n" + "\x0fLogListResponse\x12-\n" + "\aentries\x18\x01 \x03(\v2\x13.marmot.v2.LogEntryR\aentries\x12\x1d\n" + "\n" + @@ -4439,7 +4459,9 @@ const file_grpc_marmot_proto_rawDesc = "" + "\x10truncated_txn_id\x18\x04 \x01(\x04R\x0etruncatedTxnId\x12\x12\n" + "\x04more\x18\x05 \x01(\bR\x04more\x12%\n" + "\x0eschema_version\x18\x06 \x01(\x04R\rschemaVersion\x12'\n" + - "\x0fdatabase_absent\x18\a \x01(\bR\x0edatabaseAbsent\"}\n" + + "\x0fdatabase_absent\x18\a \x01(\bR\x0edatabaseAbsent\x124\n" + + "\x13remaining_committed\x18\b \x01(\x04H\x00R\x12remainingCommitted\x88\x01\x01B\x16\n" + + "\x14_remaining_committed\"}\n" + "\x18FetchTransactionsRequest\x12\x1a\n" + "\bdatabase\x18\x01 \x01(\tR\bdatabase\x12,\n" + "\x12requesting_node_id\x18\x02 \x01(\x04R\x10requestingNodeId\x12\x17\n" + @@ -4843,6 +4865,7 @@ func file_grpc_marmot_proto_init() { (*Statement_VectorIndexChange)(nil), (*Statement_DmlIntent)(nil), } + file_grpc_marmot_proto_msgTypes[23].OneofWrappers = []any{} file_grpc_marmot_proto_msgTypes[49].OneofWrappers = []any{ (*TransactionStreamMessage_Chunk)(nil), (*TransactionStreamMessage_Commit)(nil), diff --git a/grpc/marmot.proto b/grpc/marmot.proto index 21c62e9c..dfbf6df1 100644 --- a/grpc/marmot.proto +++ b/grpc/marmot.proto @@ -367,6 +367,7 @@ message LogListRequest { uint32 limit = 5; uint64 consumed_seq = 6; // Position (consumed_seq, consumed_txn_id) the requester has applied every entry through uint64 consumed_txn_id = 7; + bool count_remaining = 8; // Ask for remaining_committed in the response } message LogEntry { @@ -383,6 +384,10 @@ message LogListResponse { bool more = 5; // More stable entries follow the last one returned uint64 schema_version = 6; bool database_absent = 7; + // COMMITTED entries after the last one returned (or after the request's + // position, for an empty page), through stable_seq. Set only when the + // request asked (count_remaining); unset from an older peer. + optional uint64 remaining_committed = 8; } message FetchTransactionsRequest { diff --git a/grpc/replication_lag_test.go b/grpc/replication_lag_test.go new file mode 100644 index 00000000..fa981de7 --- /dev/null +++ b/grpc/replication_lag_test.go @@ -0,0 +1,257 @@ +package grpc + +import ( + "context" + "strconv" + "sync" + "sync/atomic" + "testing" + + "github.com/maxpert/marmot/telemetry" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// recordingGaugeVec is a telemetry.GaugeVec that keeps each label's last +// Set value, so a test asserts on its own instance, never the global one. +type recordingGaugeVec struct { + mu sync.Mutex + values map[string]float64 +} + +func newRecordingGaugeVec() *recordingGaugeVec { + return &recordingGaugeVec{values: make(map[string]float64)} +} + +func (g *recordingGaugeVec) With(labels ...string) telemetry.Gauge { + return recordingGauge{vec: g, label: labels[0]} +} + +func (g *recordingGaugeVec) Delete(labels ...string) { + g.mu.Lock() + defer g.mu.Unlock() + delete(g.values, labels[0]) +} + +// value returns peer's series value and whether the series exists. +func (g *recordingGaugeVec) value(peer uint64) (float64, bool) { + g.mu.Lock() + defer g.mu.Unlock() + v, ok := g.values[strconv.FormatUint(peer, 10)] + return v, ok +} + +type recordingGauge struct { + telemetry.NoopStat + vec *recordingGaugeVec + label string +} + +func (r recordingGauge) Set(v float64) { + r.vec.mu.Lock() + defer r.vec.mu.Unlock() + r.vec.values[r.label] = v +} + +// switchableFetchServer fails every FetchTransactions call with a transport +// error while fail is set, so listed entries stay unapplied. +type switchableFetchServer struct { + *Server + fail *atomic.Bool +} + +func (s switchableFetchServer) FetchTransactions(req *FetchTransactionsRequest, stream MarmotService_FetchTransactionsServer) error { + if s.fail.Load() { + return status.Error(codes.Unavailable, "fetch disabled by test") + } + return s.Server.FetchTransactions(req, stream) +} + +// legacyListServer answers ListCommittedLog as a peer that predates +// remaining_committed does: never setting it. +type legacyListServer struct { + *Server +} + +func (s legacyListServer) ListCommittedLog(ctx context.Context, req *LogListRequest) (*LogListResponse, error) { + resp, err := s.Server.ListCommittedLog(ctx, req) + if resp != nil { + resp.RemainingCommitted = nil + } + return resp, err +} + +// cancelledCountServer serves ListCommittedLog under an already-cancelled +// context, so any remaining-entries count it runs is cut short. +type cancelledCountServer struct { + *Server +} + +func (s cancelledCountServer) ListCommittedLog(ctx context.Context, req *LogListRequest) (*LogListResponse, error) { + cancelled, cancel := context.WithCancel(ctx) + cancel() + return s.Server.ListCommittedLog(cancelled, req) +} + +// seedRange seeds n txns (ids first..first+n-1) into d on node. +func seedRange(t *testing.T, node *aeRegNode, d string, first uint64, n int) { + t.Helper() + for i := 0; i < n; i++ { + id := first + uint64(i) + node.seed(t, d, id, node.id, int64(id), int64(id), "v") + } +} + +// TestReplicationLag_SumsUnappliedOverDatabasesThenZero: the gauge for a +// peer is the number of its committed entries this node has not applied, +// summed over databases, and 0 once a round applies them all. +func TestReplicationLag_SumsUnappliedOverDatabasesThenZero(t *testing.T) { + var fail atomic.Bool + fail.Store(true) + peer := newAERegNode(t, 2, "db1", "db2") + seedRange(t, peer, "db1", 100, 3) + seedRange(t, peer, "db2", 200, 2) + peer.serveWrapped(t, func(s *Server) MarmotServiceServer { return switchableFetchServer{Server: s, fail: &fail} }) + + local := newAERegNode(t, 1, "db1", "db2") + addPeer(local.registry, 2, peer.addr, NodeStatus_ALIVE) + ae := local.antiEntropyFor() + gauge := newRecordingGaugeVec() + ae.lagGauge = gauge + + ae.performAntiEntropy() + v, ok := gauge.value(2) + require.True(t, ok, "the lag of a peer whose whole log was listed was not published") + require.Equal(t, float64(5), v, "the lag is not the sum of unapplied entries over databases") + + fail.Store(false) + ae.performAntiEntropy() + v, ok = gauge.value(2) + require.True(t, ok) + require.Zero(t, v, "the lag is not 0 after a round applied everything") +} + +// TestReplicationLag_CountsThePeersRemainderPastThePageCap: a round that +// stops at maxPagesPerPull still publishes the exact lag, from the peer's +// count of the entries past the last page. +func TestReplicationLag_CountsThePeersRemainderPastThePageCap(t *testing.T) { + peer := newAERegNode(t, 2, "app") + seedRange(t, peer, "app", 1000, maxPagesPerPull+3) + peer.serve(t) + + local := newAERegNode(t, 1, "app") + addPeer(local.registry, 2, peer.addr, NodeStatus_ALIVE) + ae := local.antiEntropyFor() + ae.logPuller = NewLogPuller(LogPullerConfig{NodeID: 1, Client: local.client, DBManager: local.dm, PageSize: 1}) + gauge := newRecordingGaugeVec() + ae.lagGauge = gauge + + ae.performAntiEntropy() + v, ok := gauge.value(2) + require.True(t, ok) + require.Equal(t, float64(3), v, "the entries past the page cap were not counted") + + ae.performAntiEntropy() + v, _ = gauge.value(2) + require.Zero(t, v) +} + +// TestReplicationLag_OlderPeerAtThePageCapLeavesTheGaugeUnset: a peer that +// cannot report the entries past the page cap has an unknown lag, which is +// never published as a number. +func TestReplicationLag_OlderPeerAtThePageCapLeavesTheGaugeUnset(t *testing.T) { + peer := newAERegNode(t, 2, "app") + seedRange(t, peer, "app", 1000, maxPagesPerPull+3) + peer.serveWrapped(t, func(s *Server) MarmotServiceServer { return legacyListServer{Server: s} }) + + local := newAERegNode(t, 1, "app") + addPeer(local.registry, 2, peer.addr, NodeStatus_ALIVE) + ae := local.antiEntropyFor() + ae.logPuller = NewLogPuller(LogPullerConfig{NodeID: 1, Client: local.client, DBManager: local.dm, PageSize: 1}) + gauge := newRecordingGaugeVec() + ae.lagGauge = gauge + + ae.performAntiEntropy() + _, ok := gauge.value(2) + require.False(t, ok, "an unknown lag was published") +} + +// TestReplicationLag_AbandonedCountNeverFailsTheListing: a remaining-entries +// count cut short still serves the page, with remaining_committed unset, +// and the round applies the page but leaves the lag unpublished. +func TestReplicationLag_AbandonedCountNeverFailsTheListing(t *testing.T) { + peer := newAERegNode(t, 2, "app") + seedRange(t, peer, "app", 1000, maxPagesPerPull+3) + peer.serveWrapped(t, func(s *Server) MarmotServiceServer { return cancelledCountServer{Server: s} }) + + server := &Server{nodeID: 2, registry: NewNodeRegistry(2, "127.0.0.1:0")} + server.SetDatabaseManager(peer.dm) + resp, err := cancelledCountServer{Server: server}.ListCommittedLog(context.Background(), + &LogListRequest{Database: "app", Limit: 1, CountRemaining: true}) + require.NoError(t, err, "an abandoned count failed the listing") + require.Len(t, resp.Entries, 1) + require.True(t, resp.More) + require.Nil(t, resp.RemainingCommitted, "an abandoned count was reported") + + local := newAERegNode(t, 1, "app") + addPeer(local.registry, 2, peer.addr, NodeStatus_ALIVE) + ae := local.antiEntropyFor() + ae.logPuller = NewLogPuller(LogPullerConfig{NodeID: 1, Client: local.client, DBManager: local.dm, PageSize: 1}) + gauge := newRecordingGaugeVec() + ae.lagGauge = gauge + + ae.performAntiEntropy() + require.Len(t, local.rows(t, "app"), maxPagesPerPull, "the round did not apply every page it listed") + _, ok := gauge.value(2) + require.False(t, ok, "an unknown lag was published") +} + +// TestReplicationLag_RemovedPeerSeriesDeleted: a peer that leaves +// membership stops being exported. +func TestReplicationLag_RemovedPeerSeriesDeleted(t *testing.T) { + peer := newAERegNode(t, 2, "app") + seedRange(t, peer, "app", 1000, 2) + peer.serve(t) + + local := newAERegNode(t, 1, "app") + addPeer(local.registry, 2, peer.addr, NodeStatus_ALIVE) + ae := local.antiEntropyFor() + gauge := newRecordingGaugeVec() + ae.lagGauge = gauge + + ae.performAntiEntropy() + _, ok := gauge.value(2) + require.True(t, ok) + + require.NoError(t, local.registry.MarkRemoved(2)) + ae.performAntiEntropy() + _, ok = gauge.value(2) + require.False(t, ok, "a removed peer's lag series lingers") +} + +// TestReplicationLag_CorrectAfterSnapshotRestore: the round that falls back +// to a snapshot cannot count the lag and leaves it unset; the next round, +// from the cursor the restore reset, publishes the true value. +func TestReplicationLag_CorrectAfterSnapshotRestore(t *testing.T) { + source := newAERegNode(t, 2, "app") + source.seed(t, "app", 1, 2, 1000, 99, "throwaway") + source.truncateThrough(t, "app") + seedRange(t, source, "app", 100, 2) + source.serve(t) + + local := newAERegNode(t, 1, "app") + addPeer(local.registry, 2, source.addr, NodeStatus_ALIVE) + ae := local.antiEntropyFor() + gauge := newRecordingGaugeVec() + ae.lagGauge = gauge + + ae.performAntiEntropy() + _, ok := gauge.value(2) + require.False(t, ok, "the snapshot-fallback round published a lag it could not count") + + ae.performAntiEntropy() + v, ok := gauge.value(2) + require.True(t, ok) + require.Zero(t, v, "the lag after a restore is not the true value") +} diff --git a/telemetry/metrics.go b/telemetry/metrics.go index 31da3e80..5357f035 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -116,7 +116,10 @@ var ( // AntiEntropyDurationSeconds measures anti-entropy round duration AntiEntropyDurationSeconds Histogram = NoopStat{} - // ReplicationLagTxns tracks transaction lag per peer + // ReplicationLagTxns tracks, per peer, the committed transactions of + // that peer's local commit log past this node's pull cursor, summed over + // databases, as of the last pull round that could count them all: an + // upper bound on what catch-up still has to do ReplicationLagTxns GaugeVec = noopGaugeVec{} ) @@ -295,7 +298,7 @@ func InitMetrics() { ) ReplicationLagTxns = NewGaugeVec( "replication_lag_txns", - "Transaction lag behind peer", + "Committed transactions past this node's pull cursor into the peer's log, as of the last pull round, summed over databases (an upper bound on catch-up still to do)", []string{"peer"}, ) diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index c5e60f01..de9196ac 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -38,6 +38,9 @@ type CounterVec interface { type GaugeVec interface { With(labels ...string) Gauge + // Delete removes the series for labels, if any, so a label value that + // no longer exists (a removed peer, say) stops being exported. + Delete(labels ...string) } type HistogramVec interface { @@ -53,6 +56,7 @@ type noopHistogramVec struct{} func (n noopCounterVec) With(labels ...string) Counter { return NoopStat{} } func (n noopGaugeVec) With(labels ...string) Gauge { return NoopStat{} } +func (n noopGaugeVec) Delete(labels ...string) {} func (n noopHistogramVec) With(labels ...string) Histogram { return NoopStat{} } // Prometheus Vec wrappers @@ -74,6 +78,10 @@ func (p *prometheusGaugeVec) With(labelValues ...string) Gauge { return p.vec.WithLabelValues(labelValues...) } +func (p *prometheusGaugeVec) Delete(labelValues ...string) { + p.vec.DeleteLabelValues(labelValues...) +} + type prometheusHistogramVec struct { vec *prometheus.HistogramVec labels []string From 2bb68a0486d87df2bd6e38c27d7b282bc2ed556f Mon Sep 17 00:00:00 2001 From: Zohaib Sibte Hassan Date: Wed, 30 Sep 2026 07:44:05 -0500 Subject: [PATCH 17/17] chore: bump version to v2.10.0-beta Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014u4bCrv6aSsNKiY6oWuvrt --- build-linux.sh | 2 +- config.toml | 2 +- config.toml.example | 2 +- examples/README.md | 4 ++-- examples/cluster-with-replicas/node-1-config.toml | 2 +- examples/cluster-with-replicas/node-2-config.toml | 2 +- examples/cluster-with-replicas/replica-1a-config.toml | 2 +- examples/cluster-with-replicas/replica-2a-config.toml | 2 +- .../cluster-with-replicas/run-cluster-with-replicas.sh | 8 ++++---- examples/node-1-config.toml | 2 +- examples/node-2-config.toml | 2 +- examples/node-3-config.toml | 2 +- examples/run-cluster.sh | 6 +++--- examples/run-single-node.sh | 6 +++--- examples/start-replica.sh | 4 ++-- examples/wordpress-cluster/marmot-1-config.toml | 2 +- examples/wordpress-cluster/marmot-2-config.toml | 2 +- examples/wordpress-cluster/marmot-3-config.toml | 2 +- examples/wordpress-cluster/run.sh | 4 ++-- examples/wordpress/Dockerfile | 2 +- examples/wordpress/marmot-config.toml | 2 +- examples/wordpress/run.sh | 8 ++++---- marmot.go | 6 +++--- publisher/doc.go | 2 +- 24 files changed, 39 insertions(+), 39 deletions(-) diff --git a/build-linux.sh b/build-linux.sh index 0c429e81..7e5e694c 100755 --- a/build-linux.sh +++ b/build-linux.sh @@ -1,6 +1,6 @@ #!/bin/sh -# Build Marmot v2.9.16-beta for Linux (static binary) +# Build Marmot v2.10.0-beta for Linux (static binary) # Requires musl cross-compiler: brew install FiloSottile/musl-cross/musl-cross CC=x86_64-linux-musl-gcc \ diff --git a/config.toml b/config.toml index a298678c..56448819 100644 --- a/config.toml +++ b/config.toml @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta Configuration +# Marmot v2.10.0-beta Configuration # Leaderless SQLite Replication # ============================================================================== diff --git a/config.toml.example b/config.toml.example index c7a33fef..2c691f37 100644 --- a/config.toml.example +++ b/config.toml.example @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta Configuration +# Marmot v2.10.0-beta Configuration # Leaderless SQLite Replication # ============================================================================== diff --git a/examples/README.md b/examples/README.md index e110a651..3a35f970 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta Examples +# Marmot v2.10.0-beta Examples Quick start examples for running Marmot in different configurations. @@ -122,7 +122,7 @@ Each config includes: ### Full Database Replication -Marmot v2.9.16-beta uses **full database replication**, not partitioning: +Marmot v2.10.0-beta uses **full database replication**, not partitioning: - **Every node** has a complete copy of the database - **Writes** go to all nodes (broadcast) diff --git a/examples/cluster-with-replicas/node-1-config.toml b/examples/cluster-with-replicas/node-1-config.toml index 3d53dedd..0c033db3 100644 --- a/examples/cluster-with-replicas/node-1-config.toml +++ b/examples/cluster-with-replicas/node-1-config.toml @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta - Cluster Node 1 (Primary) +# Marmot v2.10.0-beta - Cluster Node 1 (Primary) # 2-node cluster with read-only replicas node_id = 1 diff --git a/examples/cluster-with-replicas/node-2-config.toml b/examples/cluster-with-replicas/node-2-config.toml index ddcba208..acbeb928 100644 --- a/examples/cluster-with-replicas/node-2-config.toml +++ b/examples/cluster-with-replicas/node-2-config.toml @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta - Cluster Node 2 (Primary) +# Marmot v2.10.0-beta - Cluster Node 2 (Primary) # 2-node cluster with read-only replicas node_id = 2 diff --git a/examples/cluster-with-replicas/replica-1a-config.toml b/examples/cluster-with-replicas/replica-1a-config.toml index cd9f2943..bd560a5b 100644 --- a/examples/cluster-with-replicas/replica-1a-config.toml +++ b/examples/cluster-with-replicas/replica-1a-config.toml @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta - Read-Only Replica 1A +# Marmot v2.10.0-beta - Read-Only Replica 1A # Follows cluster nodes for transparent failover node_id = 101 diff --git a/examples/cluster-with-replicas/replica-2a-config.toml b/examples/cluster-with-replicas/replica-2a-config.toml index a6a60428..ace2669b 100644 --- a/examples/cluster-with-replicas/replica-2a-config.toml +++ b/examples/cluster-with-replicas/replica-2a-config.toml @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta - Read-Only Replica 2A +# Marmot v2.10.0-beta - Read-Only Replica 2A # Follows cluster nodes for transparent failover node_id = 102 diff --git a/examples/cluster-with-replicas/run-cluster-with-replicas.sh b/examples/cluster-with-replicas/run-cluster-with-replicas.sh index 0b0d17d2..8625e100 100755 --- a/examples/cluster-with-replicas/run-cluster-with-replicas.sh +++ b/examples/cluster-with-replicas/run-cluster-with-replicas.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Marmot v2.9.16-beta - 2-Node Cluster with Read-Only Replicas +# Marmot v2.10.0-beta - 2-Node Cluster with Read-Only Replicas # # Topology: # Node 1 (primary) ←→ Node 2 (primary) [2-node cluster with QUORUM writes] @@ -24,7 +24,7 @@ generate_secret() { CLUSTER_SECRET=$(generate_secret) REPLICA_SECRET=$(generate_secret) -echo "=== Marmot v2.9.16-beta Cluster with Replicas ===" +echo "=== Marmot v2.10.0-beta Cluster with Replicas ===" echo "Starting 2-node cluster + 2 read-only replicas" echo "" @@ -60,7 +60,7 @@ generate_node_config() { local config_file="/tmp/marmot-cluster/node-${node_id}.toml" cat > "$config_file" << EOF -# Marmot v2.9.16-beta - Node ${node_id} (auto-generated) +# Marmot v2.10.0-beta - Node ${node_id} (auto-generated) node_id = ${node_id} data_dir = "/tmp/marmot-cluster/node-${node_id}" @@ -143,7 +143,7 @@ generate_replica_config() { local config_file="/tmp/marmot-cluster/replica-${replica_id}a.toml" cat > "$config_file" << EOF -# Marmot v2.9.16-beta - Replica ${replica_id}A (auto-generated) +# Marmot v2.10.0-beta - Replica ${replica_id}A (auto-generated) node_id = ${replica_id}00 data_dir = "/tmp/marmot-cluster/replica-${replica_id}a" diff --git a/examples/node-1-config.toml b/examples/node-1-config.toml index d4229c0b..070d7554 100644 --- a/examples/node-1-config.toml +++ b/examples/node-1-config.toml @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta - Example Node 1 +# Marmot v2.10.0-beta - Example Node 1 # Full Database Replication node_id = 1 diff --git a/examples/node-2-config.toml b/examples/node-2-config.toml index 80a402c5..1d7c0ecb 100644 --- a/examples/node-2-config.toml +++ b/examples/node-2-config.toml @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta - Example Node 2 +# Marmot v2.10.0-beta - Example Node 2 # Full Database Replication node_id = 2 diff --git a/examples/node-3-config.toml b/examples/node-3-config.toml index 0fc7b9c1..b2fb68fc 100644 --- a/examples/node-3-config.toml +++ b/examples/node-3-config.toml @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta - Example Node 3 +# Marmot v2.10.0-beta - Example Node 3 # Full Database Replication node_id = 3 diff --git a/examples/run-cluster.sh b/examples/run-cluster.sh index 007551da..d94e6783 100755 --- a/examples/run-cluster.sh +++ b/examples/run-cluster.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Marmot v2.9.16-beta - Example 3-Node Cluster +# Marmot v2.10.0-beta - Example 3-Node Cluster # Full Database Replication: ALL nodes get ALL data set -e @@ -37,7 +37,7 @@ while [[ $# -gt 0 ]]; do esac done -echo "=== Marmot v2.9.16-beta Example Cluster ===" +echo "=== Marmot v2.10.0-beta Example Cluster ===" echo "Starting 3-node cluster with full database replication" echo "" @@ -98,7 +98,7 @@ generate_node_config() { done cat > "$config_file" << EOF -# Marmot v2.9.16-beta - Node ${node_id} (auto-generated) +# Marmot v2.10.0-beta - Node ${node_id} (auto-generated) node_id = ${node_id} data_dir = "${data_dir}" diff --git a/examples/run-single-node.sh b/examples/run-single-node.sh index 7eb6b343..4fddd5c2 100755 --- a/examples/run-single-node.sh +++ b/examples/run-single-node.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Marmot v2.9.16-beta - Single Node Example +# Marmot v2.10.0-beta - Single Node Example # Simplest way to run Marmot set -e @@ -7,7 +7,7 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -echo "=== Marmot v2.9.16-beta Single Node ===" +echo "=== Marmot v2.10.0-beta Single Node ===" echo "" # Kill any existing marmot processes @@ -25,7 +25,7 @@ mkdir -p /tmp/marmot-single # Create config cat > /tmp/marmot-single/config.toml <<'TOML' -# Marmot v2.9.16-beta - Single Node Configuration (Optimized for Benchmarks) +# Marmot v2.10.0-beta - Single Node Configuration (Optimized for Benchmarks) node_id = 1 data_dir = "/tmp/marmot-single" diff --git a/examples/start-replica.sh b/examples/start-replica.sh index 54ad7670..b7c9aeb3 100755 --- a/examples/start-replica.sh +++ b/examples/start-replica.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Marmot v2.9.16-beta - Read-Only Replica with Transparent Failover +# Marmot v2.10.0-beta - Read-Only Replica with Transparent Failover # Streams from cluster nodes with automatic failover set -e @@ -29,7 +29,7 @@ generate_secret() { REPLICA_SECRET=${3:-${MARMOT_REPLICA_SECRET:-$(generate_secret)}} -echo "=== Marmot v2.9.16-beta Read-Only Replica ===" +echo "=== Marmot v2.10.0-beta Read-Only Replica ===" echo "Replica with transparent failover to cluster nodes" echo "" diff --git a/examples/wordpress-cluster/marmot-1-config.toml b/examples/wordpress-cluster/marmot-1-config.toml index 750fb23d..edbbd36e 100644 --- a/examples/wordpress-cluster/marmot-1-config.toml +++ b/examples/wordpress-cluster/marmot-1-config.toml @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta - WordPress Cluster Node 1 +# Marmot v2.10.0-beta - WordPress Cluster Node 1 node_id = 1 data_dir = "/data" diff --git a/examples/wordpress-cluster/marmot-2-config.toml b/examples/wordpress-cluster/marmot-2-config.toml index 794cc7bb..ce4b3cf4 100644 --- a/examples/wordpress-cluster/marmot-2-config.toml +++ b/examples/wordpress-cluster/marmot-2-config.toml @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta - WordPress Cluster Node 2 +# Marmot v2.10.0-beta - WordPress Cluster Node 2 node_id = 2 data_dir = "/data" diff --git a/examples/wordpress-cluster/marmot-3-config.toml b/examples/wordpress-cluster/marmot-3-config.toml index 9eec31be..bc54c0bb 100644 --- a/examples/wordpress-cluster/marmot-3-config.toml +++ b/examples/wordpress-cluster/marmot-3-config.toml @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta - WordPress Cluster Node 3 +# Marmot v2.10.0-beta - WordPress Cluster Node 3 node_id = 3 data_dir = "/data" diff --git a/examples/wordpress-cluster/run.sh b/examples/wordpress-cluster/run.sh index 85f81ab6..951ce3f5 100755 --- a/examples/wordpress-cluster/run.sh +++ b/examples/wordpress-cluster/run.sh @@ -15,7 +15,7 @@ NC='\033[0m' print_banner() { echo "" echo -e "${CYAN}╔══════════════════════════════════════════════════════════════╗${NC}" - echo -e "${CYAN}║${NC} ${BOLD}Marmot v2.9.16-beta + WordPress Cluster Demo${NC} ${CYAN}║${NC}" + echo -e "${CYAN}║${NC} ${BOLD}Marmot v2.10.0-beta + WordPress Cluster Demo${NC} ${CYAN}║${NC}" echo -e "${CYAN}║${NC} 3-Node Distributed SQLite powering 3 WordPress instances ${CYAN}║${NC}" echo -e "${CYAN}╚══════════════════════════════════════════════════════════════╝${NC}" echo "" @@ -54,7 +54,7 @@ start_stack() { echo -e "${YELLOW}[1/4]${NC} Cleaning up any existing containers..." docker compose down -v 2>/dev/null || true - echo -e "${YELLOW}[2/4]${NC} Building Marmot v2.9.16-beta image (this may take a few minutes on first run)..." + echo -e "${YELLOW}[2/4]${NC} Building Marmot v2.10.0-beta image (this may take a few minutes on first run)..." docker compose build echo -e "${YELLOW}[3/4]${NC} Starting services..." diff --git a/examples/wordpress/Dockerfile b/examples/wordpress/Dockerfile index b2b2a700..17d9ec4d 100644 --- a/examples/wordpress/Dockerfile +++ b/examples/wordpress/Dockerfile @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta Docker Image +# Marmot v2.10.0-beta Docker Image # Multi-stage build FROM golang:1.27-bookworm AS builder diff --git a/examples/wordpress/marmot-config.toml b/examples/wordpress/marmot-config.toml index 58b9fdf0..695d290f 100644 --- a/examples/wordpress/marmot-config.toml +++ b/examples/wordpress/marmot-config.toml @@ -1,4 +1,4 @@ -# Marmot v2.9.16-beta - WordPress Single Node Configuration +# Marmot v2.10.0-beta - WordPress Single Node Configuration node_id = 1 data_dir = "/data" diff --git a/examples/wordpress/run.sh b/examples/wordpress/run.sh index 0ef996c2..43e25898 100755 --- a/examples/wordpress/run.sh +++ b/examples/wordpress/run.sh @@ -15,7 +15,7 @@ NC='\033[0m' print_banner() { echo "" echo -e "${CYAN}╔══════════════════════════════════════════════════════════════╗${NC}" - echo -e "${CYAN}║${NC} ${BOLD}Marmot v2.9.16-beta + WordPress Demo${NC} ${CYAN}║${NC}" + echo -e "${CYAN}║${NC} ${BOLD}Marmot v2.10.0-beta + WordPress Demo${NC} ${CYAN}║${NC}" echo -e "${CYAN}║${NC} Distributed SQLite powering WordPress ${CYAN}║${NC}" echo -e "${CYAN}╚══════════════════════════════════════════════════════════════╝${NC}" echo "" @@ -54,7 +54,7 @@ start_stack() { echo -e "${YELLOW}[1/4]${NC} Cleaning up any existing containers..." docker compose down -v 2>/dev/null || true - echo -e "${YELLOW}[2/4]${NC} Building Marmot v2.9.16-beta image (this may take a few minutes on first run)..." + echo -e "${YELLOW}[2/4]${NC} Building Marmot v2.10.0-beta image (this may take a few minutes on first run)..." docker compose build --quiet echo -e "${YELLOW}[3/4]${NC} Starting services..." @@ -137,7 +137,7 @@ print_success() { echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}" echo "" echo -e "${BOLD}Services Running:${NC}" - echo -e " ${CYAN}Marmot v2.9.16-beta${NC} - Distributed SQLite with MySQL protocol" + echo -e " ${CYAN}Marmot v2.10.0-beta${NC} - Distributed SQLite with MySQL protocol" echo -e " ${CYAN}WordPress${NC} - Connected to Marmot as its database backend" echo "" echo -e "${BOLD}Access Points:${NC}" @@ -153,7 +153,7 @@ print_success() { echo "" echo -e "${BOLD}Architecture:${NC}" echo -e " ┌─────────────┐ ┌──────────────────┐" - echo -e " │ WordPress │───>│ Marmot v2.9.16-beta │" + echo -e " │ WordPress │───>│ Marmot v2.10.0-beta │" echo -e " │ (port 8080)│ │ MySQL: 3316 │" echo -e " └─────────────┘ │ gRPC: 8090 │" echo -e " │ (internal only) │" diff --git a/marmot.go b/marmot.go index 8de9b71c..9cd1ae04 100644 --- a/marmot.go +++ b/marmot.go @@ -119,7 +119,7 @@ func main() { // Branch based on operating mode if cfg.IsReplicaMode() { - log.Info().Msg("Marmot v2.9.16-beta - Read-Only Replica Mode") + log.Info().Msg("Marmot v2.10.0-beta - Read-Only Replica Mode") log.Info(). Strs("follow_addresses", cfg.Config.Replica.FollowAddresses). Msg("Following cluster nodes") @@ -135,7 +135,7 @@ func main() { } // Cluster mode - log.Info().Msg("Marmot v2.9.16-beta - Leaderless SQLite Replication") + log.Info().Msg("Marmot v2.10.0-beta - Leaderless SQLite Replication") // Warn if cluster authentication is not configured if !cfg.IsClusterAuthEnabled() { @@ -574,7 +574,7 @@ func main() { log.Info().Msg("Seed node fully initialized - now ALIVE") } - log.Info().Msg("Marmot v2.9.16-beta started successfully") + log.Info().Msg("Marmot v2.10.0-beta started successfully") log.Info(). Uint64("node_id", cfg.Config.NodeID). Int("grpc_port", cfg.Config.Cluster.GRPCPort). diff --git a/publisher/doc.go b/publisher/doc.go index 15be6eb3..5dab41b5 100644 --- a/publisher/doc.go +++ b/publisher/doc.go @@ -1,5 +1,5 @@ // Package publisher provides the CDC (Change Data Capture) Publishing System -// foundation for Marmot v2.9.16-beta. +// foundation for Marmot v2.10.0-beta. // // This package implements a durable, ordered event log backed by Pebble that // captures CDC events and tracks per-sink consumption cursors for reliable