From 23bdc932d3499ed1888d889eb78941aa19fe4738 Mon Sep 17 00:00:00 2001 From: Diederik Siderius Date: Fri, 4 Sep 2026 11:36:26 +0200 Subject: [PATCH 01/14] feat: rows-changed counter, flagged by codegen rather than counted by the opcode (#692) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in `src/vdbe/` reported how many rows an `INSERT`/`UPDATE`/ `DELETE` changed. Spec 013 calls this the one item on its list a consumer cannot work around: `execute_transaction_step` returns rows and the autocommit flag, so a caller cannot tell an `UPDATE` that matched from one that did not, and that is the distinction every optimistic-concurrency scheme is built on. SQE swaps a table's metadata pointer with a conditional `UPDATE` and treats zero rows affected as a lost race; without the count that becomes SELECT-then-UPDATE, sound only while the consumer guarantees a single writer. The obvious implementation is wrong, and measurably so. Counting in the `Insert`/`Delete` handlers reports, per changed row: INSERT Insert -> 1 DELETE Delete -> 1 UPDATE, single-pass Delete + Insert -> 2 UPDATE, two-pass range-seek ephemeral Insert + Delete + Insert -> 3 The two-pass plan is #666/#675's range-seek path, which stashes matched rowids in an ephemeral b-tree using the same `Opcode::Insert`. So one `UPDATE` reports 2 or 3 depending on which plan the optimizer picked, and neither is 1. Index maintenance is the same shape: a write next to a row that is not a row change. So codegen marks the one mutation that counts, with `OPFLAG_NCHANGE` (`0x01`) on `P5` — stock SQLite's flag, same bit, same job. `P5` was unread by both opcodes, so nothing had to move, and no new opcode means the frozen-set ADRs (0015/0018/0020) stay closed. An `UPDATE` flags its `Insert` and not the paired `Delete`: one changed row, counted once. `StepOutcome::changes` is `Option`, and the two cases are not the same answer. `Some(0)` is "this was a DML statement and it changed nothing" — the lost race. `None` is "not that kind of statement", so a connection tracking `sqlite3_changes()` leaves its stored count alone after a `SELECT`. `Program::counts_changes()` is the discriminator and is deliberately *static*: an `UPDATE` whose `WHERE` matches nothing never executes its flagged `Insert` but must still report `Some(0)`. `execute_transaction_step` is now a wrapper over `execute_transaction_step_counted`, per ADR-0040's pattern — one loop, the old signature expressed in terms of the new one, so they cannot drift and its ten existing call sites are untouched. ADR-0042 records all of it, including why `u64` instead of `Option` would be a bug rather than a simplification. Both wrong designs are mutation-checked, not just argued: - counting unconditionally in the handlers fails `update_of_one_row_reports_one_under_both_plans`, `index_maintenance_does_not_count` and `conditional_update_reports_match`; - additionally flagging `UPDATE`'s `Delete` fails the same three, and fails the oracle diff with 4 against the oracle's 2. Verified: 1569 unit tests (1562 baseline + 7) and 381 corpus (380 + 1) pass, clippy/fmt/mod-files clean, assurance 86/86 and 276/276 with no dead links. `tests/corpus/changes_oracle_test.rs` diffs a thirteen-statement sequence against the pinned 3.53.4 oracle's own `changes()`, covering both `UPDATE` plans, a miss, a partial `DELETE` and a full one. Not included: `Connection::changes` and the cross-statement retention rule. A `Vm` lives for one statement so it cannot own that rule; it is spec 013/Req 1's surface and belongs to the facade ticket, where it is one line — store on `Some`, ignore `None`. One note for whoever merges second: `Execution` (#683) should grow a `changes()` the same way, which is a three-line addition once both are on `main`. The requirement IDs cited here live in #678, not yet on `main`. Refs: 013/Req-1, #692, #678, #683 Co-Authored-By: Claude Opus 5 (1M context) --- ...42-rows-changed-counted-by-codegen-flag.md | 103 ++++++ .openspec/adr/index.md | 1 + Cargo.toml | 4 + src/codegen/stmt/delete.rs | 18 +- src/codegen/stmt/insert.rs | 8 +- src/codegen/stmt/update.rs | 10 +- src/vdbe.rs | 7 +- src/vdbe/cursor.rs | 17 +- src/vdbe/exec.rs | 92 +++++- src/vdbe/program.rs | 47 +++ tests/corpus/changes_oracle_test.rs | 197 ++++++++++++ tests/corpus/main.rs | 1 + tests/unit/vdbe_changes_test.rs | 299 ++++++++++++++++++ 13 files changed, 785 insertions(+), 19 deletions(-) create mode 100644 .openspec/adr/0042-rows-changed-counted-by-codegen-flag.md create mode 100644 tests/corpus/changes_oracle_test.rs create mode 100644 tests/unit/vdbe_changes_test.rs diff --git a/.openspec/adr/0042-rows-changed-counted-by-codegen-flag.md b/.openspec/adr/0042-rows-changed-counted-by-codegen-flag.md new file mode 100644 index 00000000..f9c7dcc8 --- /dev/null +++ b/.openspec/adr/0042-rows-changed-counted-by-codegen-flag.md @@ -0,0 +1,103 @@ +# 0042 — Codegen decides which mutation is a row change, and `None` is not zero + +**Status:** Accepted · **Date:** 2026-09-04 + +## Context + +Spec 013 Requirement 1 asks for `sqlite3_changes()`: how many rows the last +`INSERT`/`UPDATE`/`DELETE` changed. Spec 013 calls it the one item on its list +a consumer cannot work around, because without it a caller cannot distinguish +an `UPDATE` that matched from one that did not, and that distinction is what +every optimistic-concurrency scheme is built on. + +The obvious implementation — increment a counter in the `Insert` and `Delete` +opcode handlers — is wrong, and measurably so on the tree at 0.18.10: + +| statement | opcodes emitted per row | counted naively | +|---|---|---| +| `INSERT` | `Insert` | 1 | +| `DELETE` | `Delete` | 1 | +| `UPDATE`, single-pass | `Delete` + `Insert` (`update.rs:603,605`) | **2** | +| `UPDATE`, two-pass range-seek | ephemeral `Insert` (`update.rs:276`) + `Delete` + `Insert` | **3** | + +The two-pass plan is #666/#675's range-seek path, which stashes matched rowids +in an ephemeral b-tree using the same `Opcode::Insert`. So the same `UPDATE` +would report 2 or 3 depending on which plan the optimizer picked, and neither +is 1. Index maintenance (`IdxInsert`, `IdxDelete`, `AutoIndexInsert`) has the +same character: a write, adjacent to a row, that is not a row change. + +The opcode does not carry enough information to answer. Codegen does. + +## Decision + +**Codegen marks the one mutation that is the row change, with +`OPFLAG_NCHANGE` (`0x01`) on `P5`.** Same bit and same job as stock SQLite's +flag of that name. `cursor::insert`/`cursor::delete` increment `Vm`'s counter +only when it is set; `Instruction::with_p5` is the constructor that sets it, +alongside the existing `with_p4`. `P5` was unread by both opcodes, so nothing +had to move. + +An `UPDATE` flags its `Insert` and not the paired `Delete` — one changed row, +counted once. `INSERT` flags its table `Insert`; `DELETE` flags both of its +`Delete` sites. Nothing else is ever flagged. + +**The count is exposed as `Option`, and `None` is not `Some(0)`.** +`StepOutcome::changes` is `Some(n)` when the program is a counting statement +and `None` when it is not: + +- `Some(0)` means "this was an `INSERT`/`UPDATE`/`DELETE` and it changed + nothing" — a lost optimistic-concurrency race, which is the case the + requirement exists to make visible. +- `None` means "not that kind of statement", so a connection tracking + `sqlite3_changes()` leaves its stored count alone. + +The discriminator is **static** — `Program::counts_changes()` asks whether the +program *contains* a flagged instruction, not whether one executed. An +`UPDATE` whose `WHERE` matches nothing never runs its flagged `Insert` but +must still report `Some(0)`. + +**`execute_transaction_step` becomes a wrapper** over +`execute_transaction_step_counted`, which returns the count. Same pattern +ADR-0040 settled on for streaming: one loop, the older signature expressed in +terms of the newer one, so the two cannot drift and the existing suite is the +equivalence proof. + +## Alternatives rejected + +- **Count in the handlers, unconditionally.** Reports 2 or 3 for a one-row + `UPDATE`, plan-dependently, and counts index maintenance. This is the + alternative the table above exists to close, and + `update_of_one_row_reports_one_under_both_plans` is its regression guard: + removing the flag check fails that test and two others. +- **Return `u64` and let `0` mean both.** Collapses "changed nothing" into + "not a counting statement", which is exactly the distinction SQLite's + retention rule is built on — a `SELECT` would zero a count that should have + survived it. The two-case type is the whole point and should not be + simplified away. +- **A `Program { counts_changes: bool }` field set by codegen.** Equivalent + in behaviour, but it can disagree with the instructions it describes, and + `Program::new` has many call sites. Deriving it costs one pass over a + handful of instructions. +- **Change `execute_transaction_step`'s return type in place.** Ten call + sites across `src/bin/`, tests, benches and examples, for a value almost + none of them want. The wrapper is free. +- **Track the count across statements in the `Vm`.** A `Vm` lives for one + statement, so it cannot. Cross-statement retention is the connection's + rule and belongs to spec 013/Req 1's `Connection::changes`. + +## Consequences + +The number is correct for the statement just run, verified against the pinned +3.53.4 oracle's own `changes()` for a thirteen-statement sequence covering +both `UPDATE` plans, a miss, a partial `DELETE` and a full one +(`tests/corpus/changes_oracle_test.rs`). Both wrong designs above were +mutation-checked against that test as well as the unit suite. + +`Connection::changes` is still absent — this is the engine half. What the +facade has left to do is one line: store the value on `Some`, ignore `None`. + +Adding a `P5` flag reopens no frozen set: no new opcode, so ADR-0015, ADR-0018 +and ADR-0020 are untouched. But `P5` on `Insert` is now meaningful where its +doc comment previously said conflict-resolution flags were "not modeled", so a +future `OR REPLACE`/`OR IGNORE` implementation must pick bits other than +`0x01`. diff --git a/.openspec/adr/index.md b/.openspec/adr/index.md index 2b4d7e6c..09d3898f 100644 --- a/.openspec/adr/index.md +++ b/.openspec/adr/index.md @@ -42,3 +42,4 @@ Specs record what the system must do; ADRs record **why it is shaped this way** | [0036](0036-pragma-synchronous-fsync-policy.md) | `PRAGMA synchronous` fsync-skip policy, and why `SynchronousMode` lives in `header.rs` | 2026-08-29 | | [0037](0037-macos-plain-fsync-not-fullfsync.md) | On macOS, `Vfs::sync` calls plain `fsync(2)`, not `std`'s `F_FULLFSYNC` | 2026-08-30 | | [0038](0038-cargo-registry-opt-in-not-committed.md) | Artifactory Cargo access is opt-in local config, never a committed source replacement | 2026-09-03 | +| [0042](0042-rows-changed-counted-by-codegen-flag.md) | Codegen flags the one mutation that is a row change; `None` is not `Some(0)` | 2026-09-04 | diff --git a/Cargo.toml b/Cargo.toml index 40e704c9..be40477d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,6 +128,10 @@ path = "tests/unit/vdbe_write_opcodes_test.rs" name = "vdbe_integrity_check" path = "tests/unit/vdbe_integrity_check_test.rs" +[[test]] +name = "vdbe_changes" +path = "tests/unit/vdbe_changes_test.rs" + [[test]] name = "codegen_expr" path = "tests/unit/codegen_expr_test.rs" diff --git a/src/codegen/stmt/delete.rs b/src/codegen/stmt/delete.rs index bb53f42a..46488b45 100644 --- a/src/codegen/stmt/delete.rs +++ b/src/codegen/stmt/delete.rs @@ -28,7 +28,7 @@ use crate::codegen::select::{is_rowid_reference, top_level_equality_operands, Co use crate::codegen::{CondTargets, Emitter, RegAlloc, Scope, Target}; use crate::parser::ast::{Delete, ExprKind, Literal, ParamKind}; use crate::schema::TableSchema; -use crate::vdbe::{Instruction, Opcode, Program}; +use crate::vdbe::{Instruction, Opcode, Program, OPFLAG_NCHANGE}; const TABLE_CURSOR: i32 = 0; const FIRST_INDEX_CURSOR: i32 = 1; @@ -118,7 +118,13 @@ pub fn compile_delete_with_catalog( FIRST_INDEX_CURSOR, Opcode::IdxDelete, )?; - em.emit(Instruction::new(Opcode::Delete, TABLE_CURSOR, 0, 0)); + em.emit(Instruction::with_p5( + Opcode::Delete, + TABLE_CURSOR, + 0, + 0, + OPFLAG_NCHANGE, + )); em.place(end_label); em.emit(Instruction::new(Opcode::Halt, 0, 0, 0)); @@ -149,7 +155,13 @@ pub fn compile_delete_with_catalog( FIRST_INDEX_CURSOR, Opcode::IdxDelete, )?; - em.emit(Instruction::new(Opcode::Delete, TABLE_CURSOR, 0, 0)); + em.emit(Instruction::with_p5( + Opcode::Delete, + TABLE_CURSOR, + 0, + 0, + OPFLAG_NCHANGE, + )); em.place(row_skip); let next_addr = em.emit(Instruction::new(Opcode::Next, TABLE_CURSOR, 0, 0)); diff --git a/src/codegen/stmt/insert.rs b/src/codegen/stmt/insert.rs index a9f239a9..139811c7 100644 --- a/src/codegen/stmt/insert.rs +++ b/src/codegen/stmt/insert.rs @@ -95,7 +95,7 @@ use crate::parser::ast::{ use crate::parser::error::ParseOutcome; use crate::parser::parse_create_table; use crate::schema::TableSchema; -use crate::vdbe::{affinity_of, Instruction, Opcode, Program, P4}; +use crate::vdbe::{affinity_of, Instruction, Opcode, Program, OPFLAG_NCHANGE, P4}; /// Process-wide cache of parsed `CREATE TABLE` DDL, keyed by the exact /// `schema.sql` text — content-addressed, since the parse result depends @@ -786,11 +786,15 @@ fn compile_row( record_reg, P4::Affinity(affinities), )); - em.emit(Instruction::new( + // `OPFLAG_NCHANGE`: this is the one mutation an INSERT reports as a + // changed row (013/Req 1, #692). The index maintenance emitted just + // below is deliberately unflagged. + em.emit(Instruction::with_p5( Opcode::Insert, TABLE_CURSOR, rowid_reg, record_reg, + OPFLAG_NCHANGE, )); if !schema.indexes.is_empty() { diff --git a/src/codegen/stmt/update.rs b/src/codegen/stmt/update.rs index 6785f020..43ecff6c 100644 --- a/src/codegen/stmt/update.rs +++ b/src/codegen/stmt/update.rs @@ -55,7 +55,7 @@ use crate::parser::ast::{ ConflictAction, Expr, ExprKind, Literal, ParamKind, TableConstraint, Update, }; use crate::schema::TableSchema; -use crate::vdbe::{affinity_of, Instruction, Opcode, Program, P4}; +use crate::vdbe::{affinity_of, Instruction, Opcode, Program, OPFLAG_NCHANGE, P4}; const TABLE_CURSOR: i32 = 0; const CHECK_CURSOR: i32 = 1; @@ -600,12 +600,18 @@ fn emit_update_row_body( FIRST_INDEX_CURSOR, Opcode::IdxDelete, )?; + // One changed row, counted once (013/Req 1, #692): an UPDATE rewrites + // a row as `Delete` + `Insert`, so only the `Insert` carries + // `OPFLAG_NCHANGE`. Flagging both would report 2 per row; flagging the + // `Delete` instead would work equally well but reads as a deletion. + // Stock SQLite flags the insert side too. em.emit(Instruction::new(Opcode::Delete, TABLE_CURSOR, 0, 0)); - em.emit(Instruction::new( + em.emit(Instruction::with_p5( Opcode::Insert, TABLE_CURSOR, rowid_reg, record_reg, + OPFLAG_NCHANGE, )); if !schema.indexes.is_empty() { diff --git a/src/vdbe.rs b/src/vdbe.rs index 39e3dad1..954b7d74 100644 --- a/src/vdbe.rs +++ b/src/vdbe.rs @@ -35,8 +35,9 @@ pub use control::{ TRANSACTION_MODE_DEFERRED, TRANSACTION_MODE_EXCLUSIVE, TRANSACTION_MODE_IMMEDIATE, }; pub use exec::{ - execute, execute_transaction_step, execute_with_db, execute_with_db_and_params, - execute_with_params, execute_with_writable_db, ExecError, Step, Vm, + execute, execute_transaction_step, execute_transaction_step_counted, execute_with_db, + execute_with_db_and_params, execute_with_params, execute_with_writable_db, ExecError, Step, + StepOutcome, Vm, }; pub use explain::{explain, ExplainRow}; pub use functions::{call as call_function, like_match, FunctionError}; @@ -46,6 +47,6 @@ pub use pragma::{ }; pub use program::{ AnalyzeIndexTarget, AnalyzeTarget, GroupKeyColumn, Instruction, Opcode, Program, SortKeyColumn, - P4, + OPFLAG_NCHANGE, P4, }; pub use value::{and, is, is_not, not, or, sql_eq, sql_lt}; diff --git a/src/vdbe/cursor.rs b/src/vdbe/cursor.rs index d66e8194..e1688e21 100644 --- a/src/vdbe/cursor.rs +++ b/src/vdbe/cursor.rs @@ -58,7 +58,7 @@ use crate::record::{ record_column_count, TextEncoding, Value, }; use crate::vdbe::exec::{to_pc, ExecError, Step, Vm}; -use crate::vdbe::program::{Instruction, P4}; +use crate::vdbe::program::{Instruction, OPFLAG_NCHANGE, P4}; use crate::vdbe::{compare, Collation}; /// One open cursor slot: a real table cursor, an in-memory ephemeral @@ -1964,6 +1964,13 @@ pub fn delete(vm: &mut Vm, instr: &Instruction) -> Result { if let CursorSlot::Table(state) = vm.cursor_mut(instr.p1)? { state.set_current(None); } + // Only when codegen marked this delete as the statement's row + // change (#692). A DELETE's own `Delete` carries the flag; the + // one an UPDATE emits before re-inserting the row does not, + // because that pair is one changed row, not two. + if instr.p5 & OPFLAG_NCHANGE != 0 { + vm.record_change(); + } Ok(Step::Next) } CursorSlot::Ephemeral(_) => { @@ -2048,6 +2055,14 @@ pub fn insert(vm: &mut Vm, instr: &Instruction) -> Result { reason: e.to_string(), } })?; + drop(pager); + // See `Delete` above: the flag is codegen's call. Note the + // `EphemeralTable` arm never reaches here, so the two-pass + // UPDATE plan's rowid-stashing `Insert` cannot count even if + // a future caller flagged it by mistake. + if instr.p5 & OPFLAG_NCHANGE != 0 { + vm.record_change(); + } Ok(Step::Next) } } diff --git a/src/vdbe/exec.rs b/src/vdbe/exec.rs index 8cef6e72..bc1bb229 100644 --- a/src/vdbe/exec.rs +++ b/src/vdbe/exec.rs @@ -320,6 +320,14 @@ pub struct Vm { /// `Halt` handling for the "BEGIN with no matching COMMIT/ROLLBACK" /// safety fallback. pub(crate) autocommit: bool, + /// Rows this program has changed (013/Req 1, #692) — incremented by + /// `Insert`/`Delete` only when their `P5` carries + /// [`OPFLAG_NCHANGE`], which is codegen's decision rather than the + /// opcode's. Counts table rows: index maintenance (`IdxInsert`, + /// `IdxDelete`, `AutoIndexInsert`) and ephemeral-cursor writes never + /// set the flag, so a table with three indexes reports the same + /// number as the same table with none. + changes: u64, /// Reused byte buffer for `MakeRecord` (#454): amortizes the record /// payload's allocation across every row a statement emits, instead /// of a fresh `Vec` per `MakeRecord` execution. @@ -347,6 +355,7 @@ impl Default for Vm { once_fired: HashSet::new(), params: Vec::new(), autocommit: true, + changes: 0, record_scratch: Vec::new(), make_record_values_scratch: Vec::new(), encode_scratch: Vec::new(), @@ -370,6 +379,27 @@ impl Vm { Self::default() } + /// Counts one changed table row (013/Req 1, #692). Called by + /// `Insert`/`Delete` only when the instruction's `P5` carries + /// [`OPFLAG_NCHANGE`] — see [`Vm::changes`]'s field doc for why the + /// handler cannot decide this for itself. + pub(crate) fn record_change(&mut self) { + self.changes = self.changes.saturating_add(1); + } + + /// Rows changed so far by this program (013/Req 1, #692). + /// + /// This is a per-`Vm` count, and a `Vm` lives for one statement. The + /// cross-statement retention `sqlite3_changes()` specifies — a + /// statement that changes nothing does not clobber the previous + /// count — is the connection's rule, not the engine's, and belongs to + /// spec 013/Req 1's `Connection::changes`. What the engine promises + /// is that this number is right for the statement just run; see + /// [`StepOutcome::changes`] for how the two fit together. + pub fn changes(&self) -> u64 { + self.changes + } + /// Reused scratch buffer for `MakeRecord` (#454) — see /// [`Vm::record_scratch`]'s field doc. pub(crate) fn record_scratch(&mut self) -> &mut Vec { @@ -1009,7 +1039,7 @@ const MAX_STEPS: u32 = 50_000_000; /// Runs `program` to completion on a fresh, database-less [`Vm`] and /// returns the rows it emitted via `ResultRow`. pub fn execute(program: &Program) -> Result>, ExecError> { - run(Vm::new(), program).map(|(rows, _)| rows) + run(Vm::new(), program).map(|(rows, _, _)| rows) } /// Like [`execute`], but binds `params` for `Opcode::Variable` to read @@ -1022,7 +1052,7 @@ pub fn execute_with_params( ) -> Result>, ExecError> { let mut vm = Vm::new(); vm.bind_params(params); - run(vm, program).map(|(rows, _)| rows) + run(vm, program).map(|(rows, _, _)| rows) } /// Like [`execute`], but the `Vm` can service `OpenRead` (cursor @@ -1033,7 +1063,7 @@ pub fn execute_with_db( source: Rc, header: DatabaseHeader, ) -> Result>, ExecError> { - run(Vm::with_db(source, header), program).map(|(rows, _)| rows) + run(Vm::with_db(source, header), program).map(|(rows, _, _)| rows) } /// Like [`execute_with_db`], but the `Vm` can also service the write @@ -1044,7 +1074,7 @@ pub fn execute_with_writable_db( pager: crate::pager::Pager, header: DatabaseHeader, ) -> Result>, ExecError> { - run(Vm::with_writable_db(pager, header), program).map(|(rows, _)| rows) + run(Vm::with_writable_db(pager, header), program).map(|(rows, _, _)| rows) } /// Combines [`execute_with_db`] and [`execute_with_params`]. @@ -1056,7 +1086,7 @@ pub fn execute_with_db_and_params( ) -> Result>, ExecError> { let mut vm = Vm::with_db(source, header); vm.bind_params(params); - run(vm, program).map(|(rows, _)| rows) + run(vm, program).map(|(rows, _, _)| rows) } /// Runs one statement's `program` against a `pager` shared across @@ -1074,12 +1104,58 @@ pub fn execute_transaction_step( header: DatabaseHeader, autocommit_in: bool, ) -> Result<(Vec>, bool), ExecError> { + execute_transaction_step_counted(program, pager, header, autocommit_in) + .map(|outcome| (outcome.rows, outcome.autocommit)) +} + +/// What one statement produced: its rows, the autocommit flag to thread +/// into the next statement, and how many rows it changed (013/Req 1, +/// #692). +pub struct StepOutcome { + /// The statement's result rows, in order. + pub rows: Vec>, + /// Autocommit state after this statement — pass it as the next + /// call's `autocommit_in`, exactly as [`execute_transaction_step`]'s + /// second tuple element. + pub autocommit: bool, + /// Rows changed, or `None` when this statement does not have a + /// rows-changed count at all. + /// + /// `Some(0)` and `None` are different answers and the difference is + /// the whole point. `Some(0)` is "this was an `INSERT`/`UPDATE`/ + /// `DELETE` and it changed nothing" — a lost optimistic-concurrency + /// race, which is what a consumer needs to detect. `None` is "this + /// was not that kind of statement", so a connection tracking + /// `sqlite3_changes()` should leave its stored count untouched rather + /// than zeroing it. That retention rule is the connection's to + /// implement (spec 013/Req 1's `Connection::changes`); this type just + /// makes it a one-liner. + pub changes: Option, +} + +/// [`execute_transaction_step`] plus the rows-changed count (013/Req 1, +/// #692). +/// +/// The two share one loop rather than running in parallel: +/// `execute_transaction_step` is a wrapper that drops the count, so the +/// counted and uncounted paths cannot drift. +pub fn execute_transaction_step_counted( + program: &Program, + pager: Rc>, + header: DatabaseHeader, + autocommit_in: bool, +) -> Result { let mut vm = Vm::with_shared_writable_db(pager, header); vm.autocommit = autocommit_in; - run(vm, program) + let (rows, autocommit, changed) = run(vm, program)?; + Ok(StepOutcome { + rows, + autocommit, + changes: program.counts_changes().then_some(changed), + }) } -fn run(mut vm: Vm, program: &Program) -> Result<(Vec>, bool), ExecError> { +fn run(mut vm: Vm, program: &Program) -> Result<(Vec>, bool, u64), ExecError> { // #509: `steps`/`pc` are both backstops against pathological programs // (a step-limit runaway, a jump target past the end), not values any // real program comes close to overflowing (`MAX_STEPS` is 50_000_000, @@ -1130,7 +1206,7 @@ fn run(mut vm: Vm, program: &Program) -> Result<(Vec>, bool), ExecErr } } } - return Ok((vm.rows, vm.autocommit)); + return Ok((vm.rows, vm.autocommit, vm.changes)); } Step::Halt { code, message } => return Err(ExecError::Halted { code, message }), } diff --git a/src/vdbe/program.rs b/src/vdbe/program.rs index 0763430a..7ecf752f 100644 --- a/src/vdbe/program.rs +++ b/src/vdbe/program.rs @@ -835,6 +835,19 @@ pub struct Instruction { pub p5: u16, } +/// `P5` bit marking an `Insert`/`Delete` as *the* row change a statement +/// should report through the rows-changed counter (013/Req 1, #692). +/// +/// Same value and same job as stock SQLite's `OPFLAG_NCHANGE`. The flag +/// exists because the opcode alone cannot tell you: one `UPDATE`ed row +/// emits a `Delete` *and* an `Insert` (`codegen/stmt/update.rs`), and the +/// two-pass range-seek plan additionally emits an `Insert` against an +/// ephemeral cursor to stash matched rowids, so counting every +/// `Insert`/`Delete` as it executes reports 3 for a plan that changed 1 +/// row and 2 for the other plan of the same statement. Codegen knows +/// which mutation is the row change; the handler does not. +pub const OPFLAG_NCHANGE: u16 = 0x01; + impl Instruction { /// Builds an instruction with `P4` absent and `P5` zero — the common /// case for control/arithmetic/compare opcodes that only use @@ -861,6 +874,20 @@ impl Instruction { p5: 0, } } + + /// Builds an instruction with an explicit `P5` flags operand — the + /// `with_p4` constructor's counterpart for the flags word. Used for + /// [`OPFLAG_NCHANGE`]; `new`/`with_p4` both leave `p5` zero. + pub fn with_p5(opcode: Opcode, p1: i32, p2: i32, p3: i32, p5: u16) -> Self { + Self { + opcode, + p1, + p2, + p3, + p4: P4::None, + p5, + } + } } /// A linear, zero-indexed sequence of instructions. Execution starts at @@ -878,6 +905,26 @@ impl Program { Self { instructions } } + /// Whether this program is a statement whose rows-changed count is + /// meaningful (013/Req 1, #692) — i.e. whether codegen flagged any + /// mutation with [`OPFLAG_NCHANGE`]. + /// + /// Deliberately *static*: it asks what the program contains, not what + /// it executed. An `UPDATE` whose `WHERE` matches no row never runs + /// its flagged `Insert`, but must still report a count of zero rather + /// than "not a counting statement" — which is the distinction + /// `sqlite3_changes()` needs in order to leave the previous count + /// alone after a `SELECT`. + /// + /// Derived rather than stored so it cannot disagree with the + /// instructions it describes; `Program` is a `Vec` and + /// programs are a handful of instructions per statement. + pub fn counts_changes(&self) -> bool { + self.instructions.iter().any(|i| { + i.p5 & OPFLAG_NCHANGE != 0 && matches!(i.opcode, Opcode::Insert | Opcode::Delete) + }) + } + /// Returns the instruction at `pc`, or `None` if `pc` is out of /// range. pub fn get(&self, pc: usize) -> Option<&Instruction> { diff --git a/tests/corpus/changes_oracle_test.rs b/tests/corpus/changes_oracle_test.rs new file mode 100644 index 00000000..ab420360 --- /dev/null +++ b/tests/corpus/changes_oracle_test.rs @@ -0,0 +1,197 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! Oracle diff for the rows-changed counter (spec 013 Requirement 1, +//! #692): runs one statement sequence through this crate's write path and +//! the same sequence through the pinned `sqlite3`, and compares the count +//! each `INSERT`/`UPDATE`/`DELETE` reports. +//! +//! The unit suite (`tests/unit/vdbe_changes_test.rs`) pins the mechanism — +//! that `OPFLAG_NCHANGE` is what counts, so an `UPDATE`'s `Delete` + +//! `Insert` pair is one change and index maintenance is none. This pins +//! the *answers* against the definition of correctness, which is SQLite. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing +)] + +use std::cell::RefCell; +use std::path::Path; +use std::process::Command; +use std::rc::Rc; + +use sqlite_rs::btree::TableCursor; +use sqlite_rs::codegen::compile_statement; +use sqlite_rs::header::DatabaseHeader; +use sqlite_rs::pager::Pager; +use sqlite_rs::schema::{read_schema, read_views}; +use sqlite_rs::vdbe::execute_transaction_step_counted; +use sqlite_rs::vfs::MemoryVfs; + +use crate::oracle::{pinned_oracle, skip_no_oracle}; + +/// The sequence both engines run. Every statement is one this crate +/// compiles today, and the `UPDATE`/`DELETE` predicates are chosen to +/// cover a match, a partial match and a miss. +const STATEMENTS: &[&str] = &[ + "CREATE TABLE t(a INTEGER, b TEXT, c TEXT)", + "CREATE INDEX t_a ON t(a)", + "CREATE UNIQUE INDEX t_c ON t(c)", + "INSERT INTO t VALUES (1, 'b1', 'c1')", + "INSERT INTO t VALUES (2, 'b2', 'c2')", + "INSERT INTO t VALUES (3, 'b3', 'c3')", + "INSERT INTO t VALUES (4, 'b4', 'c4')", + // Touches the scanned index -> two-pass plan (#675). + "UPDATE t SET a = a + 10 WHERE a > 2", + // Leaves it alone -> single-pass plan. + "UPDATE t SET b = 'z' WHERE a > 2", + // Matches nothing. + "UPDATE t SET b = 'q' WHERE a = 999", + "DELETE FROM t WHERE a < 3", + "DELETE FROM t", + "DELETE FROM t", +]; + +fn is_dml(sql: &str) -> bool { + let head = sql.trim_start(); + ["INSERT", "UPDATE", "DELETE"] + .iter() + .any(|kw| head.len() >= kw.len() && head[..kw.len()].eq_ignore_ascii_case(kw)) +} + +fn empty_db(page_size: u32) -> (MemoryVfs, DatabaseHeader) { + let mut page1 = vec![0u8; page_size as usize]; + page1[0..16].copy_from_slice(b"SQLite format 3\0"); + page1[16..18].copy_from_slice(&u16::try_from(page_size).unwrap().to_be_bytes()); + page1[18] = 1; + page1[19] = 1; + page1[28..32].copy_from_slice(&1u32.to_be_bytes()); + page1[56..60].copy_from_slice(&1u32.to_be_bytes()); + page1[100] = 0x0D; + page1[105..107].copy_from_slice(&u16::try_from(page_size).unwrap().to_be_bytes()); + + let mut header_bytes = [0u8; 100]; + header_bytes.copy_from_slice(&page1[..100]); + let header = DatabaseHeader::parse(&header_bytes).unwrap(); + + let mut vfs = MemoryVfs::new(); + vfs.insert("/test.db", page1); + (vfs, header) +} + +/// Runs `STATEMENTS` through this crate, returning the count each DML +/// statement reported. +fn ours() -> Vec<(&'static str, Option)> { + let page_size = 4096; + let (vfs, header) = empty_db(page_size); + let pager = Rc::new(RefCell::new( + Pager::open(&vfs, Path::new("/test.db"), page_size).unwrap(), + )); + let mut autocommit = true; + let mut out = Vec::new(); + + for sql in STATEMENTS { + let (schemas, views) = { + let borrowed = pager.borrow(); + let mut schema_cursor = TableCursor::new(&*borrowed, &header, 1); + let schemas = read_schema(&mut schema_cursor, header.text_encoding).unwrap(); + let mut view_cursor = TableCursor::new(&*borrowed, &header, 1); + let views = read_views(&mut view_cursor, header.text_encoding).unwrap(); + (schemas, views) + }; + let program = compile_statement(sql, &schemas, &views) + .unwrap_or_else(|e| panic!("{sql} did not compile: {e}")); + let outcome = + execute_transaction_step_counted(&program, Rc::clone(&pager), header, autocommit) + .unwrap_or_else(|e| panic!("{sql} failed: {e}")); + autocommit = outcome.autocommit; + out.push((*sql, outcome.changes)); + } + out +} + +/// Runs `STATEMENTS` through the pinned oracle, returning `changes()` +/// after each DML statement. +/// +/// One `sqlite3` invocation per statement, with `SELECT changes()` +/// appended: `changes()` is per-connection state, and a fresh invocation +/// starts it at zero, so asking inside the same invocation as the +/// statement is what reports that statement's own count. +fn oracle(bin: &Path, db: &Path) -> Vec<(&'static str, Option)> { + let mut out = Vec::new(); + for sql in STATEMENTS { + let script = if is_dml(sql) { + format!("{sql};\nSELECT changes();") + } else { + format!("{sql};") + }; + let output = Command::new(bin) + .arg(db) + .arg(&script) + .output() + .unwrap_or_else(|e| panic!("oracle failed to run {sql}: {e}")); + assert!( + output.status.success(), + "oracle rejected {sql}: {}", + String::from_utf8_lossy(&output.stderr) + ); + let changed = if is_dml(sql) { + Some( + String::from_utf8_lossy(&output.stdout) + .trim() + .parse::() + .unwrap_or_else(|e| { + panic!( + "oracle's changes() after {sql} was not a number ({e}): {:?}", + String::from_utf8_lossy(&output.stdout) + ) + }), + ) + } else { + None + }; + out.push((*sql, changed)); + } + out +} + +#[test] +fn rows_changed_counts_match_the_oracle() { + let Some(bin) = pinned_oracle() else { + skip_no_oracle("rows_changed_counts_match_the_oracle"); + return; + }; + let dir = tempdir(); + let db = dir.join("changes.db"); + + let mine = ours(); + let theirs = oracle(&bin, &db); + + // Compare only the DML statements: `changes()` is undefined-by-design + // for a DDL statement (the oracle reports whatever the connection's + // previous count was; we report `None`), so the interesting claim is + // the counting statements agreeing exactly. + let mine_dml: Vec<_> = mine.iter().filter(|(sql, _)| is_dml(sql)).collect(); + let theirs_dml: Vec<_> = theirs.iter().filter(|(sql, _)| is_dml(sql)).collect(); + + assert_eq!(mine_dml, theirs_dml, "rows-changed counts diverge"); + + // And the DDL half really is `None` on our side rather than an + // accidental `Some(0)`, which is the distinction a connection needs + // in order not to clobber a retained count. + for (sql, changed) in &mine { + if !is_dml(sql) { + assert_eq!(*changed, None, "{sql} claimed a rows-changed count"); + } + } + + std::fs::remove_dir_all(&dir).ok(); +} + +fn tempdir() -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("sqlite-rs-changes-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + dir +} diff --git a/tests/corpus/main.rs b/tests/corpus/main.rs index 96ac7874..10dbad28 100644 --- a/tests/corpus/main.rs +++ b/tests/corpus/main.rs @@ -13,6 +13,7 @@ clippy::arithmetic_side_effects )] +mod changes_oracle_test; mod harness; mod oracle; diff --git a/tests/unit/vdbe_changes_test.rs b/tests/unit/vdbe_changes_test.rs new file mode 100644 index 00000000..762b52b3 --- /dev/null +++ b/tests/unit/vdbe_changes_test.rs @@ -0,0 +1,299 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! Rows-changed counter (spec 013 Requirement 1, #692). +//! +//! Spec 013 calls this the one item on its list a consumer cannot work +//! around: without it a caller cannot tell an `UPDATE` that matched from +//! one that did not, which is the distinction every +//! optimistic-concurrency scheme is built on. +//! +//! The counter is driven by `OPFLAG_NCHANGE` on `P5` rather than by the +//! opcode, and these tests are the reason. One `UPDATE`ed row emits a +//! `Delete` *and* an `Insert`, and the two-pass range-seek plan +//! (#666/#675) emits a third `Insert` against an ephemeral cursor to +//! stash matched rowids — so a handler that counted every +//! `Insert`/`Delete` would report 3 for a plan that changed 1 row, and 2 +//! for the other plan of the same statement. +//! `update_of_one_row_reports_one_under_both_plans` pins exactly that. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing +)] + +use std::cell::RefCell; +use std::path::Path; +use std::rc::Rc; + +use sqlite_rs::btree::TableCursor; +use sqlite_rs::codegen::{compile_select_with_catalog, compile_statement}; +use sqlite_rs::header::DatabaseHeader; +use sqlite_rs::pager::Pager; +use sqlite_rs::parser::error::ParseOutcome; +use sqlite_rs::parser::parse_select; +use sqlite_rs::record::Value; +use sqlite_rs::schema::{read_schema, read_views, TableSchema, ViewSchema}; +use sqlite_rs::vdbe::{execute_transaction_step_counted, Program, StepOutcome}; +use sqlite_rs::vfs::MemoryVfs; + +fn empty_db(page_size: u32) -> (MemoryVfs, DatabaseHeader) { + let mut page1 = vec![0u8; page_size as usize]; + page1[0..16].copy_from_slice(b"SQLite format 3\0"); + page1[16..18].copy_from_slice(&u16::try_from(page_size).unwrap().to_be_bytes()); + page1[18] = 1; + page1[19] = 1; + page1[28..32].copy_from_slice(&1u32.to_be_bytes()); + page1[56..60].copy_from_slice(&1u32.to_be_bytes()); + page1[100] = 0x0D; + page1[105..107].copy_from_slice(&u16::try_from(page_size).unwrap().to_be_bytes()); + + let mut header_bytes = [0u8; 100]; + header_bytes.copy_from_slice(&page1[..100]); + let header = DatabaseHeader::parse(&header_bytes).unwrap(); + + let mut vfs = MemoryVfs::new(); + vfs.insert("/test.db", page1); + (vfs, header) +} + +struct Db { + pager: Rc>, + header: DatabaseHeader, + autocommit: bool, +} + +impl Db { + fn new() -> Self { + let page_size = 4096; + let (vfs, header) = empty_db(page_size); + let pager = Pager::open(&vfs, Path::new("/test.db"), page_size).unwrap(); + Self { + pager: Rc::new(RefCell::new(pager)), + header, + autocommit: true, + } + } + + fn catalog(&self) -> (Vec, Vec) { + let borrowed = self.pager.borrow(); + let mut schema_cursor = TableCursor::new(&*borrowed, &self.header, 1); + let schemas = read_schema(&mut schema_cursor, self.header.text_encoding).unwrap(); + let mut view_cursor = TableCursor::new(&*borrowed, &self.header, 1); + let views = read_views(&mut view_cursor, self.header.text_encoding).unwrap(); + (schemas, views) + } + + fn step(&mut self, sql: &str) -> StepOutcome { + let (schemas, views) = self.catalog(); + let program = compile_statement(sql, &schemas, &views).unwrap(); + let outcome = execute_transaction_step_counted( + &program, + Rc::clone(&self.pager), + self.header, + self.autocommit, + ) + .unwrap(); + self.autocommit = outcome.autocommit; + outcome + } + + /// Runs a statement that has no rows-changed count (DDL), asserting + /// that it does not claim one. + fn exec_ddl(&mut self, sql: &str) { + assert_eq!( + self.step(sql).changes, + None, + "{sql} should not be a counting statement" + ); + } + + /// The rows-changed count for `sql`, asserting it is a counting + /// statement at all. + fn changes(&mut self, sql: &str) -> u64 { + self.step(sql) + .changes + .unwrap_or_else(|| panic!("{sql} reported no rows-changed count")) + } + + /// Compiles a write/DDL statement without running it. + fn compile(&mut self, sql: &str) -> Program { + let (schemas, views) = self.catalog(); + compile_statement(sql, &schemas, &views).unwrap() + } + + /// Compiles a `SELECT`, which `compile_statement` does not handle. + fn compile_select(&mut self, sql: &str) -> Program { + let (schemas, _) = self.catalog(); + let select = match parse_select(sql) { + ParseOutcome::Accepted(s) => s, + other => panic!("{sql} did not parse: {other:?}"), + }; + let schema = schemas + .iter() + .find(|s| s.name.eq_ignore_ascii_case("t")) + .unwrap(); + compile_select_with_catalog(&select, schema, &schemas).unwrap() + } + + fn count_rows(&mut self, sql: &str) -> i64 { + let program = self.compile_select(sql); + let outcome = execute_transaction_step_counted( + &program, + Rc::clone(&self.pager), + self.header, + self.autocommit, + ) + .unwrap(); + self.autocommit = outcome.autocommit; + match &outcome.rows[0][0] { + Value::Integer(n) => *n, + other => panic!("expected an integer, got {other:?}"), + } + } +} + +/// Spec 013/Req 1's first scenario: the same conditional `UPDATE` run +/// twice reports one row changed, then zero. +#[test] +fn conditional_update_reports_match() { + let mut db = Db::new(); + db.exec_ddl("CREATE TABLE t(k TEXT, metadata_location TEXT)"); + db.changes("INSERT INTO t VALUES ('a', 'a')"); + + let first = db.changes("UPDATE t SET metadata_location = 'b' WHERE metadata_location = 'a'"); + let second = db.changes("UPDATE t SET metadata_location = 'b' WHERE metadata_location = 'a'"); + + assert_eq!(first, 1, "the update that matched"); + assert_eq!(second, 0, "the same update, now losing the race"); +} + +/// The regression guard for `OPFLAG_NCHANGE`'s existence. Both `UPDATE` +/// plans change exactly one row and must both say so, even though they +/// emit different numbers of `Insert`/`Delete` opcodes to do it. +/// +/// Plan selection is #675's rule: the two-pass plan is taken when the +/// `SET` clause touches the index the range predicate scans, the +/// single-pass plan when it does not. +#[test] +fn update_of_one_row_reports_one_under_both_plans() { + // Two-pass: `SET n` touches the scanned index `t_n`. + let mut two_pass = Db::new(); + two_pass.exec_ddl("CREATE TABLE t(n INTEGER, v TEXT)"); + two_pass.exec_ddl("CREATE INDEX t_n ON t(n)"); + for i in 1..=5 { + two_pass.changes(&format!("INSERT INTO t VALUES ({i}, 'v{i}')")); + } + let counted = two_pass.changes("UPDATE t SET n = n + 100 WHERE n > 4"); + + // Single-pass: `SET v` leaves the scanned index alone. + let mut single_pass = Db::new(); + single_pass.exec_ddl("CREATE TABLE t(n INTEGER, v TEXT)"); + single_pass.exec_ddl("CREATE INDEX t_n ON t(n)"); + for i in 1..=5 { + single_pass.changes(&format!("INSERT INTO t VALUES ({i}, 'v{i}')")); + } + let counted_single = single_pass.changes("UPDATE t SET v = 'x' WHERE n > 4"); + + assert_eq!(counted, 1, "two-pass plan counted its own scratch writes"); + assert_eq!(counted_single, 1, "single-pass plan"); +} + +/// `Some(0)` and `None` are different answers: an `UPDATE` that matched +/// nothing is a counting statement that counted zero, which is what a +/// lost optimistic-concurrency race looks like. +#[test] +fn update_matching_nothing_reports_some_zero_not_none() { + let mut db = Db::new(); + db.exec_ddl("CREATE TABLE t(n INTEGER)"); + db.changes("INSERT INTO t VALUES (1)"); + + let outcome = db.step("UPDATE t SET n = 2 WHERE n = 999"); + + assert_eq!( + outcome.changes, + Some(0), + "an UPDATE whose WHERE matches nothing still has a count" + ); +} + +/// A `SELECT` has no rows-changed count, so a connection tracking +/// `sqlite3_changes()` leaves its stored value alone rather than zeroing +/// it (spec 013/Req 1's second scenario). +/// +/// Asserted against `Program::counts_changes` rather than through +/// `execute_transaction_step_counted`, because `compile_statement` +/// handles write and DDL statements only — a `SELECT` reaches the engine +/// by a different route entirely (`compile_select*` + `execute_with_db`), +/// which has no count to clobber in the first place. The static +/// discriminator is the thing a future facade will consult, so it is the +/// thing worth pinning. +#[test] +fn select_is_not_a_counting_statement() { + let mut db = Db::new(); + db.exec_ddl("CREATE TABLE t(n INTEGER)"); + db.changes("INSERT INTO t VALUES (1)"); + + for sql in ["SELECT n FROM t WHERE n = 999", "SELECT count(*) FROM t"] { + let program = db.compile_select(sql); + assert!( + !program.counts_changes(), + "{sql} claimed a rows-changed count" + ); + } + + // And a statement that does have one still says so, so the assertion + // above is not passing for want of any flagged program at all. + let insert = db.compile("INSERT INTO t VALUES (2)"); + assert!(insert.counts_changes()); +} + +#[test] +fn insert_and_delete_count_their_rows() { + let mut db = Db::new(); + db.exec_ddl("CREATE TABLE t(n INTEGER)"); + for i in 0..7 { + assert_eq!(db.changes(&format!("INSERT INTO t VALUES ({i})")), 1); + } + assert_eq!(db.count_rows("SELECT count(*) FROM t"), 7); + + assert_eq!(db.changes("DELETE FROM t WHERE n < 3"), 3, "partial delete"); + assert_eq!(db.changes("DELETE FROM t"), 4, "the rest"); + assert_eq!(db.changes("DELETE FROM t"), 0, "already empty"); +} + +/// Index maintenance is a row-adjacent write, not a row change: the same +/// statements against a table with three indexes must report the same +/// numbers as against a table with none. +#[test] +fn index_maintenance_does_not_count() { + let counts = |indexed: bool| { + let mut db = Db::new(); + db.exec_ddl("CREATE TABLE t(a INTEGER, b TEXT, c TEXT)"); + if indexed { + db.exec_ddl("CREATE INDEX t_a ON t(a)"); + db.exec_ddl("CREATE INDEX t_b ON t(b)"); + db.exec_ddl("CREATE UNIQUE INDEX t_c ON t(c)"); + } + let mut seen = Vec::new(); + for i in 0..4 { + seen.push(db.changes(&format!("INSERT INTO t VALUES ({i}, 'b{i}', 'c{i}')"))); + } + seen.push(db.changes("UPDATE t SET b = 'z' WHERE a >= 2")); + seen.push(db.changes("DELETE FROM t WHERE a < 2")); + seen + }; + + assert_eq!(counts(true), counts(false)); + assert_eq!(counts(false), vec![1, 1, 1, 1, 2, 2]); +} + +/// DDL is not a counting statement either, even though `CREATE TABLE` +/// writes a `sqlite_master` row. +#[test] +fn ddl_has_no_count() { + let mut db = Db::new(); + assert_eq!(db.step("CREATE TABLE t(n INTEGER)").changes, None); + assert_eq!(db.step("CREATE INDEX t_n ON t(n)").changes, None); +} From 2f0a517a4a1a56e17130f09f92ba235644573edb Mon Sep 17 00:00:00 2001 From: Diederik Siderius Date: Fri, 4 Sep 2026 14:40:56 +0200 Subject: [PATCH 02/14] refactor: lift the SELECT compile pipeline out of the CLI into the library (#695) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Connection::prepare(sql)` cannot be written today. `compile_statement` handles write and DDL statements only — hand it a `SELECT` and it answers `Unrecognized("SELECT")`, which I hit for real while writing #692's tests. A `SELECT` needs its CTEs and views expanded, its FROM tables resolved and `sqlite_stat1` stats loaded first, and the pipeline that does all of that lived in `src/bin/sqlite-rs/query.rs` — inside the executable, where no library consumer can reach it. That split is a CLI implementation detail, not something a consumer should have to know about. Stock SQLite has exactly one `sqlite3_prepare_v2()`: hand it any statement, get a handle back. Spec 013's `Connection::prepare` has to behave the same way, so the missing half moves into `src/codegen/prepare.rs` where both callers can use it. Not a rewrite. `compile_select_program` and `SelectOutcome` are the CLI's own code relocated, and the 127-line local copy is deleted rather than left to drift — `query.rs` and `repl.rs` now call the library's. `derive_headers` comes along as `result_column_names`, so `Row`'s by-name access and the CLI's column headers will agree by construction instead of by coincidence. One thing changed rather than moved: every error was `.map_err(|e| e.to_string())`, flattening a structured `CodegenError` into a message. That is fine when the next step is printing to a terminal and wrong for a library, where a caller needs to match on what went wrong. Errors are now `PrepareError`, which wraps `CodegenError` and adds the one case that is about the request rather than the SQL: asking for `EXPLAIN QUERY PLAN` on a FROM-less `SELECT`, which is not a `CodegenError::NoFromClause` because `SELECT 1` compiles fine, it just has no access path to explain. `Display` reproduces the old strings exactly, so CLI output is unchanged. `from_less_schema()` stays private. The CLI inlined that 12-field literal; a public `TableSchema::none()` would be new API surface this lift does not need. The CLI behaving identically is the claim that matters here, since this is a refactor of the path every `sqlite-rs query` and REPL statement takes. Evidence: 1575 unit, 388 corpus (which includes the CLI e2e suite) and 15 sqllogictest all pass unchanged, `make lint` clean both clippy passes, and four queries plus `EXPLAIN QUERY PLAN` hand-checked byte-identical against the pinned 3.53.4 oracle — including the FROM-less EQP error, whose message survives the `String` -> `PrepareError` change intact. No `Connection` or `Statement` yet; this is the groundwork they need. PRAGMA dispatch is deliberately not lifted: spec 013's non-goals give the PRAGMA catalogue to V7, and SQE's statement list contains none. Refs: 013/Req-3, #695, #678 Co-Authored-By: Claude Opus 5 (1M context) --- src/bin/sqlite-rs/query.rs | 137 +---------------------- src/bin/sqlite-rs/repl.rs | 5 +- src/codegen.rs | 2 + src/codegen/prepare.rs | 217 +++++++++++++++++++++++++++++++++++++ 4 files changed, 224 insertions(+), 137 deletions(-) create mode 100644 src/codegen/prepare.rs diff --git a/src/bin/sqlite-rs/query.rs b/src/bin/sqlite-rs/query.rs index a9a4744f..03804298 100644 --- a/src/bin/sqlite-rs/query.rs +++ b/src/bin/sqlite-rs/query.rs @@ -14,150 +14,17 @@ use std::process::ExitCode; use std::rc::Rc; use sqlite_rs::btree::TableCursor; -use sqlite_rs::codegen::{ - compile_select_compound, compile_select_joined, compile_select_with_catalog, - compile_select_with_catalog_and_stats, expand_with_clause, explain_query_plan, - flatten_from_subqueries, push_down_where_predicates, resolve_from_table_schema, resolve_views, - CodegenError, EqpRow, ExpandViews, -}; +use sqlite_rs::codegen::{compile_select_program, SelectOutcome}; use sqlite_rs::dump; use sqlite_rs::format::{write_csv_value, write_query_value}; -use sqlite_rs::parser::ast::Select; use sqlite_rs::parser::{parse_explain, parse_select, ParseOutcome}; use sqlite_rs::schema::{read_schema, read_views, TableSchema, ViewSchema}; -use sqlite_rs::vdbe::{execute_with_db, explain, Program}; +use sqlite_rs::vdbe::{execute_with_db, explain}; use sqlite_rs::vfs::{PageSource, UnixVfs}; use crate::common::{fatal, CSV_ROW_TERMINATOR}; use crate::pragma_query::{execute_pragma_query, parse_pragma_query}; -/// What [`compile_select_program`] produced: either `EXPLAIN QUERY -/// PLAN`'s rows (nothing further to compile — there's no bytecode to -/// run or `-explain`) or an ordinary compiled `Program`. -pub(crate) enum SelectOutcome { - Eqp(Vec), - Program(Program), -} - -/// Parses (already done by the caller — `select`/`eqp_mode` come from -/// `parse_select`/`parse_explain`), resolves every table `select` -/// touches against `schemas`, and compiles it: FROM-less (#260), -/// single-table, joined (#237), or `UNION ALL` compound (#240), -/// whichever `select`'s shape calls for. Shared by `run_query` (a -/// fresh read-only `Pager` per invocation) and the REPL (#365, one -/// shared read/write `Pager` per session) — both need exactly this -/// parse-resolve-compile pipeline, just against a different -/// `PageSource`. -pub(crate) fn compile_select_program( - select: &Select, - eqp_mode: bool, - schemas: &[TableSchema], - views: &[ViewSchema], - stats_by_table: &std::collections::HashMap, -) -> Result { - // #376: a `WITH` clause is rewritten away before any table - // resolution happens — every CTE reference in `FROM`/`JOIN` becomes - // a `TableRefKind::Subquery` wrapping that CTE's own query, so the - // rest of this pipeline (and #257's subquery-in-FROM codegen) needs - // no CTE-specific handling at all. - let cte_expanded = expand_with_clause(select); - // #380: every catalog-view reference in `FROM`/`JOIN` is rewritten - // away next, the same shape as the CTE rewrite above — into a - // `TableRefKind::Subquery` wrapping the view's own stored query, - // reusing #257's subquery-in-FROM codegen unchanged. Runs *after* - // the CTE rewrite (rather than before) so it also reaches into any - // `TableRefKind::Subquery` the CTE rewrite just produced — a CTE - // whose own body references a view is resolved this way, without - // `expand_views` needing any CTE-specific handling of its own; a - // CTE also shadows a same-named view for the scope of its declaring - // `SELECT`, matching how it already shadows a same-named real table. - let resolved_views = resolve_views(views); - let expanded = cte_expanded - .expand_views(&resolved_views) - .map_err(|e| e.to_string())?; - // `flatten_from_subqueries`/`push_down_where_predicates` below always - // need `&mut Select`, so this is where the deferred clone (if any — - // Cow was `Borrowed` for the common no-CTE/no-view case) finally - // happens, at most once total rather than once per expansion pass. - let mut expanded = expanded.into_owned(); - // #566: flatten a simple FROM-subquery/view/CTE directly into the - // enclosing query first — eliminating it outright makes any base- - // table index it hides visible to the planner, which a mere - // predicate push-down (below) can't do. - flatten_from_subqueries(&mut expanded); - // #532: push safely-movable outer WHERE conjuncts into whatever - // views/derived-tables flattening didn't eliminate, so their own - // materialization scan (below) can filter before scanning. - push_down_where_predicates(&mut expanded); - let select = &expanded; - - let resolve_table = |table_ref: &sqlite_rs::parser::ast::TableRef| { - resolve_from_table_schema(table_ref, schemas) - }; - - let Some(from) = &select.from else { - if eqp_mode { - return Err("EXPLAIN QUERY PLAN requires a FROM clause".to_string()); - } - let no_table = TableSchema { - unresolved_autoindex: false, - name: String::new(), - root_page: 0, - columns: vec![], - column_types: vec![], - column_collations: vec![], - without_rowid: false, - strict: false, - is_virtual: false, - sql: String::new(), - indexes: vec![], - rowid_alias: None, - }; - let program = - compile_select_with_catalog(select, &no_table, &[]).map_err(|e| e.to_string())?; - return Ok(SelectOutcome::Program(program)); - }; - - let schema = resolve_table(&from.first).map_err(|e| e.to_string())?; - - if eqp_mode { - let mut joined_schemas = vec![schema]; - for join in &from.joins { - joined_schemas.push(resolve_table(&join.table).map_err(|e| e.to_string())?); - } - let rows = explain_query_plan(select, &joined_schemas, stats_by_table, schemas) - .map_err(|e| e.to_string())?; - return Ok(SelectOutcome::Eqp(rows)); - } - - let program = if !select.compound.is_empty() { - let mut arm_schemas = Vec::with_capacity(select.compound.len()); - for arm in &select.compound { - let Some(arm_from) = &arm.from else { - return Err(CodegenError::NoFromClause.to_string()); - }; - arm_schemas.push(resolve_table(&arm_from.first).map_err(|e| e.to_string())?); - } - compile_select_compound(select, &schema, &arm_schemas, schemas) - .map_err(|e| e.to_string())? - } else if from.joins.is_empty() { - let stats = stats_by_table - .get(&schema.name) - .cloned() - .unwrap_or_default(); - compile_select_with_catalog_and_stats(select, &schema, schemas, &stats) - .map_err(|e| e.to_string())? - } else { - let mut joined_schemas = vec![schema]; - for join in &from.joins { - joined_schemas.push(resolve_table(&join.table).map_err(|e| e.to_string())?); - } - compile_select_joined(select, &joined_schemas, schemas, stats_by_table) - .map_err(|e| e.to_string())? - }; - Ok(SelectOutcome::Program(program)) -} - pub fn run_query(raw_args: Vec) -> ExitCode { let mut csv = false; let mut explain_flag = false; diff --git a/src/bin/sqlite-rs/repl.rs b/src/bin/sqlite-rs/repl.rs index b6fdcd7d..b2df7090 100644 --- a/src/bin/sqlite-rs/repl.rs +++ b/src/bin/sqlite-rs/repl.rs @@ -47,7 +47,8 @@ use std::rc::Rc; use sqlite_rs::btree::TableCursor; use sqlite_rs::codegen::{ - compile_statement, leading_keywords, output_column_names, resolve_from_table_schema, + compile_select_program, compile_statement, leading_keywords, output_column_names, + resolve_from_table_schema, SelectOutcome, }; use sqlite_rs::dump; use sqlite_rs::parser::{ends_with_semicolon, parse_select, split_statements, ParseOutcome}; @@ -60,7 +61,7 @@ use crate::dot_commands::{ }; use crate::mode::{print_rows, OutputMode}; use crate::pragma_query::{execute_pragma_query, parse_pragma_query}; -use crate::query::{compile_select_program, write_list_row, SelectOutcome}; +use crate::query::write_list_row; use crate::readline::{history_path, ReadlineError}; use crate::tables::{list_table_and_view_names, print_table_names}; diff --git a/src/codegen.rs b/src/codegen.rs index 177185c8..19f6040f 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -12,6 +12,7 @@ pub mod dispatch; pub mod expr; pub(crate) mod index_maintenance; pub mod pragma; +pub mod prepare; pub mod select; pub mod stmt; pub(crate) mod subquery; @@ -24,6 +25,7 @@ pub use ddl::{ }; pub use dispatch::{compile_statement, leading_keywords, DispatchError}; pub use pragma::compile_pragma; +pub use prepare::{compile_select_program, result_column_names, PrepareError, SelectOutcome}; pub use select::{ compile_select, compile_select_compound, compile_select_joined, compile_select_with_catalog, compile_select_with_catalog_and_stats, explain_query_plan, output_column_names, CodegenError, diff --git a/src/codegen/prepare.rs b/src/codegen/prepare.rs new file mode 100644 index 00000000..ad6b3407 --- /dev/null +++ b/src/codegen/prepare.rs @@ -0,0 +1,217 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! Compiling an arbitrary SQL string, whatever kind of statement it is. +//! +//! [`compile_statement`](crate::codegen::compile_statement) covers write +//! and DDL statements only — hand it a `SELECT` and it answers +//! `DispatchError::Unrecognized("SELECT")`. A `SELECT` needs its `FROM` +//! tables resolved, its CTEs and views expanded, and `sqlite_stat1` +//! statistics loaded before it can be compiled at all, and the pipeline +//! that does that lived in `src/bin/sqlite-rs/query.rs` — inside the +//! executable, where no library consumer could reach it. +//! +//! That split is a CLI implementation detail, not something a consumer +//! should have to know. Stock SQLite has exactly one +//! `sqlite3_prepare_v2()`: you hand it any statement and get a handle +//! back. Spec 013's `Connection::prepare` has to behave the same way, so +//! the `SELECT` half moves here where both the CLI and the embedding API +//! can call it (013/Req 3, #695). +//! +//! Errors are [`CodegenError`]/[`PrepareError`] rather than the `String` +//! the CLI copy produced. Flattening a structured error into a message is +//! fine when the next step is printing it to a terminal; a library +//! consumer needs to match on what went wrong. + +use std::collections::HashMap; +use std::fmt; + +use crate::codegen::{ + compile_select_compound, compile_select_joined, compile_select_with_catalog, + compile_select_with_catalog_and_stats, expand_with_clause, explain_query_plan, + flatten_from_subqueries, output_column_names, push_down_where_predicates, + resolve_from_table_schema, resolve_views, CodegenError, EqpRow, ExpandViews, +}; +use crate::parser::ast::{Select, TableRef}; +use crate::planner::Stats; +use crate::schema::{TableSchema, ViewSchema}; +use crate::vdbe::Program; + +/// What [`compile_select_program`] produced: either `EXPLAIN QUERY PLAN`'s rows +/// (nothing further to compile — there is no bytecode to run) or an +/// ordinary compiled [`Program`]. +pub enum SelectOutcome { + /// `EXPLAIN QUERY PLAN` output rows. + Eqp(Vec), + /// A compiled program, ready to execute. + Program(Program), +} + +/// Why a `SELECT` could not be compiled. +/// +/// Distinct from [`CodegenError`] only for the cases that are about the +/// *request* rather than the SQL: asking for `EXPLAIN QUERY PLAN` on a +/// statement that has no `FROM` clause to plan. +#[derive(Debug, PartialEq, Eq)] +pub enum PrepareError { + /// `EXPLAIN QUERY PLAN` was requested for a FROM-less `SELECT`. + /// + /// Not a `CodegenError::NoFromClause`: a FROM-less `SELECT 1` is + /// perfectly compilable, it just has no access path to explain. + EqpWithoutFrom, + /// The statement itself could not be compiled. + Codegen(CodegenError), +} + +impl fmt::Display for PrepareError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + PrepareError::EqpWithoutFrom => { + write!(f, "EXPLAIN QUERY PLAN requires a FROM clause") + } + PrepareError::Codegen(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for PrepareError {} + +impl From for PrepareError { + fn from(e: CodegenError) -> Self { + PrepareError::Codegen(e) + } +} + +/// Resolves every table a `SELECT` touches and compiles it — FROM-less +/// (#260), single-table, joined (#237), or compound (#240), whichever +/// its shape calls for. +/// +/// `select`/`eqp_mode` come from `parse_select`/`parse_explain`; parsing +/// is the caller's. Lifted verbatim from the CLI's own +/// `compile_select_program` (#695), with `String` errors replaced by +/// [`PrepareError`] — the CLI and the embedding API need exactly this +/// pipeline, just against a different `PageSource`. +pub fn compile_select_program( + select: &Select, + eqp_mode: bool, + schemas: &[TableSchema], + views: &[ViewSchema], + stats_by_table: &HashMap, +) -> Result { + // #376: a `WITH` clause is rewritten away before any table + // resolution happens — every CTE reference in `FROM`/`JOIN` becomes + // a `TableRefKind::Subquery` wrapping that CTE's own query, so the + // rest of this pipeline (and #257's subquery-in-FROM codegen) needs + // no CTE-specific handling at all. + let cte_expanded = expand_with_clause(select); + // #380: every catalog-view reference is rewritten away next, the + // same shape as the CTE rewrite. Runs *after* it so it also reaches + // into any `TableRefKind::Subquery` the CTE rewrite just produced. + let resolved_views = resolve_views(views); + let expanded = cte_expanded.expand_views(&resolved_views)?; + // The passes below need `&mut Select`, so this is where the deferred + // clone (if any — `Cow` was `Borrowed` for the common + // no-CTE/no-view case) finally happens, at most once total. + let mut expanded = expanded.into_owned(); + // #566: flatten a simple FROM-subquery/view/CTE into the enclosing + // query first — eliminating it outright makes any base-table index + // it hides visible to the planner, which a predicate push-down + // cannot do. + flatten_from_subqueries(&mut expanded); + // #532: push safely-movable outer WHERE conjuncts into whatever + // flattening did not eliminate. + push_down_where_predicates(&mut expanded); + let select = &expanded; + + let resolve_table = |table_ref: &TableRef| -> Result { + resolve_from_table_schema(table_ref, schemas) + }; + + let Some(from) = &select.from else { + if eqp_mode { + return Err(PrepareError::EqpWithoutFrom); + } + let program = compile_select_with_catalog(select, &from_less_schema(), &[])?; + return Ok(SelectOutcome::Program(program)); + }; + + let schema = resolve_table(&from.first)?; + + if eqp_mode { + let mut joined_schemas = vec![schema]; + for join in &from.joins { + joined_schemas.push(resolve_table(&join.table)?); + } + let rows = explain_query_plan(select, &joined_schemas, stats_by_table, schemas)?; + return Ok(SelectOutcome::Eqp(rows)); + } + + let program = if !select.compound.is_empty() { + let mut arm_schemas = Vec::with_capacity(select.compound.len()); + for arm in &select.compound { + let Some(arm_from) = &arm.from else { + return Err(CodegenError::NoFromClause.into()); + }; + arm_schemas.push(resolve_table(&arm_from.first)?); + } + compile_select_compound(select, &schema, &arm_schemas, schemas)? + } else if from.joins.is_empty() { + let stats = stats_by_table + .get(&schema.name) + .cloned() + .unwrap_or_default(); + compile_select_with_catalog_and_stats(select, &schema, schemas, &stats)? + } else { + let mut joined_schemas = vec![schema]; + for join in &from.joins { + joined_schemas.push(resolve_table(&join.table)?); + } + compile_select_joined(select, &joined_schemas, schemas, stats_by_table)? + }; + Ok(SelectOutcome::Program(program)) +} + +/// The result column names a `SELECT` produces, for access by name. +/// +/// Falls back to `column1`, `column2`, ... for shapes whose names this +/// cannot derive (joins and compounds), which is what the CLI already +/// printed as headers for them. Lifted from the CLI's `derive_headers` +/// (#695) so `Row::get_by_name` and the CLI agree by construction rather +/// than by coincidence. +pub fn result_column_names(select: &Select, schemas: &[TableSchema]) -> Vec { + let single_table = select.compound.is_empty() + && select + .from + .as_ref() + .is_some_and(|from| from.joins.is_empty()); + if single_table { + if let Some(from) = &select.from { + if let Ok(schema) = resolve_from_table_schema(&from.first, schemas) { + return output_column_names(select, &schema); + } + } + } + let count = select.columns.len().max(1); + (1..=count).map(|i| format!("column{i}")).collect() +} + +/// The placeholder schema a FROM-less `SELECT` compiles against (#260). +/// +/// Private on purpose: the CLI inlined this literal, and a public +/// `TableSchema::none()` would be new API surface this lift does not +/// need. +fn from_less_schema() -> TableSchema { + TableSchema { + name: String::new(), + root_page: 0, + columns: vec![], + column_types: vec![], + column_collations: vec![], + without_rowid: false, + strict: false, + is_virtual: false, + sql: String::new(), + indexes: vec![], + rowid_alias: None, + unresolved_autoindex: false, + } +} From 6b9f3bc6a5a911e999da63a5a03b8ecd6cd7fa38 Mon Sep 17 00:00:00 2001 From: Diederik Siderius Date: Fri, 4 Sep 2026 14:52:05 +0200 Subject: [PATCH 03/14] test: cover the lifted SELECT pipeline, and correct the bench comment it falsified (#695) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lift commit had no tests of its own. Every existing exercise of `compile_select_program` went through the `sqlite-rs` binary, so the newly-public library surface had zero direct coverage and a regression in it would only have surfaced as a CLI failure. Two files fix that. `tests/unit/prepare_test.rs` (10 tests) pins the dispatch and the error type: one case per `compile_select_program` arm (FROM-less, single-table, joined, compound), `eqp_mode` returning rows rather than a program, `PrepareError::EqpWithoutFrom` including its message being byte-identical to the string the CLI printed before the lift, an unknown table arriving as a matchable `PrepareError::Codegen` rather than a message to parse, and `result_column_names`' positional fallback for joins and compounds — asserted rather than left implicit, because `Row::get_by_name` will be built on it and `column1` is a real answer a consumer can receive. `tests/corpus/prepare_oracle_test.rs` pins the answers. Eleven queries compiled through the library and diffed against the pinned 3.53.4 oracle, on a fixture the oracle itself created — the adoption direction that matters. Shapes include SQE's own: `UNION`, `UNION ALL`, `LIMIT 1` existence probes that hit and miss, a join, aggregates. I had checked four queries by hand while doing the lift, which is worth nothing once the terminal closes; a refactor of the path every `sqlite-rs query` takes deserves a check that runs in CI. Mutation-verified that it really talks to the oracle: perturbing integer rendering fails it. `SelectOutcome` gains `derive(Debug)`. It is public and a consumer matches on it, both payloads already derive it, and without it `unwrap`/`expect_err` on a `Result` will not compile — which is a papercut for every caller, not just these tests. Also corrected: `tests/performance/v6.rs`'s comment explaining why that bench duplicates the compile pipeline said the real function "is `pub(crate)` to the binary crate and not callable from an external test/bench crate". The lift made that false. The comment now says so and points at the duplication as removable — deliberately not removed here, since replacing it changes the path this bench measures and that belongs in a change whose bench numbers are the point. One test failure worth recording: `result_column_names_honours_aliases` was written as `SELECT a AS first, b AS second` and failed to parse. Not a bad test — `FIRST` is one of 89 keywords we reserve that SQLite treats as an identifier (`parse.y:272`'s `%fallback ID`). Filed as #696, which also blocks SQE outright: its `iceberg_namespace_properties` has a `key` column, and we cannot create or read that table at all. The test here uses non-keyword aliases and the keyword case is pinned in #696 rather than smuggled in. Verified: 1572 unit (1562 + 10), 388 corpus (387 + 1), 15 sqllogictest, `make lint` clean both passes, `make check-mod-files`, assurance 86/86 and 276/276 with no dead links. Refs: 013/Req-3, #695, #696 Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.toml | 4 + src/codegen/prepare.rs | 1 + tests/corpus/main.rs | 1 + tests/corpus/prepare_oracle_test.rs | 157 +++++++++++++++++++ tests/performance/v6.rs | 13 +- tests/unit/prepare_test.rs | 224 ++++++++++++++++++++++++++++ 6 files changed, 395 insertions(+), 5 deletions(-) create mode 100644 tests/corpus/prepare_oracle_test.rs create mode 100644 tests/unit/prepare_test.rs diff --git a/Cargo.toml b/Cargo.toml index 99457437..177e7bad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,6 +132,10 @@ path = "tests/unit/vdbe_integrity_check_test.rs" name = "vdbe_streaming_execution" path = "tests/unit/vdbe_streaming_execution_test.rs" +[[test]] +name = "prepare" +path = "tests/unit/prepare_test.rs" + [[test]] name = "codegen_expr" path = "tests/unit/codegen_expr_test.rs" diff --git a/src/codegen/prepare.rs b/src/codegen/prepare.rs index ad6b3407..efa90608 100644 --- a/src/codegen/prepare.rs +++ b/src/codegen/prepare.rs @@ -39,6 +39,7 @@ use crate::vdbe::Program; /// What [`compile_select_program`] produced: either `EXPLAIN QUERY PLAN`'s rows /// (nothing further to compile — there is no bytecode to run) or an /// ordinary compiled [`Program`]. +#[derive(Debug)] pub enum SelectOutcome { /// `EXPLAIN QUERY PLAN` output rows. Eqp(Vec), diff --git a/tests/corpus/main.rs b/tests/corpus/main.rs index e0a60865..b38019f6 100644 --- a/tests/corpus/main.rs +++ b/tests/corpus/main.rs @@ -48,6 +48,7 @@ mod pager_write_test; mod parser_oracle_test; mod partial_sort_test; mod plan_parity_test; +mod prepare_oracle_test; mod regen_test; mod repl_test; mod schema_test; diff --git a/tests/corpus/prepare_oracle_test.rs b/tests/corpus/prepare_oracle_test.rs new file mode 100644 index 00000000..bce600be --- /dev/null +++ b/tests/corpus/prepare_oracle_test.rs @@ -0,0 +1,157 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! Oracle diff for the lifted `SELECT` pipeline (#695). +//! +//! `tests/unit/prepare_test.rs` pins the dispatch and the error type. +//! This pins the *answers*: the same queries, compiled through +//! `codegen::compile_select_program` from the library and executed here, +//! must produce byte-identical rows to the pinned `sqlite3` 3.53.4. +//! +//! It exists because the lift's whole claim is "same code, new home". I +//! checked four queries by hand while doing it, which is worth nothing +//! once the terminal closes — a refactor of the path every `sqlite-rs +//! query` takes deserves a check that runs in CI. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing +)] + +use std::collections::HashMap; +use std::process::Command; +use std::rc::Rc; + +use sqlite_rs::codegen::{compile_select_program, result_column_names, SelectOutcome}; +use sqlite_rs::parser::error::ParseOutcome; +use sqlite_rs::parser::parse_select; +use sqlite_rs::record::Value; +use sqlite_rs::schema::{read_schema, read_views}; +use sqlite_rs::vdbe::execute_with_db; +use sqlite_rs::vfs::PageSource; + +use crate::oracle::{pinned_oracle, skip_no_oracle}; + +const SETUP: &[&str] = &[ + "CREATE TABLE t(a INTEGER, b TEXT)", + "CREATE TABLE u(a INTEGER, c TEXT)", + "CREATE INDEX t_a ON t(a)", + "INSERT INTO t VALUES (1,'x'),(2,'y'),(3,'z')", + "INSERT INTO u VALUES (1,'p'),(3,'q')", +]; + +/// One per `compile_select_program` dispatch arm, plus the shapes SQE's +/// own statement list uses (`UNION`, `LIMIT 1` probes). +const QUERIES: &[&str] = &[ + "SELECT 1 + 1", + "SELECT a, b FROM t ORDER BY a", + "SELECT count(*) FROM t", + "SELECT a FROM t WHERE a > 1 ORDER BY a", + "SELECT b FROM t WHERE a = 2", + "SELECT t.a, u.c FROM t JOIN u ON t.a = u.a ORDER BY t.a", + "SELECT a FROM t UNION ALL SELECT a FROM u ORDER BY a", + "SELECT a FROM t UNION SELECT a FROM u ORDER BY a", + "SELECT b FROM t WHERE a = 1 LIMIT 1", + "SELECT a FROM t WHERE b = 'nope' LIMIT 1", + "SELECT max(a), min(a) FROM t", +]; + +/// Renders a row the way the oracle's default `-list` mode does, so the +/// two are comparable as text. +fn render(row: &[Value]) -> String { + row.iter() + .map(|v| match v { + Value::Null => String::new(), + Value::Integer(i) => i.to_string(), + Value::Real(r) => format!("{r}"), + Value::Text(s) => s.to_string(), + Value::Blob(b) => String::from_utf8_lossy(b).to_string(), + }) + .collect::>() + .join("|") +} + +#[test] +fn lifted_pipeline_answers_match_the_oracle() { + let Some(bin) = pinned_oracle() else { + skip_no_oracle("lifted_pipeline_answers_match_the_oracle"); + return; + }; + let dir = std::env::temp_dir().join(format!("sqlite-rs-prepare-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let db = dir.join("prepare.db"); + std::fs::remove_file(&db).ok(); + + // The oracle builds the fixture, so we are reading a file stock + // SQLite wrote — the adoption direction that matters. + let status = Command::new(&bin) + .arg(&db) + .arg(SETUP.join(";\n")) + .status() + .unwrap(); + assert!(status.success(), "oracle setup failed"); + + let (_vfs, pager, header) = open_readonly(&db); + + for sql in QUERIES { + let theirs = Command::new(&bin).arg(&db).arg(sql).output().unwrap(); + assert!( + theirs.status.success(), + "oracle rejected {sql}: {}", + String::from_utf8_lossy(&theirs.stderr) + ); + let expected = String::from_utf8_lossy(&theirs.stdout) + .trim_end() + .to_string(); + + let select = match parse_select(sql) { + ParseOutcome::Accepted(s) => *s, + other => panic!("{sql} did not parse: {other:?}"), + }; + let (schemas, views) = { + let mut c1 = sqlite_rs::btree::TableCursor::new(&*pager, &header, 1); + let schemas = read_schema(&mut c1, header.text_encoding).unwrap(); + let mut c2 = sqlite_rs::btree::TableCursor::new(&*pager, &header, 1); + let views = read_views(&mut c2, header.text_encoding).unwrap(); + (schemas, views) + }; + let program = + match compile_select_program(&select, false, &schemas, &views, &HashMap::new()) + .unwrap_or_else(|e| panic!("{sql} did not compile: {e}")) + { + SelectOutcome::Program(p) => p, + SelectOutcome::Eqp(_) => panic!("{sql} unexpectedly produced EQP rows"), + }; + + // Column names come from the same function the CLI's headers do, + // so a divergence there is a divergence for both. + let names = result_column_names(&select, &schemas); + assert!(!names.is_empty(), "{sql} derived no column names"); + + let source: Rc = Rc::clone(&pager) as Rc; + let rows = execute_with_db(&program, source, header) + .unwrap_or_else(|e| panic!("{sql} failed to execute: {e}")); + let ours = rows + .iter() + .map(|r| render(r)) + .collect::>() + .join("\n"); + + assert_eq!(ours, expected, "rows diverge for {sql}"); + } + + std::fs::remove_dir_all(&dir).ok(); +} + +fn open_readonly( + path: &std::path::Path, +) -> ( + sqlite_rs::vfs::UnixVfs, + Rc, + sqlite_rs::header::DatabaseHeader, +) { + let vfs = sqlite_rs::vfs::UnixVfs; + let (header, pager) = sqlite_rs::dump::open(&vfs, path).expect("open fixture"); + (vfs, Rc::new(pager), header) +} diff --git a/tests/performance/v6.rs b/tests/performance/v6.rs index 46e6bc6d..5297d395 100644 --- a/tests/performance/v6.rs +++ b/tests/performance/v6.rs @@ -481,11 +481,14 @@ fn open_ours(path: &Path) -> OursFixture { } /// `WITH`/view-expansion-then-resolve-then-compile pipeline mirroring -/// `src/bin/sqlite-rs/query.rs::compile_select_program` (the `sqlite-rs -/// query` CLI's real dispatch) closely enough for this bench's needs — -/// that function itself is `pub(crate)` to the binary crate and not -/// callable from an external test/bench crate, so the CTE-expansion + -/// resolve + single-table/joined dispatch is reproduced here rather than +/// `codegen::compile_select_program` (the `sqlite-rs query` CLI's real +/// dispatch) closely enough for this bench's needs. NOTE: that function +/// was `pub(crate)` to the binary crate when this duplicate was written, +/// which is why the duplicate exists; #695 lifted it into the library, so +/// this bench can now call the real thing and the duplication should be +/// removed — deliberately not done in the lift's own PR, since it changes +/// the path this bench measures. The CTE-expansion + resolve + +/// single-table/joined dispatch is reproduced here rather than /// reused. No view expansion: none of this file's SQL references a view. fn compile_ours_select(select: &Select, catalog: &[TableSchema]) -> Program { let expanded = expand_with_clause(select); diff --git a/tests/unit/prepare_test.rs b/tests/unit/prepare_test.rs new file mode 100644 index 00000000..9e5a2ecb --- /dev/null +++ b/tests/unit/prepare_test.rs @@ -0,0 +1,224 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! `codegen::prepare` — the `SELECT` compile pipeline, as a library +//! function rather than a CLI internal (#695). +//! +//! These exist because the lift made `compile_select_program` and +//! `result_column_names` public API with no direct coverage: everything +//! that exercised them went through the `sqlite-rs` binary, so a +//! regression in the library surface would only have shown up as a CLI +//! failure. The shapes below are the four `compile_select_program` +//! dispatches on (FROM-less, single-table, joined, compound) plus the +//! one error that is about the request rather than the SQL. +//! +//! Row-level agreement with the oracle is +//! `tests/corpus/prepare_oracle_test.rs`'s job; this file is about the +//! dispatch and the error type. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::collections::HashMap; + +use sqlite_rs::codegen::{ + compile_select_program, result_column_names, PrepareError, SelectOutcome, +}; +use sqlite_rs::parser::error::ParseOutcome; +use sqlite_rs::parser::{parse_explain, parse_select}; +use sqlite_rs::planner::Stats; +use sqlite_rs::record::Collation; +use sqlite_rs::schema::{TableSchema, ViewSchema}; + +/// Two tables, declared the way `ddl_reader` would report them. +fn catalog() -> Vec { + vec![table("t", &["a", "b"]), table("u", &["a", "c"])] +} + +fn table(name: &str, columns: &[&str]) -> TableSchema { + TableSchema { + name: name.to_string(), + root_page: 2, + columns: columns.iter().map(|c| (*c).to_string()).collect(), + column_types: columns.iter().map(|_| "INTEGER".to_string()).collect(), + column_collations: columns.iter().map(|_| Collation::Binary).collect(), + without_rowid: false, + strict: false, + is_virtual: false, + sql: format!("CREATE TABLE {name}({})", columns.join(", ")), + indexes: vec![], + rowid_alias: None, + unresolved_autoindex: false, + } +} + +fn select_of(sql: &str) -> sqlite_rs::parser::ast::Select { + match parse_select(sql) { + ParseOutcome::Accepted(s) => *s, + other => panic!("{sql} did not parse: {other:?}"), + } +} + +fn compile(sql: &str) -> Result { + let select = select_of(sql); + compile_select_program(&select, false, &catalog(), &[], &HashMap::new()) +} + +fn program_of(sql: &str) -> sqlite_rs::vdbe::Program { + match compile(sql).unwrap_or_else(|e| panic!("{sql} did not compile: {e}")) { + SelectOutcome::Program(p) => p, + SelectOutcome::Eqp(_) => panic!("{sql} produced EQP rows, not a program"), + } +} + +#[test] +fn compiles_all_four_select_shapes() { + // The dispatch inside `compile_select_program`, one case each. A + // non-empty program is the assertion: each arm reaches a different + // compiler, and a silent fallthrough would come back empty or error. + for sql in [ + "SELECT 1 + 1", // FROM-less (#260) + "SELECT a, b FROM t WHERE a > 1", // single-table + "SELECT t.a, u.c FROM t JOIN u ON t.a = u.a", // joined (#237) + "SELECT a FROM t UNION ALL SELECT a FROM u", // compound (#240) + ] { + let program = program_of(sql); + assert!( + !program.instructions.is_empty(), + "{sql} compiled to an empty program" + ); + } +} + +#[test] +fn explain_query_plan_returns_rows_not_a_program() { + let sql = "EXPLAIN QUERY PLAN SELECT a FROM t WHERE a > 1"; + let select = match parse_explain(sql) { + ParseOutcome::Accepted(e) => e, + other => panic!("{sql} did not parse: {other:?}"), + }; + let outcome = + compile_select_program(&select.select, true, &catalog(), &[], &HashMap::new()).unwrap(); + match outcome { + SelectOutcome::Eqp(rows) => assert!(!rows.is_empty(), "EQP produced no rows"), + SelectOutcome::Program(_) => panic!("eqp_mode returned a program"), + } +} + +/// The one error that is about the *request* rather than the SQL, and +/// the reason `PrepareError` exists rather than reusing `CodegenError`: +/// `SELECT 1` compiles perfectly well, it just has no access path to +/// explain. +#[test] +fn eqp_without_from_is_its_own_error() { + let select = select_of("SELECT 1"); + let err = compile_select_program(&select, true, &catalog(), &[], &HashMap::new()) + .expect_err("EQP on a FROM-less SELECT should fail"); + + assert_eq!(err, PrepareError::EqpWithoutFrom); + // Byte-identical to the string the CLI printed before the lift. + assert_eq!(err.to_string(), "EXPLAIN QUERY PLAN requires a FROM clause"); + + // And the same statement without `eqp_mode` still compiles, which is + // what makes this distinct from `CodegenError::NoFromClause`. + assert!(matches!( + compile_select_program(&select, false, &catalog(), &[], &HashMap::new()), + Ok(SelectOutcome::Program(_)) + )); +} + +#[test] +fn a_missing_table_surfaces_as_a_codegen_error_not_a_string() { + let select = select_of("SELECT a FROM nonexistent"); + let err = compile_select_program(&select, false, &catalog(), &[], &HashMap::new()) + .expect_err("unknown table should fail"); + + // The point of the lift's error change: a caller can match on the + // variant instead of parsing a message. + assert!( + matches!(err, PrepareError::Codegen(_)), + "expected a wrapped CodegenError, got {err:?}" + ); + assert!( + err.to_string().contains("nonexistent"), + "message should name the table: {err}" + ); +} + +#[test] +fn result_column_names_uses_real_names_for_a_single_table() { + let select = select_of("SELECT a, b FROM t"); + assert_eq!(result_column_names(&select, &catalog()), vec!["a", "b"]); +} + +#[test] +fn result_column_names_honours_aliases() { + // Non-keyword aliases on purpose. `AS first` is what this test used + // first, and it failed — not because aliasing is broken but because + // `FIRST` is one of the 89 keywords we reserve that SQLite treats as + // an identifier (#696). That is a parser bug, not a `prepare` bug, + // so it is pinned there rather than smuggled in here. + let select = select_of("SELECT a AS alpha, b AS beta FROM t"); + assert_eq!( + result_column_names(&select, &catalog()), + vec!["alpha", "beta"] + ); +} + +/// Joins and compounds fall back to positional names. Asserted rather +/// than left implicit because `Row::get_by_name` will be built on this, +/// and "column1" is a real answer a consumer can receive — not a bug. +#[test] +fn result_column_names_falls_back_positionally_for_joins_and_compounds() { + let joined = select_of("SELECT t.a, u.c FROM t JOIN u ON t.a = u.a"); + assert_eq!( + result_column_names(&joined, &catalog()), + vec!["column1", "column2"] + ); + + let compound = select_of("SELECT a FROM t UNION ALL SELECT a FROM u"); + assert_eq!(result_column_names(&compound, &catalog()), vec!["column1"]); +} + +/// An unknown table makes name derivation fall back rather than panic — +/// `result_column_names` returns names, not a `Result`, so it has to +/// have an answer for every input. +#[test] +fn result_column_names_falls_back_when_the_table_is_unknown() { + let select = select_of("SELECT a, b FROM nonexistent"); + assert_eq!( + result_column_names(&select, &catalog()), + vec!["column1", "column2"] + ); +} + +/// `views` is threaded through the pipeline; an empty catalog must not +/// make a plain `SELECT` fail. +#[test] +fn an_empty_view_catalog_is_fine() { + let select = select_of("SELECT a FROM t"); + let views: Vec = vec![]; + assert!(matches!( + compile_select_program(&select, false, &catalog(), &views, &HashMap::new()), + Ok(SelectOutcome::Program(_)) + )); +} + +/// Stats are consulted for the single-table path; a table with no +/// `sqlite_stat1` row must compile the same as one with stats present. +#[test] +fn missing_stats_compile_the_same_as_present_stats() { + let select = select_of("SELECT a FROM t WHERE a > 1"); + let without = program_of("SELECT a FROM t WHERE a > 1"); + + let mut stats = HashMap::new(); + stats.insert("t".to_string(), Stats::default()); + let with = match compile_select_program(&select, false, &catalog(), &[], &stats).unwrap() { + SelectOutcome::Program(p) => p, + SelectOutcome::Eqp(_) => panic!("unexpected EQP"), + }; + + assert_eq!( + without.instructions.len(), + with.instructions.len(), + "default stats should not change the plan" + ); +} From 606a176cec0c1954cd500c407a8bc52a0ddcc287 Mon Sep 17 00:00:00 2001 From: Diederik Siderius Date: Wed, 9 Sep 2026 20:04:03 +0200 Subject: [PATCH 04/14] =?UTF-8?q?feat:=20engine=20seams=20for=20the=20embe?= =?UTF-8?q?dding=20API=20=E2=80=94=20last-insert=20rowid=20and=20placehold?= =?UTF-8?q?er=20counting=20(013/Req=201,=20Req=203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec 013's `Connection` needs three things from the engine that it cannot synthesise, and this is all of them bar the ones #692 already landed. **`last_insert_rowid`, flagged by codegen** (`OPFLAG_LASTROWID`, 0x20 — same bit as stock SQLite, `sqliteInt.h:4068` at the pinned 3.53.4). A row inserted into a table with a surrogate key is unaddressable until the caller learns its rowid, so this is not a convenience. Two properties of the upstream handler are reproduced deliberately, and neither is guessable from the name: * It nests *inside* `OPFLAG_NCHANGE`. `vdbe.c:5803` updates `db->lastRowid` within the `if( p5 & OPFLAG_NCHANGE )` arm, and `vdbe.c:5800` asserts the implication. A mutation that is not a counted row change must not move the rowid either. * An `UPDATE` does not set it. `insert.c:2834` reads `pik_flags |= (update_flags ? update_flags : OPFLAG_LASTROWID)` — the flag is the *else* branch. An UPDATE emits an `Insert`, so hooking the opcode instead of the flag would report the updated row's rowid and destroy the value the caller was about to use. The hook is on `Insert`, not `NewRowid`, because for `INSERT INTO t(id, ...)` over an `INTEGER PRIMARY KEY` the rowid comes from the bound value and `NewRowid` never executes — which is exactly the shape a consumer with its own keys uses. **`Program::param_count()`**, derived rather than stored, for the same reason #692 gave for `counts_changes()`: a stored field can disagree with the instructions it describes, and `Program::new` is public. It reads the ceiling off the emitted `Variable` instructions, matching `sqlite3_bind_parameter_count`'s "largest index, not number of distinct" (`expr.c:1331` for bare `?`, `expr.c:1356` for `?nnn`). **Named parameters are now refused** rather than compiled. `:name`/ `@name`/`$name` parse but were never wired to an index, and compiled to a fresh NULL-reading register — turning `WHERE x = :name` into `WHERE x = NULL`, which matches no row and raises no error. A silent wrong answer is the worst failure mode for a consumer binding by name, so this is a deliberate behaviour change. The indices still don't exist; the refusal is honest until they do. `StepOutcome` gains `last_insert_rowid: Option`, `None` meaning "leave the connection's stored value alone" — the same retention rule as `changes`, and an `Option` rather than a sentinel because rowid 0 is legal. Tests ----- `tests/corpus/last_insert_rowid_oracle_test.rs` diffs the *retained* value against the pinned 3.53.4 after all 14 statements of a sequence, in one oracle invocation (the value is connection-scoped, so a fresh connection per statement would reset it and make retention untestable). Comparing only the inserting statements would pass even if `UPDATE` cleared the value — the bug the flag exists to prevent. Confirmed discriminating: with `OPFLAG_LASTROWID` wrongly added to `update.rs`, `UPDATE ... WHERE a = 1` drags the value to 1 where the oracle holds at 2. `tests/unit/vdbe_last_insert_rowid_test.rs` (8 tests) pins the mechanism, including the explicit-`INTEGER PRIMARY KEY` case, a deliberately *decreasing* rowid so a stale-value bug cannot hide behind a monotonic sequence, and two structural assertions: an INSERT flags exactly one instruction, and no program may ever carry the rowid flag without the change flag. `tests/unit/param_binding_test.rs` (9 tests) covers the counting rules and every named-parameter form, in reads and writes. Gates: make test (1603 passed), make test-corpus (390 passed), make lint (both clippy passes + fmt), check-mod-files, check-assurance 100%/100%. Spend: within estimate. Co-Authored-By: Claude Opus 5 --- Cargo.toml | 8 + src/codegen/expr/value.rs | 44 ++- src/codegen/stmt/insert.rs | 12 +- src/codegen/stmt/update.rs | 6 + src/vdbe.rs | 2 +- src/vdbe/cursor.rs | 10 +- src/vdbe/exec.rs | 70 +++- src/vdbe/program.rs | 45 +++ tests/corpus/last_insert_rowid_oracle_test.rs | 265 ++++++++++++++ tests/corpus/main.rs | 1 + tests/unit/param_binding_test.rs | 309 +++++++++++++++++ tests/unit/vdbe_last_insert_rowid_test.rs | 327 ++++++++++++++++++ 12 files changed, 1073 insertions(+), 26 deletions(-) create mode 100644 tests/corpus/last_insert_rowid_oracle_test.rs create mode 100644 tests/unit/param_binding_test.rs create mode 100644 tests/unit/vdbe_last_insert_rowid_test.rs diff --git a/Cargo.toml b/Cargo.toml index 086e2875..65c55440 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,10 +136,18 @@ path = "tests/unit/vdbe_streaming_execution_test.rs" name = "vdbe_changes" path = "tests/unit/vdbe_changes_test.rs" +[[test]] +name = "vdbe_last_insert_rowid" +path = "tests/unit/vdbe_last_insert_rowid_test.rs" + [[test]] name = "prepare" path = "tests/unit/prepare_test.rs" +[[test]] +name = "param_binding" +path = "tests/unit/param_binding_test.rs" + [[test]] name = "codegen_expr" path = "tests/unit/codegen_expr_test.rs" diff --git a/src/codegen/expr/value.rs b/src/codegen/expr/value.rs index 75a779a5..d943d3ba 100644 --- a/src/codegen/expr/value.rs +++ b/src/codegen/expr/value.rs @@ -215,23 +215,39 @@ pub(crate) fn compile_value( // `?` and `?NNN` compile to `Variable`, reading whatever the // caller bound via `Vm::bind_params`/`execute_with_params` - // (#137). Named forms (`:name`/`@name`/`$name`) aren't wired to - // an index yet — out of #137's bounded scope — so they still - // compile to an always-NULL register (known simplification, - // same as before). + // (#137). + // + // Named forms (`:name`/`@name`/`$name`) are rejected rather than + // compiled (013/Req 3). They are still not wired to an index — that + // was out of #137's bounded scope and remains out of scope here — + // but the previous behaviour was to emit nothing and hand back a + // fresh (NULL-reading) register. That turns `WHERE x = :name` into + // `WHERE x = NULL`, which matches no row and raises no error: a + // silent wrong answer, and the worst possible failure mode for a + // consumer binding parameters by name. Refusing to compile is the + // honest answer until the indices exist. ExprKind::Param(kind) => { - let r = reg.alloc(); let index = match kind { - ParamKind::Anonymous => Some(reg.anonymous_param()), - ParamKind::Numbered(n) => Some(reg.numbered_param(*n)), - ParamKind::Colon(_) | ParamKind::At(_) | ParamKind::Dollar(_) => None, + ParamKind::Anonymous => reg.anonymous_param(), + ParamKind::Numbered(n) => reg.numbered_param(*n), + ParamKind::Colon(name) | ParamKind::At(name) | ParamKind::Dollar(name) => { + let sigil = match kind { + ParamKind::At(_) => '@', + ParamKind::Dollar(_) => '$', + _ => ':', + }; + return Err(CodegenError::Unsupported { + reason: format!( + "named parameter {sigil}{name} is not supported — bind by position with ? or ?NNN" + ), + }); + } }; - if let Some(index) = index { - let p1 = i32::try_from(index).map_err(|_| CodegenError::Unsupported { - reason: format!("parameter index {index} is out of range"), - })?; - em.emit(Instruction::new(Opcode::Variable, p1, r, 0)); - } + let r = reg.alloc(); + let p1 = i32::try_from(index).map_err(|_| CodegenError::Unsupported { + reason: format!("parameter index {index} is out of range"), + })?; + em.emit(Instruction::new(Opcode::Variable, p1, r, 0)); Ok(r) } diff --git a/src/codegen/stmt/insert.rs b/src/codegen/stmt/insert.rs index 067a1c9f..2af9f4ab 100644 --- a/src/codegen/stmt/insert.rs +++ b/src/codegen/stmt/insert.rs @@ -95,7 +95,9 @@ use crate::parser::ast::{ use crate::parser::error::ParseOutcome; use crate::parser::parse_create_table; use crate::schema::TableSchema; -use crate::vdbe::{affinity_of, Instruction, Opcode, Program, OPFLAG_NCHANGE, P4}; +use crate::vdbe::{ + affinity_of, Instruction, Opcode, Program, OPFLAG_LASTROWID, OPFLAG_NCHANGE, P4, +}; /// Process-wide cache of parsed `CREATE TABLE` DDL, keyed by the exact /// `schema.sql` text — content-addressed, since the parse result depends @@ -804,12 +806,18 @@ fn compile_row( // `OPFLAG_NCHANGE`: this is the one mutation an INSERT reports as a // changed row (013/Req 1, #692). The index maintenance emitted just // below is deliberately unflagged. + // + // `OPFLAG_LASTROWID` rides along on the same instruction, matching + // `insert.c:2834`'s `pik_flags |= (update_flags ? update_flags : + // OPFLAG_LASTROWID)` — an INSERT takes the `else` branch and so sets + // it, while `codegen/stmt/update.rs`'s `Insert` takes the other and + // does not. That is why `UPDATE` leaves the last-insert rowid alone. em.emit(Instruction::with_p5( Opcode::Insert, TABLE_CURSOR, rowid_reg, record_reg, - OPFLAG_NCHANGE, + OPFLAG_NCHANGE | OPFLAG_LASTROWID, )); if !schema.indexes.is_empty() { diff --git a/src/codegen/stmt/update.rs b/src/codegen/stmt/update.rs index 0b988297..4a2e8f57 100644 --- a/src/codegen/stmt/update.rs +++ b/src/codegen/stmt/update.rs @@ -620,6 +620,12 @@ fn emit_update_row_body( // `OPFLAG_NCHANGE`. Flagging both would report 2 per row; flagging the // `Delete` instead would work equally well but reads as a deletion. // Stock SQLite flags the insert side too. + // + // Deliberately *not* `OPFLAG_LASTROWID`, even though this is an + // `Insert`: `insert.c:2834` sets that flag only on the `else` branch of + // `pik_flags |= (update_flags ? update_flags : OPFLAG_LASTROWID)`, and + // an UPDATE supplies `update_flags`. So rewriting a row must leave + // `last_insert_rowid` reporting whatever the last real INSERT set. em.emit(Instruction::new(Opcode::Delete, TABLE_CURSOR, 0, 0)); em.emit(Instruction::with_p5( Opcode::Insert, diff --git a/src/vdbe.rs b/src/vdbe.rs index e4db4e39..6243f826 100644 --- a/src/vdbe.rs +++ b/src/vdbe.rs @@ -47,6 +47,6 @@ pub use pragma::{ }; pub use program::{ AnalyzeIndexTarget, AnalyzeTarget, GroupKeyColumn, Instruction, Opcode, Program, SortKeyColumn, - OPFLAG_NCHANGE, P4, + OPFLAG_LASTROWID, OPFLAG_NCHANGE, P4, }; pub use value::{and, is, is_not, not, or, sql_eq, sql_lt}; diff --git a/src/vdbe/cursor.rs b/src/vdbe/cursor.rs index e186da5e..374e3e4b 100644 --- a/src/vdbe/cursor.rs +++ b/src/vdbe/cursor.rs @@ -59,7 +59,7 @@ use crate::record::{ record_column_count, TextEncoding, Value, }; use crate::vdbe::exec::{to_pc, ExecError, Step, Vm}; -use crate::vdbe::program::{Instruction, OPFLAG_NCHANGE, P4}; +use crate::vdbe::program::{Instruction, OPFLAG_LASTROWID, OPFLAG_NCHANGE, P4}; use crate::vdbe::{compare, Collation}; /// One open cursor slot: a real table cursor, an in-memory ephemeral @@ -2063,6 +2063,14 @@ pub fn insert(vm: &mut Vm, instr: &Instruction) -> Result { // a future caller flagged it by mistake. if instr.p5 & OPFLAG_NCHANGE != 0 { vm.record_change(); + // Nested inside `NCHANGE`, not beside it — `vdbe.c:5803` + // updates `db->lastRowid` within the same arm and + // `vdbe.c:5800` asserts `LASTROWID` implies `NCHANGE`. + // `rowid` here is the key the insert actually used, which + // is the value `sqlite3_last_insert_rowid()` returns. + if instr.p5 & OPFLAG_LASTROWID != 0 { + vm.record_last_insert_rowid(rowid); + } } Ok(Step::Next) } diff --git a/src/vdbe/exec.rs b/src/vdbe/exec.rs index bdb460d9..b857e6ee 100644 --- a/src/vdbe/exec.rs +++ b/src/vdbe/exec.rs @@ -328,6 +328,11 @@ pub struct Vm { /// set the flag, so a table with three indexes reports the same /// number as the same table with none. changes: u64, + /// Rowid of the last row this program inserted (013/Req 1), or `None` + /// when it inserted none. Set by `Insert` only when its `P5` carries + /// [`OPFLAG_LASTROWID`] — see that constant for why an `UPDATE` and + /// an index/ephemeral write leave it alone. + last_insert_rowid: Option, /// Reused byte buffer for `MakeRecord` (#454): amortizes the record /// payload's allocation across every row a statement emits, instead /// of a fresh `Vec` per `MakeRecord` execution. @@ -356,6 +361,7 @@ impl Default for Vm { params: Vec::new(), autocommit: true, changes: 0, + last_insert_rowid: None, record_scratch: Vec::new(), make_record_values_scratch: Vec::new(), encode_scratch: Vec::new(), @@ -400,6 +406,27 @@ impl Vm { self.changes } + /// Records `rowid` as this program's last inserted rowid (013/Req 1). + /// Called by `Insert` only when the instruction's `P5` carries + /// [`OPFLAG_LASTROWID`], and only from inside the + /// [`OPFLAG_NCHANGE`] arm, mirroring `vdbe.c:5803`. + pub(crate) fn record_last_insert_rowid(&mut self, rowid: i64) { + self.last_insert_rowid = Some(rowid); + } + + /// Rowid of the last row this program inserted, or `None` if it + /// inserted none (013/Req 1). + /// + /// Per-`Vm`, and a `Vm` lives for one statement — so `None` means + /// "this statement inserted nothing", not "there is no last rowid". + /// `sqlite3_last_insert_rowid()` is connection-scoped (`db->lastRowid`) + /// and a statement that inserts nothing must leave the previous value + /// standing; that retention is the connection's rule, exactly as it is + /// for [`Self::changes`]. See [`StepOutcome::last_insert_rowid`]. + pub fn last_insert_rowid(&self) -> Option { + self.last_insert_rowid + } + /// Reused scratch buffer for `MakeRecord` (#454) — see /// [`Vm::record_scratch`]'s field doc. pub(crate) fn record_scratch(&mut self) -> &mut Vec { @@ -1039,7 +1066,7 @@ const MAX_STEPS: u32 = 50_000_000; /// Runs `program` to completion on a fresh, database-less [`Vm`] and /// returns the rows it emitted via `ResultRow`. pub fn execute(program: &Program) -> Result>, ExecError> { - run(Vm::new(), program).map(|(rows, _, _)| rows) + run(Vm::new(), program).map(|(rows, _, _, _)| rows) } /// Like [`execute`], but binds `params` for `Opcode::Variable` to read @@ -1052,7 +1079,7 @@ pub fn execute_with_params( ) -> Result>, ExecError> { let mut vm = Vm::new(); vm.bind_params(params); - run(vm, program).map(|(rows, _, _)| rows) + run(vm, program).map(|(rows, _, _, _)| rows) } /// Like [`execute`], but the `Vm` can service `OpenRead` (cursor @@ -1063,7 +1090,7 @@ pub fn execute_with_db( source: Rc, header: DatabaseHeader, ) -> Result>, ExecError> { - run(Vm::with_db(source, header), program).map(|(rows, _, _)| rows) + run(Vm::with_db(source, header), program).map(|(rows, _, _, _)| rows) } /// Like [`execute_with_db`], but the `Vm` can also service the write @@ -1074,7 +1101,7 @@ pub fn execute_with_writable_db( pager: crate::pager::Pager, header: DatabaseHeader, ) -> Result>, ExecError> { - run(Vm::with_writable_db(pager, header), program).map(|(rows, _, _)| rows) + run(Vm::with_writable_db(pager, header), program).map(|(rows, _, _, _)| rows) } /// Combines [`execute_with_db`] and [`execute_with_params`]. @@ -1086,7 +1113,7 @@ pub fn execute_with_db_and_params( ) -> Result>, ExecError> { let mut vm = Vm::with_db(source, header); vm.bind_params(params); - run(vm, program).map(|(rows, _, _)| rows) + run(vm, program).map(|(rows, _, _, _)| rows) } /// Runs one statement's `program` against a `pager` shared across @@ -1131,6 +1158,14 @@ pub struct StepOutcome { /// implement (spec 013/Req 1's `Connection::changes`); this type just /// makes it a one-liner. pub changes: Option, + /// Rowid of the last row this statement inserted, or `None` when it + /// inserted none (013/Req 1). + /// + /// `None` is "leave the connection's stored value alone", not "reset + /// it" — the same retention rule as [`Self::changes`], and the reason + /// this is an `Option` rather than a sentinel `0`. Rowid `0` is a legal + /// rowid, so a sentinel could not be distinguished from a real insert. + pub last_insert_rowid: Option, } /// [`execute_transaction_step`] plus the rows-changed count (013/Req 1, @@ -1147,11 +1182,12 @@ pub fn execute_transaction_step_counted( ) -> Result { let mut vm = Vm::with_shared_writable_db(pager, header); vm.autocommit = autocommit_in; - let (rows, autocommit, changed) = run(vm, program)?; + let (rows, autocommit, changed, last_insert_rowid) = run(vm, program)?; Ok(StepOutcome { rows, autocommit, changes: program.counts_changes().then_some(changed), + last_insert_rowid, }) } @@ -1300,15 +1336,33 @@ impl<'p> Execution<'p> { pub fn changes(&self) -> u64 { self.vm.changes() } + + /// Rowid of the last row this execution inserted, or `None` if it has + /// inserted none *so far* (013/Req 1). + /// + /// Like [`Self::changes`], this is readable mid-stream and reflects + /// only the inserts already executed. Statements that stream rows and + /// insert are rare, but the honest answer for a partially-drained + /// execution is a partial one rather than a wrong one. + pub fn last_insert_rowid(&self) -> Option { + self.vm.last_insert_rowid() + } } -fn run(vm: Vm, program: &Program) -> Result<(Vec>, bool, u64), ExecError> { +type RunOutcome = (Vec>, bool, u64, Option); + +fn run(vm: Vm, program: &Program) -> Result { let mut execution = Execution::new(vm, program); let mut rows = Vec::new(); while let Some(row) = execution.next_row()? { rows.push(row); } - Ok((rows, execution.autocommit(), execution.changes())) + Ok(( + rows, + execution.autocommit(), + execution.changes(), + execution.last_insert_rowid(), + )) } #[cfg(test)] diff --git a/src/vdbe/program.rs b/src/vdbe/program.rs index 7ecf752f..9d9da5be 100644 --- a/src/vdbe/program.rs +++ b/src/vdbe/program.rs @@ -848,6 +848,30 @@ pub struct Instruction { /// which mutation is the row change; the handler does not. pub const OPFLAG_NCHANGE: u16 = 0x01; +/// `P5` bit marking the `Insert` whose rowid `Connection::last_insert_rowid` +/// should report (013/Req 1). +/// +/// Same value (`0x20`) and same job as stock SQLite's `OPFLAG_LASTROWID` +/// (`sqliteInt.h:4068`, pinned 3.53.4). Two properties of the upstream +/// handler are worth stating because they are not guessable from the name, +/// and both are reproduced in `cursor::insert`: +/// +/// 1. **It nests inside [`OPFLAG_NCHANGE`].** `vdbe.c:5803` updates +/// `db->lastRowid` *within* the `if( p5 & OPFLAG_NCHANGE )` arm, and +/// `vdbe.c:5800` asserts `LASTROWID` implies `NCHANGE`. So a mutation +/// that is not a counted row change never moves the rowid either. +/// 2. **An `UPDATE` does not set it.** `insert.c:2834` reads +/// `pik_flags |= (update_flags ? update_flags : OPFLAG_LASTROWID)` — the +/// flag is the *else* branch, so rewriting a row leaves the last-insert +/// rowid alone. `codegen/stmt/update.rs`'s `Insert` therefore carries +/// `OPFLAG_NCHANGE` only. +/// +/// The rowid recorded is the one the insert actually used, which is why the +/// hook is here and not on `NewRowid`: for `INSERT INTO t(id, ...)` over an +/// `INTEGER PRIMARY KEY`, the rowid comes from the bound value and +/// `NewRowid` never executes. +pub const OPFLAG_LASTROWID: u16 = 0x20; + impl Instruction { /// Builds an instruction with `P4` absent and `P5` zero — the common /// case for control/arithmetic/compare opcodes that only use @@ -925,6 +949,27 @@ impl Program { }) } + /// The highest 1-based parameter index this program will read, or `0` + /// when it has no placeholders (013/Req 3). + /// + /// Derived rather than stored, for the same reason + /// [`Self::counts_changes`] is: a stored field can disagree with the + /// instructions it describes, and [`Program::new`] is public. + /// + /// It also measures the right thing. This counts what the VM will + /// *read* — `Opcode::Variable`'s `P1` — not what the compiler handed + /// out, so `?3` alone reports 3 (matching `sqlite3_bind_parameter_count`, + /// which returns the largest index, not the number of distinct ones) + /// and a placeholder the optimizer folded away is not demanded. + pub fn param_count(&self) -> usize { + self.instructions + .iter() + .filter(|i| matches!(i.opcode, Opcode::Variable)) + .filter_map(|i| usize::try_from(i.p1).ok()) + .max() + .unwrap_or(0) + } + /// Returns the instruction at `pc`, or `None` if `pc` is out of /// range. pub fn get(&self, pc: usize) -> Option<&Instruction> { diff --git a/tests/corpus/last_insert_rowid_oracle_test.rs b/tests/corpus/last_insert_rowid_oracle_test.rs new file mode 100644 index 00000000..d719bb31 --- /dev/null +++ b/tests/corpus/last_insert_rowid_oracle_test.rs @@ -0,0 +1,265 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! Oracle diff for the last-inserted-rowid signal (spec 013 Requirement 1): +//! runs one statement sequence through this crate's write path and the same +//! sequence through the pinned `sqlite3`, and compares what +//! `sqlite3_last_insert_rowid()` reports after *every* statement. +//! +//! The claim under test is deliberately the *retained* value, not the +//! per-statement one. `StepOutcome::last_insert_rowid` is `Option` — +//! `None` meaning "this statement inserted nothing, leave the stored value +//! alone" — and the interesting question is whether folding that signal the +//! way a connection will (`retained = reported.unwrap_or(retained)`) +//! reproduces SQLite's connection-scoped `db->lastRowid`. Comparing only the +//! statements that *do* insert would pass even if `UPDATE` wrongly cleared +//! the value, which is exactly the bug `OPFLAG_LASTROWID` exists to prevent. +//! +//! The unit suite (`tests/unit/vdbe_last_insert_rowid_test.rs`) pins the +//! mechanism; this pins the answers against the definition of correctness. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing +)] + +use std::cell::RefCell; +use std::path::Path; +use std::process::Command; +use std::rc::Rc; + +use std::collections::HashMap; + +use sqlite_rs::btree::TableCursor; +use sqlite_rs::codegen::{compile_select_program, compile_statement, SelectOutcome}; +use sqlite_rs::header::DatabaseHeader; +use sqlite_rs::pager::Pager; +use sqlite_rs::parser::{parse_select, ParseOutcome}; +use sqlite_rs::schema::{read_schema, read_views, TableSchema, ViewSchema}; +use sqlite_rs::vdbe::{execute_transaction_step_counted, Program}; +use sqlite_rs::vfs::MemoryVfs; + +use crate::oracle::{pinned_oracle, skip_no_oracle}; + +/// The sequence both engines run. +/// +/// Every statement is one this crate compiles today, and between them they +/// cover each way the value can move or must hold still: +/// +/// * an implicit rowid from `NewRowid` (`t`), +/// * an *explicit* rowid supplied as a value for an `INTEGER PRIMARY KEY` +/// (`u`), where `NewRowid` never runs at all — the case that makes hooking +/// `Insert` rather than `NewRowid` load-bearing, +/// * a non-contiguous explicit rowid, so a stale-value bug cannot hide +/// behind a coincidence, +/// * `UPDATE` (matching and non-matching), `DELETE` and `SELECT`, none of +/// which may move the value, +/// * inserts into an indexed table, where index maintenance must not be +/// mistaken for a row insert. +const STATEMENTS: &[&str] = &[ + "CREATE TABLE t(a INTEGER, b TEXT)", + "INSERT INTO t VALUES (1, 'b1')", + "INSERT INTO t VALUES (2, 'b2')", + // Must not move the value: an UPDATE rewrites a row as Delete+Insert, + // and only the INSERT path carries OPFLAG_LASTROWID. + "UPDATE t SET b = 'z' WHERE a = 1", + "UPDATE t SET b = 'q' WHERE a = 999", + // Must not move the value. + "DELETE FROM t WHERE a = 1", + "SELECT a FROM t", + // Explicit rowid via INTEGER PRIMARY KEY: NewRowid never executes, so + // the reported rowid has to come from the insert's own key. + "CREATE TABLE u(id INTEGER PRIMARY KEY, v TEXT)", + "INSERT INTO u(id, v) VALUES (42, 'forty-two')", + "INSERT INTO u(id, v) VALUES (7, 'seven')", + // Back to an implicit rowid on the other table — the value must follow + // the most recent insert, not the highest rowid ever seen. + "INSERT INTO t VALUES (3, 'b3')", + // Index maintenance must not count as an insert. + "CREATE INDEX u_v ON u(v)", + "INSERT INTO u(id, v) VALUES (100, 'hundred')", + "DELETE FROM u", +]; + +/// Compiles any statement in `STATEMENTS`, whichever kind it is. +/// +/// `compile_statement` handles writes and DDL but answers +/// `Unrecognized("SELECT")` for a read, so a sequence containing both needs +/// the two entry points dispatched between — which is exactly the shape +/// spec 013's `Connection::prepare` has to present as one call, and why +/// #695 lifted the `SELECT` pipeline into the library. This helper is that +/// dispatch in miniature; the `SELECT` in the sequence is load-bearing +/// (a read must not move the value) so it cannot simply be dropped. +fn compile_any(sql: &str, schemas: &[TableSchema], views: &[ViewSchema]) -> Program { + if !sql.trim_start().to_ascii_uppercase().starts_with("SELECT") { + return compile_statement(sql, schemas, views) + .unwrap_or_else(|e| panic!("{sql} did not compile: {e}")); + } + let select = match parse_select(sql) { + ParseOutcome::Accepted(select) => *select, + ParseOutcome::Unsupported { message, .. } | ParseOutcome::Invalid { message, .. } => { + panic!("{sql} did not parse: {message}") + } + }; + let stats: HashMap = HashMap::new(); + match compile_select_program(&select, false, schemas, views, &stats) { + Ok(SelectOutcome::Program(program)) => program, + Ok(SelectOutcome::Eqp(_)) => panic!("{sql} unexpectedly compiled to EXPLAIN QUERY PLAN"), + Err(e) => panic!("{sql} did not compile: {e}"), + } +} + +fn empty_db(page_size: u32) -> (MemoryVfs, DatabaseHeader) { + let mut page1 = vec![0u8; page_size as usize]; + page1[0..16].copy_from_slice(b"SQLite format 3\0"); + page1[16..18].copy_from_slice(&u16::try_from(page_size).unwrap().to_be_bytes()); + page1[18] = 1; + page1[19] = 1; + page1[28..32].copy_from_slice(&1u32.to_be_bytes()); + page1[56..60].copy_from_slice(&1u32.to_be_bytes()); + page1[100] = 0x0D; + page1[105..107].copy_from_slice(&u16::try_from(page_size).unwrap().to_be_bytes()); + + let mut header_bytes = [0u8; 100]; + header_bytes.copy_from_slice(&page1[..100]); + let header = DatabaseHeader::parse(&header_bytes).unwrap(); + + let mut vfs = MemoryVfs::new(); + vfs.insert("/test.db", page1); + (vfs, header) +} + +/// Runs `STATEMENTS` through this crate, folding the per-statement signal +/// into a connection-scoped value the way spec 013's `Connection` will, and +/// returning that retained value after each statement. +/// +/// Also returns the raw per-statement signal so the caller can assert the +/// `None`-means-retain half directly rather than only through the fold. +fn ours() -> Vec<(&'static str, i64, Option)> { + let page_size = 4096; + let (vfs, header) = empty_db(page_size); + let pager = Rc::new(RefCell::new( + Pager::open(&vfs, Path::new("/test.db"), page_size).unwrap(), + )); + let mut autocommit = true; + // SQLite's `db->lastRowid` starts at 0 on a fresh connection. + let mut retained: i64 = 0; + let mut out = Vec::new(); + + for sql in STATEMENTS { + let (schemas, views) = { + let borrowed = pager.borrow(); + let mut schema_cursor = TableCursor::new(&*borrowed, &header, 1); + let schemas = read_schema(&mut schema_cursor, header.text_encoding).unwrap(); + let mut view_cursor = TableCursor::new(&*borrowed, &header, 1); + let views = read_views(&mut view_cursor, header.text_encoding).unwrap(); + (schemas, views) + }; + let program = compile_any(sql, &schemas, &views); + let outcome = + execute_transaction_step_counted(&program, Rc::clone(&pager), header, autocommit) + .unwrap_or_else(|e| panic!("{sql} failed: {e}")); + autocommit = outcome.autocommit; + let reported = outcome.last_insert_rowid; + retained = reported.unwrap_or(retained); + out.push((*sql, retained, reported)); + } + out +} + +/// Runs `STATEMENTS` through the pinned oracle in a *single* invocation, +/// reading `last_insert_rowid()` after each one. +/// +/// One invocation, unlike the rows-changed diff's one-per-statement: the +/// value under test is connection-scoped and retained across statements, so +/// a fresh connection per statement would reset it to 0 and make the +/// retention claim untestable. Results are tagged with a `LIR|` marker so +/// the probe output can be separated from the output of the statements +/// themselves (one of which is a `SELECT`). +fn oracle(bin: &Path, db: &Path) -> Vec<(&'static str, i64)> { + let mut script = String::new(); + for sql in STATEMENTS { + script.push_str(sql); + script.push_str(";\nSELECT 'LIR', last_insert_rowid();\n"); + } + + let output = Command::new(bin) + .arg(db) + .arg(&script) + .output() + .unwrap_or_else(|e| panic!("oracle failed to run the script: {e}")); + assert!( + output.status.success(), + "oracle rejected the script: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let stdout = String::from_utf8_lossy(&output.stdout); + let values: Vec = stdout + .lines() + .filter_map(|line| line.strip_prefix("LIR|")) + .map(|v| { + v.trim().parse::().unwrap_or_else(|e| { + panic!("oracle's last_insert_rowid() was not a number ({e}): {v:?}") + }) + }) + .collect(); + + assert_eq!( + values.len(), + STATEMENTS.len(), + "expected one probe per statement, got {} for {} statements; stdout was {stdout:?}", + values.len(), + STATEMENTS.len() + ); + + STATEMENTS.iter().copied().zip(values).collect() +} + +#[test] +fn last_insert_rowid_matches_the_oracle() { + let Some(bin) = pinned_oracle() else { + skip_no_oracle("last_insert_rowid_matches_the_oracle"); + return; + }; + let dir = tempdir(); + let db = dir.join("last_rowid.db"); + + let mine = ours(); + let theirs = oracle(&bin, &db); + + let mine_retained: Vec<(&str, i64)> = mine.iter().map(|(sql, r, _)| (*sql, *r)).collect(); + assert_eq!( + mine_retained, theirs, + "last_insert_rowid diverges from the oracle" + ); + + // The fold above could also be satisfied by reporting `Some(previous)` + // for a non-inserting statement, which would be wrong in a way the + // comparison cannot see: a connection would then have no way to tell + // "nothing inserted" from "inserted the same rowid again". Assert the + // signal itself is `None` for every statement that inserts no row. + for (sql, _, reported) in &mine { + let inserts = sql.trim_start().to_ascii_uppercase().starts_with("INSERT"); + if inserts { + assert!( + reported.is_some(), + "{sql} inserted a row but reported no rowid" + ); + } else { + assert_eq!( + *reported, None, + "{sql} inserted nothing but claimed a last-insert rowid" + ); + } + } + + std::fs::remove_dir_all(&dir).ok(); +} + +fn tempdir() -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("sqlite-rs-last-rowid-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + dir +} diff --git a/tests/corpus/main.rs b/tests/corpus/main.rs index 885bae13..5984e1f7 100644 --- a/tests/corpus/main.rs +++ b/tests/corpus/main.rs @@ -40,6 +40,7 @@ mod index_ordered_group_by_test; mod index_ordered_scan_test; mod join_test; mod journal_interop_test; +mod last_insert_rowid_oracle_test; mod lock_state_interop_test; mod no_stats_optimizations_test; mod or_to_in_test; diff --git a/tests/unit/param_binding_test.rs b/tests/unit/param_binding_test.rs new file mode 100644 index 00000000..986ee303 --- /dev/null +++ b/tests/unit/param_binding_test.rs @@ -0,0 +1,309 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! Placeholder counting and named-parameter rejection (spec 013 +//! Requirement 3). +//! +//! An embedding consumer binds parameters, and needs two things this crate +//! did not offer: +//! +//! 1. **How many parameters a statement wants.** Without it, a caller +//! cannot be told it supplied the wrong number, and a transposed or +//! short argument list silently becomes NULLs. +//! [`Program::param_count`] answers that. +//! 2. **A refusal for the forms that don't work.** `:name`/`@name`/`$name` +//! parse but were never wired to a parameter index, and used to compile +//! to a fresh (NULL-reading) register. `WHERE x = :name` therefore +//! became `WHERE x = NULL`, matched no row, and reported no error — a +//! silent wrong answer. +//! +//! ## Numbering matches stock SQLite +//! +//! Verified against the pinned 3.53.4 source rather than from memory +//! (`src/expr.c:1317` `sqlite3ExprAssignVarNumber`): +//! +//! * a bare `?` takes `x = ++pParse->nVar` (`expr.c:1331`) — the next free +//! number, and +//! * `?nnn` takes `x = nnn` and raises the ceiling, +//! `if( x>pParse->nVar ) pParse->nVar = x` (`expr.c:1356`). +//! +//! `sqlite3_bind_parameter_count()` returns that `nVar`, so the count is +//! the *largest index used*, not the number of distinct placeholders. Our +//! `RegAlloc::anonymous_param`/`numbered_param` (`src/codegen.rs:328`) are +//! the same two rules, and `param_count` reads the ceiling back off the +//! emitted `Variable` instructions. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing +)] + +use std::cell::RefCell; +use std::collections::HashMap; +use std::path::Path; +use std::rc::Rc; + +use sqlite_rs::btree::TableCursor; +use sqlite_rs::codegen::{ + compile_select_program, compile_statement, DispatchError, PrepareError, SelectOutcome, +}; +use sqlite_rs::header::DatabaseHeader; +use sqlite_rs::pager::Pager; +use sqlite_rs::parser::error::ParseOutcome; +use sqlite_rs::parser::parse_select; +use sqlite_rs::planner::Stats; +use sqlite_rs::schema::{read_schema, read_views, TableSchema, ViewSchema}; +use sqlite_rs::vdbe::{execute_transaction_step_counted, Program}; +use sqlite_rs::vfs::MemoryVfs; + +fn empty_db(page_size: u32) -> (MemoryVfs, DatabaseHeader) { + let mut page1 = vec![0u8; page_size as usize]; + page1[0..16].copy_from_slice(b"SQLite format 3\0"); + page1[16..18].copy_from_slice(&u16::try_from(page_size).unwrap().to_be_bytes()); + page1[18] = 1; + page1[19] = 1; + page1[28..32].copy_from_slice(&1u32.to_be_bytes()); + page1[56..60].copy_from_slice(&1u32.to_be_bytes()); + page1[100] = 0x0D; + page1[105..107].copy_from_slice(&u16::try_from(page_size).unwrap().to_be_bytes()); + + let mut header_bytes = [0u8; 100]; + header_bytes.copy_from_slice(&page1[..100]); + let header = DatabaseHeader::parse(&header_bytes).unwrap(); + + let mut vfs = MemoryVfs::new(); + vfs.insert("/test.db", page1); + (vfs, header) +} + +struct Db { + pager: Rc>, + header: DatabaseHeader, + autocommit: bool, +} + +impl Db { + fn new() -> Self { + let page_size = 4096; + let (vfs, header) = empty_db(page_size); + let pager = Pager::open(&vfs, Path::new("/test.db"), page_size).unwrap(); + Self { + pager: Rc::new(RefCell::new(pager)), + header, + autocommit: true, + } + } + + fn catalog(&self) -> (Vec, Vec) { + let borrowed = self.pager.borrow(); + let mut schema_cursor = TableCursor::new(&*borrowed, &self.header, 1); + let schemas = read_schema(&mut schema_cursor, self.header.text_encoding).unwrap(); + let mut view_cursor = TableCursor::new(&*borrowed, &self.header, 1); + let views = read_views(&mut view_cursor, self.header.text_encoding).unwrap(); + (schemas, views) + } + + fn ddl(&mut self, sql: &str) { + let (schemas, views) = self.catalog(); + let program = compile_statement(sql, &schemas, &views).unwrap(); + let outcome = execute_transaction_step_counted( + &program, + Rc::clone(&self.pager), + self.header, + self.autocommit, + ) + .unwrap(); + self.autocommit = outcome.autocommit; + } + + fn select(&self, sql: &str) -> Result { + let (schemas, views) = self.catalog(); + let select = match parse_select(sql) { + ParseOutcome::Accepted(select) => *select, + ParseOutcome::Unsupported { message, .. } | ParseOutcome::Invalid { message, .. } => { + panic!("{sql} did not parse: {message}") + } + }; + let stats: HashMap = HashMap::new(); + match compile_select_program(&select, false, &schemas, &views, &stats)? { + SelectOutcome::Program(p) => Ok(p), + SelectOutcome::Eqp(_) => panic!("{sql} compiled to EXPLAIN QUERY PLAN"), + } + } + + fn write(&self, sql: &str) -> Result { + let (schemas, views) = self.catalog(); + compile_statement(sql, &schemas, &views) + } +} + +fn db_with_table() -> Db { + let mut db = Db::new(); + db.ddl("CREATE TABLE t(a INTEGER, b TEXT, c TEXT)"); + db +} + +#[test] +fn a_statement_without_placeholders_wants_no_parameters() { + let db = db_with_table(); + assert_eq!(db.select("SELECT a FROM t").unwrap().param_count(), 0); + assert_eq!( + db.write("INSERT INTO t VALUES (1, 'x', 'y')") + .unwrap() + .param_count(), + 0 + ); +} + +#[test] +fn anonymous_placeholders_are_numbered_in_order() { + let db = db_with_table(); + assert_eq!( + db.select("SELECT a FROM t WHERE a = ?") + .unwrap() + .param_count(), + 1 + ); + assert_eq!( + db.select("SELECT a FROM t WHERE a = ? AND b = ?") + .unwrap() + .param_count(), + 2 + ); + assert_eq!( + db.write("INSERT INTO t VALUES (?, ?, ?)") + .unwrap() + .param_count(), + 3 + ); +} + +/// `sqlite3_bind_parameter_count` returns the largest index, not the number +/// of distinct placeholders (`expr.c:1356`). So `?3` alone wants 3. +#[test] +fn numbered_placeholders_report_the_largest_index_not_the_count() { + let db = db_with_table(); + assert_eq!( + db.select("SELECT a FROM t WHERE a = ?3") + .unwrap() + .param_count(), + 3, + "?3 alone should want 3 parameters, matching sqlite3_bind_parameter_count" + ); +} + +/// A repeated `?NNN` is one parameter bound once and read twice — the +/// ceiling does not move. +#[test] +fn a_repeated_numbered_placeholder_is_still_one_parameter() { + let db = db_with_table(); + assert_eq!( + db.select("SELECT a FROM t WHERE a = ?1 OR b = ?1") + .unwrap() + .param_count(), + 1 + ); +} + +/// Mixed forms: a bare `?` after `?5` takes 6, because `++nVar` reads the +/// ceiling `?5` already raised (`expr.c:1331` and `expr.c:1356` together). +#[test] +fn a_bare_placeholder_after_a_numbered_one_continues_from_the_ceiling() { + let db = db_with_table(); + assert_eq!( + db.select("SELECT a FROM t WHERE a = ?5 AND b = ?") + .unwrap() + .param_count(), + 6, + "a bare ? following ?5 should take index 6" + ); +} + +/// The silent-wrong-answer fix. Each named form must be refused at compile +/// time rather than compiling to an always-NULL register. +#[test] +fn named_parameters_are_refused_rather_than_bound_to_null() { + let db = db_with_table(); + for sql in [ + "SELECT a FROM t WHERE a = :name", + "SELECT a FROM t WHERE a = @name", + "SELECT a FROM t WHERE a = $name", + ] { + let err = db + .select(sql) + .expect_err("a named parameter should not compile"); + let message = err.to_string(); + assert!( + message.contains("named parameter"), + "{sql} was refused, but not for being a named parameter: {message}" + ); + assert!( + message.contains("bind by position"), + "{sql}'s refusal should say what to do instead: {message}" + ); + } +} + +/// The refusal names the placeholder, sigil included, so a caller with a +/// large statement can find it. +#[test] +fn the_refusal_names_the_offending_placeholder() { + let db = db_with_table(); + for (sql, expected) in [ + ("SELECT a FROM t WHERE a = :tenant", ":tenant"), + ("SELECT a FROM t WHERE a = @tenant", "@tenant"), + ("SELECT a FROM t WHERE a = $tenant", "$tenant"), + ] { + let message = db.select(sql).expect_err("should be refused").to_string(); + assert!( + message.contains(expected), + "{sql}'s refusal should name {expected}: {message}" + ); + } +} + +/// A named parameter anywhere in a write statement is refused too — the +/// rejection lives in expression compilation, so it covers every statement +/// kind that compiles an expression. +#[test] +fn named_parameters_are_refused_in_writes_as_well_as_reads() { + let db = db_with_table(); + for sql in [ + "INSERT INTO t VALUES (:a, 'x', 'y')", + "UPDATE t SET b = :b WHERE a = 1", + "DELETE FROM t WHERE a = :a", + ] { + let message = db + .write(sql) + .expect_err("a named parameter should not compile") + .to_string(); + assert!( + message.contains("named parameter"), + "{sql} should be refused for being a named parameter: {message}" + ); + } +} + +/// Positional placeholders still work in every statement kind — the +/// rejection above must not have caught the supported forms with it. +#[test] +fn positional_placeholders_still_compile_everywhere() { + let db = db_with_table(); + assert_eq!( + db.write("INSERT INTO t VALUES (?, ?, ?)") + .unwrap() + .param_count(), + 3 + ); + assert_eq!( + db.write("UPDATE t SET b = ? WHERE a = ?") + .unwrap() + .param_count(), + 2 + ); + assert_eq!( + db.write("DELETE FROM t WHERE a = ?").unwrap().param_count(), + 1 + ); +} diff --git a/tests/unit/vdbe_last_insert_rowid_test.rs b/tests/unit/vdbe_last_insert_rowid_test.rs new file mode 100644 index 00000000..cb84a9fd --- /dev/null +++ b/tests/unit/vdbe_last_insert_rowid_test.rs @@ -0,0 +1,327 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! Last-inserted-rowid signal (spec 013 Requirement 1). +//! +//! `sqlite3_last_insert_rowid()` is the other half of what an embedding +//! consumer cannot work around: a row inserted into a table with a +//! surrogate key is unaddressable until the caller learns its rowid. +//! +//! Like the rows-changed counter, this is driven by a `P5` flag +//! (`OPFLAG_LASTROWID`) rather than by the opcode, and for a sharper reason. +//! An `UPDATE` rewrites a row as `Delete` + `Insert`, so an implementation +//! that hooked the `Insert` *opcode* would report the updated row's rowid +//! and quietly destroy the value a caller was about to use. Stock SQLite +//! avoids this by setting the flag only on the insert path +//! (`insert.c:2834`), and nesting the update inside the `OPFLAG_NCHANGE` +//! arm (`vdbe.c:5800`-`5803`, pinned 3.53.4). +//! +//! These tests pin the mechanism. `tests/corpus/last_insert_rowid_oracle_test.rs` +//! pins the answers against the pinned `sqlite3`. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing +)] + +use std::cell::RefCell; +use std::collections::HashMap; +use std::path::Path; +use std::rc::Rc; + +use sqlite_rs::btree::TableCursor; +use sqlite_rs::codegen::{compile_select_program, compile_statement, SelectOutcome}; +use sqlite_rs::header::DatabaseHeader; +use sqlite_rs::pager::Pager; +use sqlite_rs::parser::error::ParseOutcome; +use sqlite_rs::parser::parse_select; +use sqlite_rs::planner::Stats; +use sqlite_rs::schema::{read_schema, read_views, TableSchema, ViewSchema}; +use sqlite_rs::vdbe::{ + execute_transaction_step_counted, Opcode, Program, StepOutcome, OPFLAG_LASTROWID, +}; +use sqlite_rs::vfs::MemoryVfs; + +fn empty_db(page_size: u32) -> (MemoryVfs, DatabaseHeader) { + let mut page1 = vec![0u8; page_size as usize]; + page1[0..16].copy_from_slice(b"SQLite format 3\0"); + page1[16..18].copy_from_slice(&u16::try_from(page_size).unwrap().to_be_bytes()); + page1[18] = 1; + page1[19] = 1; + page1[28..32].copy_from_slice(&1u32.to_be_bytes()); + page1[56..60].copy_from_slice(&1u32.to_be_bytes()); + page1[100] = 0x0D; + page1[105..107].copy_from_slice(&u16::try_from(page_size).unwrap().to_be_bytes()); + + let mut header_bytes = [0u8; 100]; + header_bytes.copy_from_slice(&page1[..100]); + let header = DatabaseHeader::parse(&header_bytes).unwrap(); + + let mut vfs = MemoryVfs::new(); + vfs.insert("/test.db", page1); + (vfs, header) +} + +struct Db { + pager: Rc>, + header: DatabaseHeader, + autocommit: bool, + /// The connection-scoped value, folded the way spec 013's + /// `Connection` will: `None` retains, `Some` replaces. + retained: i64, +} + +impl Db { + fn new() -> Self { + let page_size = 4096; + let (vfs, header) = empty_db(page_size); + let pager = Pager::open(&vfs, Path::new("/test.db"), page_size).unwrap(); + Self { + pager: Rc::new(RefCell::new(pager)), + header, + autocommit: true, + retained: 0, + } + } + + fn catalog(&self) -> (Vec, Vec) { + let borrowed = self.pager.borrow(); + let mut schema_cursor = TableCursor::new(&*borrowed, &self.header, 1); + let schemas = read_schema(&mut schema_cursor, self.header.text_encoding).unwrap(); + let mut view_cursor = TableCursor::new(&*borrowed, &self.header, 1); + let views = read_views(&mut view_cursor, self.header.text_encoding).unwrap(); + (schemas, views) + } + + fn compile(&self, sql: &str) -> Program { + let (schemas, views) = self.catalog(); + if !sql.trim_start().to_ascii_uppercase().starts_with("SELECT") { + return compile_statement(sql, &schemas, &views) + .unwrap_or_else(|e| panic!("{sql} did not compile: {e}")); + } + let select = match parse_select(sql) { + ParseOutcome::Accepted(select) => *select, + ParseOutcome::Unsupported { message, .. } | ParseOutcome::Invalid { message, .. } => { + panic!("{sql} did not parse: {message}") + } + }; + let stats: HashMap = HashMap::new(); + match compile_select_program(&select, false, &schemas, &views, &stats) { + Ok(SelectOutcome::Program(p)) => p, + Ok(SelectOutcome::Eqp(_)) => panic!("{sql} compiled to EXPLAIN QUERY PLAN"), + Err(e) => panic!("{sql} did not compile: {e}"), + } + } + + fn step(&mut self, sql: &str) -> StepOutcome { + let program = self.compile(sql); + let outcome = execute_transaction_step_counted( + &program, + Rc::clone(&self.pager), + self.header, + self.autocommit, + ) + .unwrap_or_else(|e| panic!("{sql} failed: {e}")); + self.autocommit = outcome.autocommit; + self.retained = outcome.last_insert_rowid.unwrap_or(self.retained); + outcome + } + + /// The rowid this statement reported, if any. + fn reported(&mut self, sql: &str) -> Option { + self.step(sql).last_insert_rowid + } +} + +#[test] +fn an_insert_reports_the_rowid_it_used() { + let mut db = Db::new(); + db.step("CREATE TABLE t(a INTEGER, b TEXT)"); + + assert_eq!(db.reported("INSERT INTO t VALUES (1, 'x')"), Some(1)); + assert_eq!(db.reported("INSERT INTO t VALUES (2, 'y')"), Some(2)); + assert_eq!(db.reported("INSERT INTO t VALUES (3, 'z')"), Some(3)); +} + +/// The case that makes hooking `Insert` rather than `NewRowid` load-bearing. +/// +/// When the caller supplies a value for an `INTEGER PRIMARY KEY`, that value +/// *is* the rowid and `NewRowid` never executes. An implementation that read +/// the rowid off `NewRowid`'s output register would report a stale value here +/// — and this is the shape a consumer with its own surrogate keys uses. +#[test] +fn an_explicit_integer_primary_key_reports_the_supplied_value() { + let mut db = Db::new(); + db.step("CREATE TABLE u(id INTEGER PRIMARY KEY, v TEXT)"); + + assert_eq!( + db.reported("INSERT INTO u(id, v) VALUES (42, 'a')"), + Some(42) + ); + // Deliberately *lower* than the previous rowid: a stale-value bug + // cannot hide behind a monotonically increasing sequence. + assert_eq!(db.reported("INSERT INTO u(id, v) VALUES (7, 'b')"), Some(7)); +} + +/// An `UPDATE` emits an `Insert`, so this is the test that separates +/// "flagged by codegen" from "hooked on the opcode". +#[test] +fn an_update_reports_nothing_and_leaves_the_value_standing() { + let mut db = Db::new(); + db.step("CREATE TABLE t(a INTEGER, b TEXT)"); + db.step("INSERT INTO t VALUES (1, 'x')"); + db.step("INSERT INTO t VALUES (2, 'y')"); + assert_eq!(db.retained, 2); + + // Matches row 1 — an opcode-level hook would report 1 here. + assert_eq!(db.reported("UPDATE t SET b = 'z' WHERE a = 1"), None); + assert_eq!(db.retained, 2, "an UPDATE moved the last-insert rowid"); + + // And the two-pass plan (#675), which touches a scanned index. + db.step("CREATE INDEX t_a ON t(a)"); + assert_eq!(db.reported("UPDATE t SET a = a + 10 WHERE a > 0"), None); + assert_eq!(db.retained, 2, "the two-pass UPDATE plan moved the value"); +} + +#[test] +fn deletes_and_reads_report_nothing() { + let mut db = Db::new(); + db.step("CREATE TABLE t(a INTEGER, b TEXT)"); + db.step("INSERT INTO t VALUES (1, 'x')"); + db.step("INSERT INTO t VALUES (2, 'y')"); + + assert_eq!(db.reported("DELETE FROM t WHERE a = 1"), None); + assert_eq!(db.reported("SELECT a FROM t"), None); + assert_eq!(db.reported("DELETE FROM t"), None); + assert_eq!( + db.retained, 2, + "a delete or a read moved the last-insert rowid" + ); +} + +/// `None` and `Some(0)` are different answers, and so are `None` and +/// `Some(previous)`. A connection needs `None` to mean "leave the stored +/// value alone"; reporting the previous value instead would satisfy any +/// retention test while making "nothing inserted" indistinguishable from +/// "inserted the same rowid again". +#[test] +fn nothing_inserted_is_none_rather_than_the_previous_value() { + let mut db = Db::new(); + db.step("CREATE TABLE t(a INTEGER)"); + db.step("INSERT INTO t VALUES (1)"); + assert_eq!(db.retained, 1); + + let reported = db.reported("UPDATE t SET a = 9 WHERE a = 1"); + assert_eq!(reported, None); + assert_ne!( + reported, + Some(1), + "a non-inserting statement echoed the retained value instead of None" + ); +} + +/// Index maintenance writes index entries, not rows. Those go through +/// `IdxInsert`, but a table with indexes still emits exactly one flagged +/// `Insert` per row — so an indexed table must report the same rowid an +/// unindexed one does. +#[test] +fn index_maintenance_is_not_an_insert() { + let mut db = Db::new(); + db.step("CREATE TABLE t(a INTEGER, b TEXT, c TEXT)"); + db.step("CREATE INDEX t_a ON t(a)"); + db.step("CREATE UNIQUE INDEX t_c ON t(c)"); + + assert_eq!(db.reported("INSERT INTO t VALUES (1, 'b1', 'c1')"), Some(1)); + assert_eq!(db.reported("INSERT INTO t VALUES (2, 'b2', 'c2')"), Some(2)); +} + +/// Codegen's decision, asserted on the emitted program rather than only +/// through behaviour: exactly one instruction in an `INSERT` carries the +/// flag, and an `UPDATE` carries none despite emitting an `Insert`. +/// +/// This is the structural counterpart to the behavioural tests above. If a +/// future change flags the `Insert` an `UPDATE` emits, this fails at the +/// point of the mistake instead of as a surprising rowid three layers away. +#[test] +fn only_an_inserts_table_write_carries_the_flag() { + let mut db = Db::new(); + db.step("CREATE TABLE t(a INTEGER, b TEXT)"); + db.step("CREATE INDEX t_a ON t(a)"); + + let flagged = |program: &Program| -> usize { + program + .instructions + .iter() + .filter(|i| i.p5 & OPFLAG_LASTROWID != 0) + .count() + }; + + let insert = db.compile("INSERT INTO t VALUES (1, 'x')"); + assert_eq!( + flagged(&insert), + 1, + "an INSERT should flag exactly one instruction" + ); + assert!( + insert + .instructions + .iter() + .any(|i| i.p5 & OPFLAG_LASTROWID != 0 && matches!(i.opcode, Opcode::Insert)), + "the flagged instruction should be the table Insert" + ); + + let update = db.compile("UPDATE t SET b = 'z' WHERE a = 1"); + assert!( + update + .instructions + .iter() + .any(|i| matches!(i.opcode, Opcode::Insert)), + "an UPDATE is expected to emit an Insert — otherwise this test proves nothing" + ); + assert_eq!( + flagged(&update), + 0, + "an UPDATE must not flag any instruction with OPFLAG_LASTROWID" + ); + + let delete = db.compile("DELETE FROM t WHERE a = 1"); + assert_eq!( + flagged(&delete), + 0, + "a DELETE must not flag any instruction" + ); +} + +/// `OPFLAG_LASTROWID` nests inside `OPFLAG_NCHANGE` upstream +/// (`vdbe.c:5800` asserts the implication). Reproduce that as a property of +/// every program this compiler emits: no instruction may carry the rowid +/// flag without also carrying the change flag. +#[test] +fn the_rowid_flag_never_appears_without_the_change_flag() { + let mut db = Db::new(); + db.step("CREATE TABLE t(a INTEGER, b TEXT)"); + db.step("CREATE INDEX t_a ON t(a)"); + db.step("CREATE TABLE u(id INTEGER PRIMARY KEY, v TEXT)"); + + for sql in [ + "INSERT INTO t VALUES (1, 'x')", + "INSERT INTO u(id, v) VALUES (5, 'y')", + "UPDATE t SET b = 'z' WHERE a = 1", + "UPDATE t SET a = a + 1 WHERE a > 0", + "DELETE FROM t WHERE a = 1", + "DELETE FROM t", + ] { + let program = db.compile(sql); + for (pc, i) in program.instructions.iter().enumerate() { + if i.p5 & OPFLAG_LASTROWID != 0 { + assert!( + i.p5 & sqlite_rs::vdbe::OPFLAG_NCHANGE != 0, + "{sql}: instruction {pc} ({:?}) carries OPFLAG_LASTROWID without \ + OPFLAG_NCHANGE, which stock SQLite asserts cannot happen", + i.opcode + ); + } + } + } +} From d71d9f7602467ee330502f7b170e951a396b0b2f Mon Sep 17 00:00:00 2001 From: Diederik Siderius Date: Wed, 9 Sep 2026 20:08:43 +0200 Subject: [PATCH 05/14] test: prove a database we create from scratch is valid to stock sqlite3 (013/Req 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in the tree verified this before, and the gap was structural rather than an oversight. Both bootstrap-adjacent suites seed fixtures the same way — `tests/tiers/tier2.rs`'s `seed_db` and `tests/corpus/cli_write_test.rs`'s `seed_db` both read: if let Some(oracle) = pinned_oracle() { /* oracle builds the file */ } else { /* our CLI builds the file */ } They *prefer* the oracle and fall back to our own creation path only when no oracle is installed. So with an oracle present it builds the file and our path is never exercised; with it absent our path runs and there is no oracle left to check the result. The two never run together, so "a file we create is a valid SQLite database" was never actually asserted. Spec 013 makes it load-bearing: `Connection::open` creates the database if it does not exist, and a consumer's first file is one nothing else has ever touched. Four tests, all against the pinned 3.53.4: * `new_empty_page1` at all eight supported page sizes — integrity_check is `ok`, the page size round-trips, the schema is empty, and the oracle can then grow the file. 65536 is the interesting one: it cannot be stored literally in the 16-bit page-size field and is encoded as `1`, with the cell-content-area offset wrapping to 0, so it is the only branch in the function. * A file created and written entirely by our path (DDL, two index kinds, inserts, an update, a delete) is `ok` to the oracle, and the oracle reads identical schema/rows/index lookups from it and from a file it built itself from the same statements. * A byte-level header comparison — the class of bug integrity_check cannot see, since it validates b-tree structure rather than every header byte. Exactly three ranges differ and all three are fields this crate does not model, so the divergence set is asserted exhaustively: bytes 0..24 and 28..92 must match the oracle's, and 24..28, 92..96, 96..100 must be zero. The test also asserts the oracle's own copies of those fields are non-zero, so it cannot pass by comparing two sets of zeroes. * Writing to an oracle-created file preserves those unmodelled fields rather than zeroing them — which a writer that re-serialised the header from its own struct would do. One finding recorded rather than fixed: we never increment the file change counter (offset 24). It stays self-consistent with version-valid-for (offset 92), so the cached page count remains trusted and integrity_check passes — which is exactly why no existing test caught it. It is a real interop limitation, though: another SQLite connection already holding a cached image of the file has no way to learn our writes happened. A fresh connection reads from disk and sees them, which is why it is latent. Not a malformation, and out of scope here; the header test states it so a fix shows up as a failure there. Tests only — no src/ changes. Gates: make test-corpus (394 passed), make lint (both clippy passes + fmt). Spend: within estimate. Co-Authored-By: Claude Opus 5 --- tests/corpus/bootstrap_oracle_test.rs | 336 ++++++++++++++++++++++++++ tests/corpus/main.rs | 1 + 2 files changed, 337 insertions(+) create mode 100644 tests/corpus/bootstrap_oracle_test.rs diff --git a/tests/corpus/bootstrap_oracle_test.rs b/tests/corpus/bootstrap_oracle_test.rs new file mode 100644 index 00000000..70458b8f --- /dev/null +++ b/tests/corpus/bootstrap_oracle_test.rs @@ -0,0 +1,336 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! Does a database file *this crate creates from scratch* satisfy stock +//! SQLite? (spec 013 Requirement 2 prerequisite.) +//! +//! Nothing in the tree answered that before this file, and the reason is a +//! structural blind spot rather than an oversight. Both bootstrap-adjacent +//! suites seed their fixtures like this: +//! +//! ```text +//! if let Some(oracle) = pinned_oracle() { /* oracle builds the file */ } +//! else { /* our CLI builds the file */ } +//! ``` +//! +//! — `tests/tiers/tier2.rs`'s `seed_db` and `tests/corpus/cli_write_test.rs`'s +//! `seed_db`. They *prefer* the oracle and fall back to our own path only +//! when no oracle is installed. So when the oracle is present it creates +//! the file and our creation path is never exercised; when it is absent our +//! path runs but there is no oracle left to check the result. The two never +//! run together, and the claim "a file we create is a valid SQLite database" +//! went unverified for the life of the write path. +//! +//! An embedding consumer makes this load-bearing: spec 013's `Connection::open` +//! creates the database if it does not exist, and SQE's catalog is a file +//! nothing else has ever touched. If our page 1 were subtly wrong, every +//! consumer's first file would be malformed and only a third-party tool +//! would ever say so. +//! +//! `tests/unit/vdbe_integrity_check_test.rs` is not this test: it hand-rolls +//! page 1 and checks it with *our* integrity checker, which shares any +//! misconception the writer has. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing +)] + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use sqlite_rs::header::DatabaseHeader; + +use crate::oracle::{pinned_oracle, skip_no_oracle}; + +const CLI: &str = env!("CARGO_BIN_EXE_sqlite-rs"); + +fn scratch_dir(label: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "sqlite-rs-bootstrap-{}-{label}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn oracle_says(bin: &Path, db: &Path, sql: &str) -> String { + let output = Command::new(bin) + .arg(db) + .arg(sql) + .output() + .unwrap_or_else(|e| panic!("oracle failed to run {sql:?}: {e}")); + assert!( + output.status.success(), + "oracle rejected {sql:?} against {}: {}", + db.display(), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +fn our_exec(db: &Path, sql: &str) { + let output = Command::new(CLI) + .arg("exec") + .arg(db) + .arg(sql) + .output() + .unwrap_or_else(|e| panic!("running {CLI} exec {sql:?}: {e}")); + assert!( + output.status.success(), + "our CLI rejected {sql:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// `DatabaseHeader::new_empty_page1` is the bootstrap: the bytes written to +/// a brand-new file before its first statement runs. Every supported page +/// size must produce a file stock SQLite accepts as an empty database. +/// +/// 65536 is included deliberately — it is the one size that cannot be +/// stored literally in the 16-bit page-size field and is encoded as `1` +/// (and whose cell-content-area offset wraps to 0), so it exercises the +/// only branch in the function. +#[test] +fn an_empty_database_we_build_is_valid_at_every_page_size() { + let Some(bin) = pinned_oracle() else { + skip_no_oracle("an_empty_database_we_build_is_valid_at_every_page_size"); + return; + }; + let dir = scratch_dir("empty"); + + for page_size in [512u32, 1024, 2048, 4096, 8192, 16384, 32768, 65536] { + let db = dir.join(format!("empty_{page_size}.db")); + std::fs::write(&db, DatabaseHeader::new_empty_page1(page_size)).unwrap(); + + assert_eq!( + oracle_says(&bin, &db, "PRAGMA integrity_check;"), + "ok", + "page 1 built for page_size={page_size} is malformed to stock sqlite3" + ); + assert_eq!( + oracle_says(&bin, &db, "PRAGMA page_size;"), + page_size.to_string(), + "the oracle read back a different page size for page_size={page_size}" + ); + assert_eq!( + oracle_says(&bin, &db, "SELECT count(*) FROM sqlite_master;"), + "0", + "a freshly built database should have an empty schema" + ); + // And it must be *usable*, not merely well-formed: the oracle has + // to be able to grow it. + oracle_says( + &bin, + &db, + "CREATE TABLE probe(x); INSERT INTO probe VALUES (1);", + ); + assert_eq!( + oracle_says(&bin, &db, "PRAGMA integrity_check;"), + "ok", + "the oracle's own write to our page_size={page_size} file left it malformed" + ); + } + + std::fs::remove_dir_all(&dir).ok(); +} + +/// The end-to-end claim: our CLI creates a file that never existed, runs a +/// DDL + DML sequence through our write path only, and the result is a +/// valid database whose contents the oracle agrees with. +/// +/// The comparison file is built by the oracle from the identical sequence, +/// so a divergence points at our writer rather than at the SQL. +#[test] +fn a_database_we_create_from_scratch_survives_writes_and_matches_the_oracle() { + let Some(bin) = pinned_oracle() else { + skip_no_oracle("a_database_we_create_from_scratch_survives_writes_and_matches_the_oracle"); + return; + }; + let dir = scratch_dir("scratch"); + let ours = dir.join("ours.db"); + let theirs = dir.join("theirs.db"); + + // Deliberately no composite PRIMARY KEY: a declared composite PK is + // #687, still open, and would fail here for an unrelated reason. + let statements = [ + "CREATE TABLE t(a INTEGER, b TEXT)", + "CREATE INDEX t_a ON t(a)", + "CREATE UNIQUE INDEX t_b ON t(b)", + "INSERT INTO t VALUES (1, 'x')", + "INSERT INTO t VALUES (2, 'y')", + "INSERT INTO t VALUES (3, 'z')", + "UPDATE t SET b = 'q' WHERE a = 2", + "DELETE FROM t WHERE a = 1", + ]; + + for sql in statements { + our_exec(&ours, sql); + oracle_says(&bin, &theirs, &format!("{sql};")); + } + + assert_eq!( + oracle_says(&bin, &ours, "PRAGMA integrity_check;"), + "ok", + "a database built entirely by our write path is malformed to stock sqlite3" + ); + + // The oracle must also *agree with itself* about the two files: same + // schema, same rows, same index contents. + for probe in [ + "SELECT type, name, tbl_name FROM sqlite_master ORDER BY name;", + "SELECT a, b FROM t ORDER BY a;", + "SELECT a FROM t WHERE a = 3;", + "SELECT b FROM t WHERE b = 'q';", + "SELECT count(*) FROM t;", + ] { + assert_eq!( + oracle_says(&bin, &ours, probe), + oracle_says(&bin, &theirs, probe), + "the oracle read different results from our file and its own for {probe:?}" + ); + } + + std::fs::remove_dir_all(&dir).ok(); +} + +/// Byte-level comparison of a fresh header we wrote against a fresh header +/// the oracle wrote, for the same DDL. +/// +/// This is the test that would catch an omitted header field that +/// `DatabaseHeader::parse` happens to tolerate — the class of bug +/// `integrity_check` can miss, because `integrity_check` validates the +/// b-tree structure rather than every header byte. +/// +/// Exactly three ranges are expected to differ, and all three are fields +/// this crate does not model at all (they are absent from +/// `DatabaseHeader`), so they read as zero in a file we create: +/// +/// | offset | field | ours | why it is not a validity problem | +/// |---|---|---|---| +/// | 24..28 | change counter | 0 | self-consistent with 92..96, so the cached page count at 28 stays trusted | +/// | 92..96 | version-valid-for | 0 | must equal the change counter, and does | +/// | 96..100 | SQLite version | 0 | advisory; a file last written by another writer legitimately has its own value | +/// +/// The stuck change counter is a real interop limitation rather than a +/// cosmetic one — another SQLite connection that already holds a cached +/// image of this file has no way to learn our writes happened — but it is +/// not a malformation, which is precisely why it needs asserting here +/// instead of being left to `integrity_check`. Asserting the divergence set +/// *exhaustively* means fixing the counter, or drifting any other header +/// field, both show up as a failure here. +#[test] +fn our_fresh_header_differs_from_the_oracles_only_in_fields_we_do_not_model() { + let Some(bin) = pinned_oracle() else { + skip_no_oracle("our_fresh_header_differs_from_the_oracles_only_in_fields_we_do_not_model"); + return; + }; + let dir = scratch_dir("header"); + let ours = dir.join("ours.db"); + let theirs = dir.join("theirs.db"); + + let ddl = "CREATE TABLE t(a INTEGER, b TEXT)"; + our_exec(&ours, ddl); + oracle_says(&bin, &theirs, &format!("{ddl};")); + + let a = std::fs::read(&ours).unwrap(); + let b = std::fs::read(&theirs).unwrap(); + assert!(a.len() >= 100 && b.len() >= 100); + + // Everything before the change counter: magic, page size, file format + // versions, reserved space, and all three payload fractions. + assert_eq!( + &a[0..24], + &b[0..24], + "header bytes 0..24 diverge — magic, page size, format versions or payload fractions" + ); + + // Everything from the page count to the version-valid-for field: + // page count, freelist, schema cookie, schema format, cache size, + // auto-vacuum root, text encoding, user version, incremental vacuum, + // application id, and the reserved expansion space. + assert_eq!( + &a[28..92], + &b[28..92], + "header bytes 28..92 diverge — page count, freelist, schema cookie/format, \ + text encoding or one of the version/id fields" + ); + + // The three we do not model read as zero. Asserted positively so this + // test states the divergence rather than merely tolerating it. + assert_eq!( + &a[24..28], + &[0, 0, 0, 0], + "we appear to write a change counter now — update this test and the \ + interop note attached to it" + ); + assert_eq!( + &a[92..96], + &[0, 0, 0, 0], + "we appear to write version-valid-for now" + ); + assert_eq!( + &a[96..100], + &[0, 0, 0, 0], + "we appear to write a SQLite version now" + ); + + // And the oracle really does populate them, so the comparison above is + // meaningful rather than comparing two sets of zeroes. + assert_ne!( + &b[24..28], + &[0, 0, 0, 0], + "the oracle left the change counter at zero — this test proves nothing" + ); + assert_ne!( + &b[96..100], + &[0, 0, 0, 0], + "the oracle left its version number at zero — this test proves nothing" + ); + + std::fs::remove_dir_all(&dir).ok(); +} + +/// Writing to a file the *oracle* created must preserve the fields we do +/// not model rather than zeroing them — a writer that re-serialised the +/// header from its own struct would silently drop the oracle's change +/// counter and version number. +#[test] +fn writing_to_an_oracle_created_file_preserves_the_fields_we_do_not_model() { + let Some(bin) = pinned_oracle() else { + skip_no_oracle("writing_to_an_oracle_created_file_preserves_the_fields_we_do_not_model"); + return; + }; + let dir = scratch_dir("preserve"); + let db = dir.join("mixed.db"); + + oracle_says( + &bin, + &db, + "CREATE TABLE t(a INTEGER, b TEXT); INSERT INTO t VALUES (1, 'x');", + ); + let before = std::fs::read(&db).unwrap(); + let version_before = &before[96..100].to_vec(); + assert_ne!(version_before.as_slice(), &[0, 0, 0, 0]); + + our_exec(&db, "INSERT INTO t VALUES (2, 'y')"); + + let after = std::fs::read(&db).unwrap(); + assert_eq!( + &after[96..100], + version_before.as_slice(), + "our write zeroed the oracle's SQLite version number" + ); + assert_eq!( + oracle_says(&bin, &db, "PRAGMA integrity_check;"), + "ok", + "our write left an oracle-created file malformed" + ); + assert_eq!( + oracle_says(&bin, &db, "SELECT a, b FROM t ORDER BY a;"), + "1|x\n2|y" + ); + + std::fs::remove_dir_all(&dir).ok(); +} diff --git a/tests/corpus/main.rs b/tests/corpus/main.rs index 5984e1f7..04f4c4e3 100644 --- a/tests/corpus/main.rs +++ b/tests/corpus/main.rs @@ -20,6 +20,7 @@ mod oracle; mod analyze_test; mod autoindex_maintenance_test; mod begin_immediate_lock_interop_test; +mod bootstrap_oracle_test; mod btree_delete_test; mod btree_index_insert_delete_test; mod btree_insert_test; From 80a835421ee114eddf75b1eca16c81c18ed9ae56 Mon Sep 17 00:00:00 2001 From: Diederik Siderius Date: Wed, 9 Sep 2026 20:27:32 +0200 Subject: [PATCH 06/14] =?UTF-8?q?feat:=20src/api.rs=20=E2=80=94=20a=20Send?= =?UTF-8?q?=20+=20Sync=20Connection=20over=20an=20owned=20worker=20thread?= =?UTF-8?q?=20(013/Reqs=201,=202,=204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The facade spec 013 specifies, in the shape ADR-0041 decided. `Connection` with `open`/`open_with`/`open_in_memory`, `execute`/`execute_with`/ `execute_batch`, `changes`/`last_insert_rowid`, and a flat `Error` carrying SQLite result codes. Why a worker thread, precisely ------------------------------ The engine's page-source graph is `Rc`/`RefCell` by decision (ADR-0013, ADR-0017: a read path that pays no atomic refcount) and `Rc` is not `Send`. No wrapper fixes that — `Mutex` is `Send` only when `T: Send`, so wrapping the pager graph in a lock changes nothing at the type level. That leaves an `Arc` refactor of `Pager`/`PageSource`, rejected on read-path cost, or a thread that owns the engine. One further constraint makes the thread the only *expressible* design rather than merely the preferred one, and it is worth recording because neither ADR-0041 nor Jacob's proposal knew it: `make check-mvl-limit` forbids named lifetime parameters in `src/`, so no type in `src/api.rs` may hold an `Execution` — it borrows its `Program`. The execution has to live inside a single worker stack frame, which is exactly what this gives it. (`src/codegen/stmt/insert.rs::ColumnSource` already carries a comment making the same trade for the same reason.) `Connection` is `Arc`, so cloning is cheap and every clone addresses one worker. The request channel is a `SyncSender`, not a `Sender`: `Sender` is `Send` but not `Sync`, and Requirement 4 needs both. One bug found and fixed while building it, worth naming because it is invisible in review: `Drop for Shared` originally joined the worker while still holding the only sender. The worker's loop ends when `recv` fails, which needs every sender gone — so the drop waited for the thread and the thread waited for the drop. The channel is now closed explicitly first, and `worker_thread_joins_on_drop` would hang rather than fail if that regressed, which the test says out loud. Read-only mode -------------- Enforced per statement, not by the pager, and recorded as an ADR-0004 divergence: `Pager::open` calls `open_write` unconditionally, there is no read-only pager, and the read-only page source that does exist bypasses `Pager` and so merges no WAL frames — using it would silently serve stale data. The guard keys on `OpenWrite` and the DDL opcodes, *not* on `Insert`/`Delete`. Those also target ephemeral cursors: measured on this tree, `SELECT s.a FROM (SELECT a FROM t LIMIT 5) AS s` compiles to Insert x1 / OpenWrite x0 — a pure read that emits an `Insert`. A guard keyed on `Insert` would refuse it. The `LIMIT` matters too: without it the subquery is flattened and no ephemeral write is emitted, so the test that covers this says why the LIMIT must not be tidied away. The error type -------------- Flat, not wrapping: every payload is a `String`, an `i32` or a `Copy` enum. It has to be unconditionally `Send + Sync + 'static` because every error travels back over a channel, and it derives `PartialEq` so tests assert the error rather than substring-matching a message. The price is no `source()` chain, which is small — the engine's error enums barely implement it. `sqlite_code()` returns the *primary* code and `extended_sqlite_code()` the extended one, mirroring `sqlite3_errcode()`/`sqlite3_extended_errcode()`. That answers the open question in the plan (primary 19 or extended 2067?) by offering both under the names SQLite already uses, rather than picking one. Codes verified against `sqlite3.h` at the pinned 3.53.4. Also here: `From` conversions for `Value` (`i64`, `i32`, `bool`, `f64`, `&str`, `String`, `&[u8]`, `Vec`, `Option`), so binding a parameter does not require the caller to name `Arc`. Requirement 6 asks that a consumer never reach into the engine, and `Value::Text(Arc)` is a storage detail. Catalog invalidation after DDL is correctness, not caching: a program addresses tables by root page, so compiling against a stale catalog after a DROP/CREATE could read a recycled page. Same conservative rule the CLI already uses. Tests ----- 28 new tests. `tests/unit/api_connection_test.rs` (8), `api_changes_test.rs` (9) and `api_threading_test.rs` (5) use the scenario names spec 013 cites. `tests/corpus/api_oracle_test.rs` (4) diffs against the pinned 3.53.4: rows-affected counts for a twelve-statement sequence, the resulting file read back through the oracle, and parameterised writes covering all five storage classes with `typeof()` agreeing. That corpus file imports only `sqlite_rs::api` and `sqlite_rs::record::Value` — no `pager`, `vdbe`, `codegen` or `dump` — so it compiling is itself Requirement 6's no-escape-hatch claim for the write path. `handle_is_send_sync` is a `const` assertion, not a runtime check: Requirement 4 is a type-level claim. It also runs 8 threads x 25 statements through one cloned handle, and `worker_thread_joins_on_drop` does 200 sequential open/write/drop cycles on one path — a drop that returned before the join would meet its predecessor's file lock. No new dependencies: `std::sync::mpsc` only. Gates: make test (1625 passed), make test-corpus (398 passed), make lint (both clippy passes + fmt), check-mod-files, check-assurance 100%/100%. No named lifetimes, no `dyn`, no `unsafe` in src/api.rs. Spend: within estimate. Co-Authored-By: Claude Opus 5 --- Cargo.toml | 12 + src/api.rs | 929 ++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/record/value.rs | 74 +++ tests/corpus/api_oracle_test.rs | 272 +++++++++ tests/corpus/main.rs | 1 + tests/unit/api_changes_test.rs | 263 +++++++++ tests/unit/api_connection_test.rs | 212 +++++++ tests/unit/api_threading_test.rs | 181 ++++++ 9 files changed, 1945 insertions(+) create mode 100644 src/api.rs create mode 100644 tests/corpus/api_oracle_test.rs create mode 100644 tests/unit/api_changes_test.rs create mode 100644 tests/unit/api_connection_test.rs create mode 100644 tests/unit/api_threading_test.rs diff --git a/Cargo.toml b/Cargo.toml index 65c55440..83790050 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -148,6 +148,18 @@ path = "tests/unit/prepare_test.rs" name = "param_binding" path = "tests/unit/param_binding_test.rs" +[[test]] +name = "api_connection" +path = "tests/unit/api_connection_test.rs" + +[[test]] +name = "api_changes" +path = "tests/unit/api_changes_test.rs" + +[[test]] +name = "api_threading" +path = "tests/unit/api_threading_test.rs" + [[test]] name = "codegen_expr" path = "tests/unit/codegen_expr_test.rs" diff --git a/src/api.rs b/src/api.rs new file mode 100644 index 00000000..14248f26 --- /dev/null +++ b/src/api.rs @@ -0,0 +1,929 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! The embedding API: a `Connection` an application links against. +//! +//! This is the supported surface (spec 013). Everything else this crate +//! exports — `btree`, `codegen`, `pager`, `parser`, `planner`, `vdbe`, +//! `vfs` — is the engine, and a consumer should not have to name any of it. +//! +//! ## Why a worker thread +//! +//! The engine's page-source graph is `Rc`/`RefCell` by decision +//! (ADR-0013, ADR-0017: a read path that pays no atomic refcount cost), and +//! `Rc` is not `Send`. A handle a connection pool, an async task, or a +//! trait with `Send + Sync` bounds can hold therefore cannot own the engine +//! directly, and *no* wrapper fixes that: `Mutex` is `Send` only when +//! `T: Send`, so wrapping the pager graph in a lock changes nothing at the +//! type level. The two achievable designs are an `Arc`/lock refactor of +//! `Pager` and `PageSource`, which ADR-0013/ADR-0017 rejected on read-path +//! cost, or a thread that owns the engine and is spoken to over a channel. +//! ADR-0041 chose the second; `sqlx`'s own SQLite driver does the same for +//! a C `sqlite3*`. +//! +//! One constraint makes it the only *expressible* design rather than merely +//! the preferred one. `make check-mvl-limit` forbids named lifetime +//! parameters in `src/`, so no type here may hold an +//! [`Execution`](crate::vdbe::Execution) — it borrows its `Program`, and a +//! field of that type would need a lifetime. The execution has to live +//! inside a single worker stack frame, which is exactly what this design +//! gives it. +//! +//! Rows cross the channel without copying: ADR-0039 made [`Value`]'s +//! payloads `Arc`, so a `Value` is `Send + Sync` already. +//! +//! ## What is here so far +//! +//! `Connection` with [`Connection::open`], [`Connection::execute`] and the +//! statement-level counters spec 013 Requirement 1 asks for. Streaming +//! reads, prepared statements and explicit transactions are separate +//! phases; the protocol below is shaped to take them. + +use std::path::{Path, PathBuf}; +use std::sync::mpsc::{sync_channel, Receiver, SyncSender}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; + +use crate::header::{DatabaseHeader, DEFAULT_PAGE_SIZE}; +use crate::pager::Pager; +use crate::record::Value; +use crate::schema::{TableSchema, ViewSchema}; +use crate::vdbe::{Opcode, Program}; +use crate::vfs::{MemoryVfs, UnixVfs, Vfs}; + +/// SQLite primary result codes, from `sqlite3.h` at the pinned 3.53.4. +/// +/// Only the ones this module can actually produce are defined; the full +/// list is not this crate's to publish. +mod code { + /// `SQLITE_ERROR` — generic error. + pub const ERROR: i32 = 1; + /// `SQLITE_BUSY` — the database file is locked. + pub const BUSY: i32 = 5; + /// `SQLITE_READONLY` — attempt to write a read-only database. + pub const READONLY: i32 = 8; + /// `SQLITE_IOERR` — a disk I/O error. + pub const IOERR: i32 = 10; + /// `SQLITE_CORRUPT` — the database disk image is malformed. + pub const CORRUPT: i32 = 11; + /// `SQLITE_CANTOPEN` — unable to open the database file. + pub const CANTOPEN: i32 = 14; + /// `SQLITE_MISUSE` — the library was used incorrectly. + pub const MISUSE: i32 = 21; + /// `SQLITE_RANGE` — a bind index is out of range. + pub const RANGE: i32 = 25; +} + +/// How to open a database file. +/// +/// Mirrors the three modes a `sqlite://` URL can ask for, which is what a +/// consumer configures (spec 013 Requirement 2). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum OpenMode { + /// Open an existing database and refuse every statement that would + /// write to it. + /// + /// Enforced per statement rather than by the pager, and that is a + /// deliberate divergence from stock SQLite recorded under ADR-0004. + /// [`Pager::open`] calls `Vfs::open_write` unconditionally and there is + /// no read-only pager; the read-only page source that does exist + /// (`VfsPageSource`) bypasses `Pager` entirely and so merges no WAL + /// frames, which would silently serve stale data for a WAL database. + /// Refusing writes above the pager is the honest option until spec 007 + /// grows a read-only pager: the file is opened for writing, but nothing + /// this connection accepts will write to it. + ReadOnly, + /// Open an existing database for reading and writing. Fails if the file + /// does not exist, and creates nothing. + ReadWrite, + /// Open a database for reading and writing, creating a valid empty one + /// if no file exists yet. + ReadWriteCreate, +} + +/// Why an API call failed. +/// +/// Deliberately flat: every payload is a `String`, an `i32` or a `Copy` +/// enum, and layer errors arrive as already-formatted text rather than as +/// wrapped values. Two things fall out that matter more than a +/// [`source`](std::error::Error::source) chain would. +/// +/// It is unconditionally `Send + Sync + 'static`, which it *must* be — +/// every error travels back from the worker thread over a channel, so an +/// error type that borrowed from engine state could not be returned at all. +/// And it derives [`PartialEq`], so a test can assert the exact error +/// instead of substring-matching a message. +/// +/// The price is that the originating layer error is not recoverable from +/// here. That is a real loss, and a small one: the sixteen engine error +/// enums barely implement `source()` themselves, and the message they +/// format is the diagnostic. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + /// The SQL could not be parsed. + Parse { + /// What the parser objected to. + message: String, + /// 1-based line of the offending token. + line: u32, + /// 1-based column of the offending token. + column: u32, + }, + /// The statement parsed but could not be compiled. + Compile { + /// What the compiler objected to. + message: String, + }, + /// A named parameter form (`:name`, `@name`, `$name`) was used. + /// + /// Separate from [`Error::Compile`] because it is the one compile + /// failure a consumer is likely to hit by habit rather than by mistake: + /// binding by name is what most drivers do. Positional `?`/`?NNN` is + /// what this crate supports. + NamedParameter { + /// The placeholder as written, sigil included. + placeholder: String, + }, + /// The number of bound parameters did not match what the statement + /// wants. + /// + /// Stricter than stock SQLite, which leaves an unbound parameter NULL. + /// A deliberate divergence under ADR-0004: refusing to run is the safe + /// direction, and catching a transposed or short argument list is the + /// stated value of spec 013 Requirement 3. + ParamCount { + /// How many the statement reads. + expected: usize, + /// How many the caller supplied. + found: usize, + }, + /// More than one statement was given where exactly one was required. + MultipleStatements { + /// How many statements the text contained. + count: usize, + }, + /// The engine halted with a SQLite result code — a constraint + /// violation, typically. + /// + /// `code` is the *extended* code (e.g. 2067 for a UNIQUE violation); + /// [`Error::sqlite_code`] narrows it to the primary one. + Sqlite { + /// The extended SQLite result code. + code: i32, + /// The engine's message, if it supplied one. + message: String, + }, + /// The database is locked by another connection or process (spec 007's + /// `VfsError::Locked`). Retryable — see [`Error::is_retryable`]. + Busy { + /// The path that was locked. + path: String, + }, + /// The database file could not be opened. + CannotOpen { + /// The path that could not be opened. + path: String, + /// Why. + message: String, + }, + /// A write was attempted on a connection opened [`OpenMode::ReadOnly`]. + ReadOnly { + /// The statement that was refused. + statement: String, + }, + /// The database image is malformed. + Corrupt { + /// What was malformed. + message: String, + }, + /// An I/O error. + Io { + /// What failed. + message: String, + }, + /// The connection's worker thread is no longer running. + /// + /// Returned rather than blocking, per spec 013 Requirement 4. Reachable + /// two ways: the connection was closed, or the worker panicked (which + /// would be a bug in this crate). + ConnectionClosed, + /// Execution failed for a reason with no more specific variant. + Execution { + /// What went wrong. + message: String, + }, +} + +impl Error { + /// The primary SQLite result code for this error, as + /// `sqlite3_errcode()` reports it. + /// + /// Primary, not extended: `sqlite3_errcode()` returns the low byte and + /// `sqlite3_extended_errcode()` the whole word, so both are offered + /// here under the names that match. A caller switching on "is this a + /// constraint violation" wants 19; one distinguishing UNIQUE from + /// NOT NULL wants 2067 and should call + /// [`Error::extended_sqlite_code`]. + pub fn sqlite_code(&self) -> i32 { + // The extended-to-primary rule is the low byte (`sqlite3.h`: every + // extended code is `primary | (n<<8)`). + self.extended_sqlite_code() & 0xff + } + + /// The extended SQLite result code for this error, as + /// `sqlite3_extended_errcode()` reports it. + pub fn extended_sqlite_code(&self) -> i32 { + match self { + Error::Parse { .. } | Error::Compile { .. } | Error::NamedParameter { .. } => { + code::ERROR + } + Error::ParamCount { .. } => code::RANGE, + Error::MultipleStatements { .. } | Error::ConnectionClosed => code::MISUSE, + Error::Sqlite { code, .. } => *code, + Error::Busy { .. } => code::BUSY, + Error::CannotOpen { .. } => code::CANTOPEN, + Error::ReadOnly { .. } => code::READONLY, + Error::Corrupt { .. } => code::CORRUPT, + Error::Io { .. } => code::IOERR, + Error::Execution { .. } => code::ERROR, + } + } + + /// Whether retrying the same call could succeed without any change on + /// the caller's part. + /// + /// True only for [`Error::Busy`]: the lock it names is held by someone + /// else and may be released. Every other variant describes something + /// that will fail identically on a retry. + pub fn is_retryable(&self) -> bool { + matches!(self, Error::Busy { .. }) + } +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::Parse { + message, + line, + column, + } => { + write!(f, "syntax error (line {line}, column {column}): {message}") + } + Error::Compile { message } => write!(f, "cannot compile statement: {message}"), + Error::NamedParameter { placeholder } => write!( + f, + "named parameter {placeholder} is not supported — bind by position with ? or ?NNN" + ), + Error::ParamCount { expected, found } => write!( + f, + "statement wants {expected} parameter(s) but {found} were bound" + ), + Error::MultipleStatements { count } => write!( + f, + "expected a single statement but found {count} — use execute_batch" + ), + Error::Sqlite { code, message } => { + if message.is_empty() { + write!(f, "SQLite error {code}") + } else { + write!(f, "{message} (SQLite error {code})") + } + } + Error::Busy { path } => write!(f, "database is locked: {path}"), + Error::CannotOpen { path, message } => { + write!(f, "cannot open {path}: {message}") + } + Error::ReadOnly { statement } => { + write!(f, "connection is read-only; refused: {statement}") + } + Error::Corrupt { message } => write!(f, "database image is malformed: {message}"), + Error::Io { message } => write!(f, "I/O error: {message}"), + Error::ConnectionClosed => write!(f, "connection is closed"), + Error::Execution { message } => write!(f, "{message}"), + } + } +} + +impl std::error::Error for Error {} + +/// A connection to one database. +/// +/// `Send + Sync` and cheap to clone: every clone talks to the same worker +/// thread, and the thread is joined when the last clone drops. Statements +/// are serialized, which is what spec 013 Requirement 4 specifies — a +/// pointer store's throughput is irrelevant, and reachability is the point. +#[derive(Debug, Clone)] +pub struct Connection { + inner: Arc, +} + +/// The shared half of a [`Connection`], so clones address one worker. +#[derive(Debug)] +struct Shared { + /// `SyncSender` rather than `Sender` deliberately: `Sender` is + /// `Send` but not `Sync`, and a handle several threads hold at once + /// needs both. + /// + /// `Option` so [`Shared::drop`] can *close* the channel before joining. + /// This is load-bearing rather than tidy: the worker's loop ends when + /// `recv` fails, which only happens once every sender is gone, so + /// joining while still holding this one deadlocks — the drop waits for + /// a thread that is waiting for the drop. + requests: Option>, + /// Taken by [`Shared::drop`] to join the worker. `Mutex` because a + /// `JoinHandle` has to be owned to be joined, and because `Shared` is + /// reachable from several threads until the last clone goes. + worker: Mutex>>, +} + +impl Drop for Shared { + fn drop(&mut self) { + // Order matters. Dropping the sender closes the channel, so the + // worker's `recv` returns `Err` and its loop ends; only then is + // there anything to join. Field-drop order would run this *after* + // `drop`, hence the explicit `take`. + self.requests = None; + + // Then wait for it, so a caller that drops a connection and + // immediately reopens the same path cannot race its predecessor's + // file locks — the `Pager` releases those when the worker's stack + // unwinds, which has not happened yet when the channel closes. + // + // `Mutex::get_mut` rather than `lock`: this is `&mut self`, so + // there is no contention to wait on and a poisoned mutex cannot + // block the join. + if let Ok(slot) = self.worker.get_mut() { + if let Some(handle) = slot.take() { + handle.join().ok(); + } + } + } +} + +/// What the API asks the worker to do. +/// +/// Every variant carries its own reply channel, so several threads holding +/// clones of one [`Connection`] each wait on their own answer while the +/// worker serves them in arrival order. +enum Request { + /// Run exactly one statement. + Execute { + /// The statement text. + sql: String, + /// Values for its `?`/`?NNN` placeholders. + params: Vec, + /// Where to send the outcome. + reply: SyncSender>, + }, + /// Run every statement in a script, stopping at the first failure. + ExecuteBatch { + /// The script. + sql: String, + /// Where to send the outcome. + reply: SyncSender>, + }, + /// Read the connection-scoped counters. + Counters { + /// Where to send them. + reply: SyncSender, + }, +} + +/// What one [`Connection::execute`] did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Applied { + /// Rows this statement changed, or `None` if it was not a counting + /// statement. + changes: Option, +} + +/// The connection-scoped counters, retained across statements exactly as +/// `sqlite3_changes()`/`sqlite3_last_insert_rowid()` are. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +struct Counters { + changes: u64, + last_insert_rowid: i64, +} + +impl Connection { + /// Opens `path`, creating a valid empty database if no file exists. + /// + /// The default mode, matching what a `sqlite://?mode=rwc` URL + /// asks for. + pub fn open(path: impl AsRef) -> Result { + Self::open_with(path, OpenMode::ReadWriteCreate) + } + + /// Opens `path` in `mode`. + pub fn open_with(path: impl AsRef, mode: OpenMode) -> Result { + Self::spawn(Target::File(path.as_ref().to_path_buf()), mode) + } + + /// Opens a private in-memory database, discarded when the last clone of + /// this connection drops. + /// + /// Backed by `MemoryVfs`, so it exercises the same pager, journal and + /// b-tree code a file does rather than a separate code path. + pub fn open_in_memory() -> Result { + Self::spawn(Target::Memory, OpenMode::ReadWriteCreate) + } + + /// Spawns the worker and waits for it to report whether it opened. + /// + /// Opening happens *on the worker thread* because the engine state it + /// produces is not `Send` — the `Pager` cannot be built here and moved + /// there. So the outcome comes back over a channel like everything + /// else, and a failure to open leaves no thread behind. + fn spawn(target: Target, mode: OpenMode) -> Result { + let (request_tx, request_rx) = sync_channel::(0); + let (open_tx, open_rx) = sync_channel::>(0); + + let handle = std::thread::Builder::new() + .name("sqlite-rs-connection".to_string()) + .spawn(move || worker_main(target, mode, request_rx, open_tx)) + .map_err(|e| Error::Io { + message: format!("could not spawn the connection's worker thread: {e}"), + })?; + + match open_rx.recv() { + Ok(Ok(())) => Ok(Self { + inner: Arc::new(Shared { + requests: Some(request_tx), + worker: Mutex::new(Some(handle)), + }), + }), + Ok(Err(e)) => { + // The worker returns straight after reporting a failure; + // join it so no thread outlives the failed open. Safe to + // join here without closing `request_tx` first: the worker + // has already left its serve loop. + handle.join().ok(); + Err(e) + } + // The worker vanished without reporting — only reachable if it + // panicked, which is a bug here rather than a caller error. + Err(_) => { + handle.join().ok(); + Err(Error::ConnectionClosed) + } + } + } + + /// Runs one statement with no parameters, returning how many rows it + /// changed. + /// + /// Zero for a statement that is not an `INSERT`/`UPDATE`/`DELETE`; see + /// [`Connection::changes`] for the retained count, which a + /// non-counting statement deliberately leaves alone. + pub fn execute(&self, sql: &str) -> Result { + self.execute_with(sql, Vec::new()) + } + + /// Runs one statement with `params` bound to its `?`/`?NNN` + /// placeholders, 1-based, returning how many rows it changed. + /// + /// The count must match the statement's placeholder count exactly, or + /// this fails with [`Error::ParamCount`] rather than binding NULLs. + pub fn execute_with(&self, sql: &str, params: Vec) -> Result { + let (reply_tx, reply_rx) = sync_channel(0); + self.send(Request::Execute { + sql: sql.to_string(), + params, + reply: reply_tx, + })?; + let applied = self.recv(reply_rx)??; + Ok(applied.changes.unwrap_or(0)) + } + + /// Runs every statement in `sql`, stopping at the first failure. + /// + /// For schema setup, where a caller has a script rather than a + /// statement. Not a transaction: statements that already ran stay + /// applied, exactly as `sqlite3_exec` leaves them. + pub fn execute_batch(&self, sql: &str) -> Result<(), Error> { + let (reply_tx, reply_rx) = sync_channel(0); + self.send(Request::ExecuteBatch { + sql: sql.to_string(), + reply: reply_tx, + })?; + self.recv(reply_rx)? + } + + /// Rows changed by the most recent counting statement, as + /// `sqlite3_changes()` reports it. + /// + /// A statement that is not an `INSERT`/`UPDATE`/`DELETE` does not reset + /// this — so a `SELECT` after a `DELETE` of two rows still reports two. + /// That retention rule is the whole reason this is not just + /// [`Connection::execute`]'s return value. + pub fn changes(&self) -> Result { + Ok(self.counters()?.changes) + } + + /// Rowid of the most recent successful `INSERT` into a rowid table, as + /// `sqlite3_last_insert_rowid()` reports it. + /// + /// Zero if this connection has inserted nothing yet. Like + /// [`Connection::changes`], a statement that inserts nothing leaves the + /// value standing. + pub fn last_insert_rowid(&self) -> Result { + Ok(self.counters()?.last_insert_rowid) + } + + fn counters(&self) -> Result { + let (reply_tx, reply_rx) = sync_channel(0); + self.send(Request::Counters { reply: reply_tx })?; + self.recv(reply_rx) + } + + /// Hands `request` to the worker, or reports the worker is gone. + fn send(&self, request: Request) -> Result<(), Error> { + self.inner + .requests + .as_ref() + .ok_or(Error::ConnectionClosed)? + .send(request) + .map_err(|_| Error::ConnectionClosed) + } + + /// Waits for the worker's answer, or reports the worker is gone. + /// + /// The two halves are separate because either can be the one to notice: + /// `send` fails if the worker died before the request, `recv` fails if + /// it died while serving it. Both mean the same thing to a caller, and + /// neither blocks forever — which is what Requirement 4 asks for. + fn recv(&self, reply: Receiver) -> Result { + reply.recv().map_err(|_| Error::ConnectionClosed) + } +} + +/// What the worker should open. +enum Target { + /// A real file, through `UnixVfs`. + File(PathBuf), + /// A private in-memory database, through `MemoryVfs`. + Memory, +} + +/// The engine state one connection owns, and the only place it is touched. +struct Engine { + pager: std::rc::Rc>, + header: DatabaseHeader, + mode: OpenMode, + /// Threaded from each statement into the next, so a multi-statement + /// transaction is one unit (`execute_transaction_step`'s contract). + autocommit: bool, + counters: Counters, + /// The decoded catalog, reused across statements and dropped after any + /// statement that can change the schema. + /// + /// Correctness, not only speed: a prepared program addresses tables by + /// root page, so compiling against a stale catalog after a `DROP`/ + /// `CREATE` could read a recycled page. Invalidating on every possibly + /// schema-changing statement is the conservative rule the CLI already + /// uses (`src/bin/sqlite-rs/exec.rs::is_schema_changing`). + catalog: Option<(Vec, Vec)>, +} + +/// Sends a reply, tolerating a caller that has stopped waiting. +/// +/// A dropped receiver is not an error worth reporting: the caller gave up +/// — its thread unwound, or it was interrupted between sending the request +/// and reading the answer — so there is nobody to tell, and no reason for +/// the worker to stop serving this connection's other handles. +fn answer(reply: &SyncSender, value: T) { + reply.send(value).ok(); +} + +/// The worker thread's body: open, report, then serve requests until the +/// last [`Connection`] clone drops. +fn worker_main( + target: Target, + mode: OpenMode, + requests: Receiver, + open_reply: SyncSender>, +) { + let mut engine = match Engine::open(target, mode) { + Ok(engine) => { + if open_reply.send(Ok(())).is_err() { + // The caller gave up between spawning us and hearing back, + // so there is nobody to serve. Drop the engine, releasing + // its file locks. + return; + } + engine + } + Err(e) => { + answer(&open_reply, Err(e)); + return; + } + }; + // Dropped before the first request is served: `Connection::spawn` has + // its answer, and holding it would keep a channel alive for nothing. + drop(open_reply); + + while let Ok(request) = requests.recv() { + match request { + Request::Execute { sql, params, reply } => { + answer(&reply, engine.execute_one(&sql, params)); + } + Request::ExecuteBatch { sql, reply } => { + answer(&reply, engine.execute_batch(&sql)); + } + Request::Counters { reply } => { + answer(&reply, engine.counters); + } + } + } +} + +impl Engine { + fn open(target: Target, mode: OpenMode) -> Result { + let (header, pager) = match target { + Target::File(path) => Self::open_file(&path, mode)?, + Target::Memory => Self::open_memory()?, + }; + Ok(Self { + pager: std::rc::Rc::new(std::cell::RefCell::new(pager)), + header, + mode, + autocommit: true, + counters: Counters::default(), + catalog: None, + }) + } + + fn open_file(path: &Path, mode: OpenMode) -> Result<(DatabaseHeader, Pager), Error> { + let exists = UnixVfs.exists(path).map_err(|e| Error::CannotOpen { + path: path.display().to_string(), + message: e.to_string(), + })?; + + if !exists { + if mode != OpenMode::ReadWriteCreate { + // Requirement 2 is explicit that this creates nothing, so + // the check is here rather than letting `open_write` bring + // the file into existence as a side effect. + return Err(Error::CannotOpen { + path: path.display().to_string(), + message: "no such database, and this mode does not create one".to_string(), + }); + } + // Give the file a valid empty page 1 before anything tries to + // parse a header out of it — the same bootstrap the CLI's + // `exec` does, and proven against the oracle in + // `tests/corpus/bootstrap_oracle_test.rs`. + let file = UnixVfs + .create_or_open_write(path) + .map_err(|e| open_error(path, &e))?; + file.write_at(&DatabaseHeader::new_empty_page1(DEFAULT_PAGE_SIZE), 0) + .map_err(|e| open_error(path, &e))?; + } + + crate::dump::open(&UnixVfs, path).map_err(|e| open_error(path, &e)) + } + + fn open_memory() -> Result<(DatabaseHeader, Pager), Error> { + const PATH: &str = "/sqlite-rs-memory.db"; + let mut vfs = MemoryVfs::new(); + vfs.insert(PATH, DatabaseHeader::new_empty_page1(DEFAULT_PAGE_SIZE)); + crate::dump::open(&vfs, Path::new(PATH)).map_err(|e| open_error(Path::new(PATH), &e)) + } + + /// Reads the catalog, or reuses the cached decode. + fn catalog(&mut self) -> Result<&(Vec, Vec), Error> { + if self.catalog.is_none() { + let borrowed = self.pager.borrow(); + let mut cursor = crate::btree::TableCursor::new(&*borrowed, &self.header, 1); + let decoded = + crate::schema::read_schema_and_views(&mut cursor, self.header.text_encoding) + .map_err(|e| Error::Corrupt { + message: format!("cannot read the schema: {e}"), + })?; + drop(borrowed); + self.catalog = Some(decoded); + } + self.catalog.as_ref().ok_or(Error::Execution { + message: "catalog cache was empty immediately after filling it".to_string(), + }) + } + + fn execute_one(&mut self, sql: &str, params: Vec) -> Result { + let statements = crate::parser::split_statements(sql); + let count = statements.len(); + let Some(statement) = statements.into_iter().next().filter(|_| count == 1) else { + return Err(Error::MultipleStatements { count }); + }; + self.run(&statement, params) + } + + fn execute_batch(&mut self, sql: &str) -> Result<(), Error> { + for statement in crate::parser::split_statements(sql) { + self.run(&statement, Vec::new())?; + } + Ok(()) + } + + /// Compiles and runs one statement, updating the connection-scoped + /// counters and invalidating the catalog if it could have changed. + fn run(&mut self, sql: &str, params: Vec) -> Result { + let program = self.compile(sql)?; + + if self.mode == OpenMode::ReadOnly && writes(&program) { + return Err(Error::ReadOnly { + statement: sql.to_string(), + }); + } + + let wanted = program.param_count(); + if wanted != params.len() { + return Err(Error::ParamCount { + expected: wanted, + found: params.len(), + }); + } + + let outcome = self.step(&program, params)?; + + if is_schema_changing(sql) { + self.catalog = None; + } + Ok(outcome) + } + + /// Runs `program`, threading the transaction state and folding the + /// counters. + fn step(&mut self, program: &Program, params: Vec) -> Result { + let mut vm = + crate::vdbe::Vm::with_shared_writable_db(std::rc::Rc::clone(&self.pager), self.header); + vm.autocommit = self.autocommit; + vm.bind_params(params); + + let mut execution = crate::vdbe::Execution::new(vm, program); + // Rows are discarded here; `execute` reports a count, not results. + // Draining through `next_row` rather than a collecting entry point + // keeps this on the one loop ADR-0040 specifies, so the eventual + // streaming path cannot diverge from this one. + while execution.next_row().map_err(exec_error)?.is_some() {} + + self.autocommit = execution.autocommit(); + let changes = program.counts_changes().then(|| execution.changes()); + if let Some(changed) = changes { + self.counters.changes = changed; + } + if let Some(rowid) = execution.last_insert_rowid() { + self.counters.last_insert_rowid = rowid; + } + Ok(Applied { changes }) + } + + fn compile(&mut self, sql: &str) -> Result { + // One entry point for every statement kind, which is what + // `sqlite3_prepare_v2` presents and what #695's lift made possible: + // `compile_statement` answers `Unrecognized("SELECT")` for a read, + // and the SELECT pipeline needs its FROM tables resolved and its + // views and CTEs expanded first. + if is_select(sql) { + return self.compile_select(sql); + } + let (schemas, views) = self.catalog()?; + crate::codegen::compile_statement(sql, schemas, views).map_err(|e| Error::Compile { + message: e.to_string(), + }) + } + + fn compile_select(&mut self, sql: &str) -> Result { + use crate::parser::error::ParseOutcome; + + let select = match crate::parser::parse_select(sql) { + ParseOutcome::Accepted(select) => *select, + ParseOutcome::Unsupported { message, span } + | ParseOutcome::Invalid { message, span } => { + return Err(Error::Parse { + message, + line: span.line, + column: span.column, + }) + } + }; + let stats = std::collections::HashMap::new(); + let (schemas, views) = self.catalog()?; + match crate::codegen::compile_select_program(&select, false, schemas, views, &stats) { + Ok(crate::codegen::SelectOutcome::Program(program)) => Ok(program), + Ok(crate::codegen::SelectOutcome::Eqp(_)) => Err(Error::Compile { + message: "EXPLAIN QUERY PLAN has no rows to execute".to_string(), + }), + Err(e) => Err(prepare_error(&e)), + } + } +} + +/// Whether `program` can modify the database. +/// +/// Keyed on `OpenWrite` and the DDL opcodes, not on `Insert`/`Delete`. +/// Those two also target *ephemeral* cursors: a materialized FROM-subquery +/// emits an `Insert` (`src/codegen/subquery/from_clause.rs:296`) in a plain +/// `SELECT`, so keying on them would refuse read queries in +/// [`OpenMode::ReadOnly`]. `OpenWrite` is the only way to obtain a writable +/// table cursor, and every DML path emits one. +fn writes(program: &Program) -> bool { + program.instructions.iter().any(|i| { + matches!( + i.opcode, + Opcode::OpenWrite + | Opcode::CreateTable + | Opcode::DropTable + | Opcode::CreateIndex + | Opcode::DropIndex + | Opcode::CreateView + | Opcode::Analyze + | Opcode::SetJournalMode + ) + }) +} + +/// Whether `sql` should go through the `SELECT` compile pipeline. +fn is_select(sql: &str) -> bool { + let head = sql.trim_start(); + ["SELECT", "VALUES", "WITH"] + .iter() + .any(|kw| starts_with_keyword(head, kw)) +} + +/// Whether `sql` can change the `sqlite_master` catalog. +/// +/// Conservative by design: any statement starting with `CREATE`, `DROP` or +/// `ALTER` invalidates the cached catalog, even one that fails or turns out +/// to be a no-op. The cost of an unnecessary re-read is one b-tree walk; +/// the cost of a missed one is compiling against a stale root page. +fn is_schema_changing(sql: &str) -> bool { + let head = sql.trim_start(); + ["CREATE", "DROP", "ALTER"] + .iter() + .any(|kw| starts_with_keyword(head, kw)) +} + +fn starts_with_keyword(head: &str, keyword: &str) -> bool { + head.get(..keyword.len()) + .is_some_and(|h| h.eq_ignore_ascii_case(keyword)) +} + +fn open_error(path: &Path, e: &impl std::fmt::Display) -> Error { + let message = e.to_string(); + // spec 007's `VfsError::Locked` has to arrive as the busy variant even + // when it surfaces during open, since that is when a competing writer's + // lock is most likely to be met. + if message.contains("locked") { + return Error::Busy { + path: path.display().to_string(), + }; + } + Error::CannotOpen { + path: path.display().to_string(), + message, + } +} + +fn prepare_error(e: &crate::codegen::PrepareError) -> Error { + let message = e.to_string(); + if let Some(placeholder) = named_placeholder_of(&message) { + return Error::NamedParameter { placeholder }; + } + Error::Compile { message } +} + +/// Recovers the placeholder from codegen's named-parameter refusal so the +/// API can report [`Error::NamedParameter`] rather than a generic compile +/// failure. +/// +/// Reading it back out of the message is not elegant. The alternative is a +/// dedicated `CodegenError` variant, which is a change to a shared engine +/// error enum with sixteen match sites — worth doing, and worth doing on +/// its own rather than inside the facade. Recorded so the seam is visible. +fn named_placeholder_of(message: &str) -> Option { + let rest = message.strip_prefix("unsupported: named parameter ")?; + let placeholder = rest.split_whitespace().next()?; + Some(placeholder.to_string()) +} + +fn exec_error(e: crate::vdbe::ExecError) -> Error { + use crate::vdbe::ExecError; + match e { + // The engine's route for constraint violations: `Halt` carries the + // extended SQLite result code codegen chose. + ExecError::Halted { code, message } => Error::Sqlite { + code, + message: message.unwrap_or_default(), + }, + other => { + let message = other.to_string(); + if message.contains("locked") { + return Error::Busy { + path: String::new(), + }; + } + Error::Execution { message } + } + } +} diff --git a/src/lib.rs b/src/lib.rs index e339579e..c9a438fe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,6 +26,7 @@ #![deny(unsafe_code)] #![warn(missing_docs)] +pub mod api; pub mod btree; pub mod codegen; pub mod dump; diff --git a/src/record/value.rs b/src/record/value.rs index 15068acd..8e903717 100644 --- a/src/record/value.rs +++ b/src/record/value.rs @@ -29,6 +29,80 @@ pub enum Value { const fn assert_value_send_sync() {} const _: () = assert_value_send_sync::(); +/// Conversions into [`Value`], so binding a parameter does not require a +/// caller to name `Arc`. +/// +/// Spec 013 Requirement 6 asks that a consumer using only the embedding API +/// never has to reach into the engine. `Value::Text` and `Value::Blob` hold +/// `Arc` payloads (ADR-0039, so a row can cross a thread), which is an +/// implementation detail of the *storage*, not something a caller binding +/// the string `"x"` should have to construct. +/// +/// `bool` maps to `Integer(0)`/`Integer(1)` because that is what SQLite +/// stores — it has no boolean storage class. `Option` maps `None` to +/// `Null`, which is what makes a nullable column bindable without a match. +mod conversions { + use super::Value; + use std::sync::Arc; + + impl From for Value { + fn from(v: i64) -> Self { + Value::Integer(v) + } + } + + impl From for Value { + fn from(v: i32) -> Self { + Value::Integer(i64::from(v)) + } + } + + impl From for Value { + fn from(v: bool) -> Self { + Value::Integer(i64::from(v)) + } + } + + impl From for Value { + fn from(v: f64) -> Self { + Value::Real(v) + } + } + + impl From<&str> for Value { + fn from(v: &str) -> Self { + Value::Text(Arc::from(v)) + } + } + + impl From for Value { + fn from(v: String) -> Self { + Value::Text(Arc::from(v.as_str())) + } + } + + impl From<&[u8]> for Value { + fn from(v: &[u8]) -> Self { + Value::Blob(Arc::from(v)) + } + } + + impl From> for Value { + fn from(v: Vec) -> Self { + Value::Blob(Arc::from(v.as_slice())) + } + } + + impl> From> for Value { + fn from(v: Option) -> Self { + match v { + Some(inner) => inner.into(), + None => Value::Null, + } + } + } +} + /// The database's text encoding, from database header byte 56. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TextEncoding { diff --git a/tests/corpus/api_oracle_test.rs b/tests/corpus/api_oracle_test.rs new file mode 100644 index 00000000..8d813516 --- /dev/null +++ b/tests/corpus/api_oracle_test.rs @@ -0,0 +1,272 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! The embedding API against the pinned oracle (spec 013 Requirements 1 +//! and 2). +//! +//! The unit suites (`tests/unit/api_*.rs`) pin the API's behaviour against +//! itself. This pins it against the definition of correctness: a file the +//! API creates has to be a database stock `sqlite3` reads, and the +//! rows-affected counts it reports have to be the numbers `sqlite3` +//! reports for the same statements. +//! +//! Deliberately driven through `sqlite_rs::api` alone — no `pager`, no +//! `vdbe`, no `codegen`, no `dump`. That the file compiles is itself +//! Requirement 6's "the facade needs no escape hatch" for the write path. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing +)] + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use sqlite_rs::api::{Connection, OpenMode}; +use sqlite_rs::record::Value; + +use crate::oracle::{pinned_oracle, skip_no_oracle}; + +fn scratch_dir(label: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "sqlite-rs-api-oracle-{}-{label}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn oracle_says(bin: &Path, db: &Path, sql: &str) -> String { + let output = Command::new(bin) + .arg(db) + .arg(sql) + .output() + .unwrap_or_else(|e| panic!("oracle failed to run {sql:?}: {e}")); + assert!( + output.status.success(), + "oracle rejected {sql:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// Requirement 2's first scenario. +#[test] +fn create_then_oracle_reads_empty_schema() { + let Some(bin) = pinned_oracle() else { + skip_no_oracle("create_then_oracle_reads_empty_schema"); + return; + }; + let dir = scratch_dir("create"); + let db = dir.join("fresh.db"); + assert!(!db.exists()); + + // Opened and dropped without running a single statement: the file has + // to be a valid empty database on the strength of the open alone. + drop(Connection::open(&db).unwrap()); + + assert!(db.exists(), "open should have created the file"); + assert_eq!(oracle_says(&bin, &db, "PRAGMA integrity_check;"), "ok"); + assert_eq!( + oracle_says(&bin, &db, "SELECT count(*) FROM sqlite_master;"), + "0", + "a freshly created database should have an empty schema" + ); + + std::fs::remove_dir_all(&dir).ok(); +} + +/// Requirement 2's second scenario, with the oracle confirming the absence +/// rather than only our own `Path::exists`. +#[test] +fn readwrite_creates_nothing_the_oracle_can_find() { + let Some(bin) = pinned_oracle() else { + skip_no_oracle("readwrite_creates_nothing_the_oracle_can_find"); + return; + }; + let dir = scratch_dir("nocreate"); + let db = dir.join("absent.db"); + + Connection::open_with(&db, OpenMode::ReadWrite) + .expect_err("ReadWrite on a missing file must fail"); + assert!(!db.exists(), "nothing should have been written"); + + // And the oracle agrees the path holds no database — it would create + // one itself, so ask it *before* letting it near the path. + assert!( + !db.exists(), + "the failed open left a file behind: {}", + db.display() + ); + // Sanity: the oracle can create one here, so the directory was writable + // all along and the refusal was the mode, not the filesystem. + oracle_says(&bin, &db, "CREATE TABLE probe(x);"); + assert!(db.exists()); + + std::fs::remove_dir_all(&dir).ok(); +} + +/// The sequence both engines run: every statement the API compiles today, +/// chosen so the rows-affected counts cover a match, a partial match and a +/// miss, and so the file ends up with rows, indexes and deletions in it. +const STATEMENTS: &[&str] = &[ + "CREATE TABLE t(a INTEGER, b TEXT, c TEXT)", + "CREATE INDEX t_a ON t(a)", + "CREATE UNIQUE INDEX t_c ON t(c)", + "INSERT INTO t VALUES (1, 'b1', 'c1')", + "INSERT INTO t VALUES (2, 'b2', 'c2')", + "INSERT INTO t VALUES (3, 'b3', 'c3')", + "INSERT INTO t VALUES (4, 'b4', 'c4')", + "UPDATE t SET a = a + 10 WHERE a > 2", + "UPDATE t SET b = 'z' WHERE a > 2", + "UPDATE t SET b = 'q' WHERE a = 999", + "DELETE FROM t WHERE a < 3", + "DELETE FROM t WHERE a = 999", +]; + +fn is_dml(sql: &str) -> bool { + let head = sql.trim_start(); + ["INSERT", "UPDATE", "DELETE"] + .iter() + .any(|kw| head.len() >= kw.len() && head[..kw.len()].eq_ignore_ascii_case(kw)) +} + +#[test] +fn api_rows_affected_and_resulting_file_match_the_oracle() { + let Some(bin) = pinned_oracle() else { + skip_no_oracle("api_rows_affected_and_resulting_file_match_the_oracle"); + return; + }; + let dir = scratch_dir("writes"); + let ours = dir.join("ours.db"); + let theirs = dir.join("theirs.db"); + + let conn = Connection::open(&ours).unwrap(); + let mut mine = Vec::new(); + for sql in STATEMENTS { + let changed = conn + .execute(sql) + .unwrap_or_else(|e| panic!("{sql} failed through the API: {e}")); + if is_dml(sql) { + mine.push((*sql, changed)); + } + } + // Release the worker (and its file locks) before handing the file over. + drop(conn); + + // One oracle invocation per statement, with `changes()` appended: + // `changes()` is per-connection, so a fresh invocation reports that + // statement's own count. + let mut theirs_counts = Vec::new(); + for sql in STATEMENTS { + let script = if is_dml(sql) { + format!("{sql};\nSELECT changes();") + } else { + format!("{sql};") + }; + let out = oracle_says(&bin, &theirs, &script); + if is_dml(sql) { + let count = out.trim().parse::().unwrap_or_else(|e| { + panic!("oracle's changes() after {sql} was not a number ({e}): {out:?}") + }); + theirs_counts.push((*sql, count)); + } + } + + assert_eq!( + mine, theirs_counts, + "rows-affected counts diverge between the API and the oracle" + ); + + // And the file the API produced is one the oracle reads identically. + assert_eq!(oracle_says(&bin, &ours, "PRAGMA integrity_check;"), "ok"); + for probe in [ + "SELECT type, name FROM sqlite_master ORDER BY name;", + "SELECT a, b, c FROM t ORDER BY a;", + "SELECT count(*) FROM t;", + ] { + assert_eq!( + oracle_says(&bin, &ours, probe), + oracle_says(&bin, &theirs, probe), + "the oracle read different results from the API's file and its own for {probe:?}" + ); + } + + std::fs::remove_dir_all(&dir).ok(); +} + +/// Parameterised writes through the API produce the same file as the +/// oracle running the same statements with the values inlined. +/// +/// Six of *SQE*'s eight statements are parameterised writes, and this is +/// the combination the engine had no entry point for at all before the +/// facade: `execute_with_db_and_params` is read-only, and +/// `execute_transaction_step` takes no parameters. +#[test] +fn parameterised_writes_match_the_oracle() { + let Some(bin) = pinned_oracle() else { + skip_no_oracle("parameterised_writes_match_the_oracle"); + return; + }; + let dir = scratch_dir("params"); + let ours = dir.join("ours.db"); + let theirs = dir.join("theirs.db"); + + let conn = Connection::open(&ours).unwrap(); + conn.execute("CREATE TABLE t(a INTEGER, b TEXT, r REAL, d BLOB)") + .unwrap(); + assert_eq!( + conn.execute_with( + "INSERT INTO t VALUES (?1, ?2, ?3, ?4)", + vec![ + Value::from(1), + Value::from("hello"), + Value::from(2.5), + Value::from(vec![0xde_u8, 0xad, 0xbe, 0xef]), + ], + ) + .unwrap(), + 1 + ); + assert_eq!( + conn.execute_with( + "INSERT INTO t VALUES (?1, ?2, ?3, ?4)", + vec![Value::from(2), Value::Null, Value::from(-0.5), Value::Null], + ) + .unwrap(), + 1 + ); + assert_eq!( + conn.execute_with( + "UPDATE t SET b = ?1 WHERE a = ?2", + vec![Value::from("updated"), Value::from(1)], + ) + .unwrap(), + 1 + ); + drop(conn); + + oracle_says( + &bin, + &theirs, + "CREATE TABLE t(a INTEGER, b TEXT, r REAL, d BLOB); + INSERT INTO t VALUES (1, 'hello', 2.5, x'deadbeef'); + INSERT INTO t VALUES (2, NULL, -0.5, NULL); + UPDATE t SET b = 'updated' WHERE a = 1;", + ); + + assert_eq!(oracle_says(&bin, &ours, "PRAGMA integrity_check;"), "ok"); + for probe in [ + "SELECT a, b, r, quote(d) FROM t ORDER BY a;", + "SELECT typeof(a), typeof(b), typeof(r), typeof(d) FROM t ORDER BY a;", + ] { + assert_eq!( + oracle_says(&bin, &ours, probe), + oracle_says(&bin, &theirs, probe), + "bound values round-tripped differently for {probe:?}" + ); + } + + std::fs::remove_dir_all(&dir).ok(); +} diff --git a/tests/corpus/main.rs b/tests/corpus/main.rs index 04f4c4e3..7b7af5aa 100644 --- a/tests/corpus/main.rs +++ b/tests/corpus/main.rs @@ -18,6 +18,7 @@ mod harness; mod oracle; mod analyze_test; +mod api_oracle_test; mod autoindex_maintenance_test; mod begin_immediate_lock_interop_test; mod bootstrap_oracle_test; diff --git a/tests/unit/api_changes_test.rs b/tests/unit/api_changes_test.rs new file mode 100644 index 00000000..50298cf5 --- /dev/null +++ b/tests/unit/api_changes_test.rs @@ -0,0 +1,263 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! The rows-affected count, at the connection level (spec 013 +//! Requirement 1). +//! +//! Spec 013 calls this the one item on its list a consumer cannot work +//! around. The engine half is `StepOutcome::changes` (#692); what this +//! covers is the *connection*'s rule, which is the half with the surprising +//! semantics: `sqlite3_changes()` is not "what the last statement did", it +//! is "what the last *counting* statement did". A `SELECT` in between must +//! leave it alone. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use sqlite_rs::api::{Connection, Error}; +use sqlite_rs::record::Value; + +fn seeded() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE t(id INTEGER, metadata_location TEXT); + INSERT INTO t VALUES (1, 'a'); + INSERT INTO t VALUES (2, 'a');", + ) + .unwrap(); + conn +} + +/// The optimistic-concurrency case, and the reason the requirement exists: +/// *SQE* swaps a table's metadata pointer with a conditional `UPDATE` and +/// treats zero rows affected as a lost race. +#[test] +fn conditional_update_reports_match() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE t(id INTEGER, metadata_location TEXT); + INSERT INTO t VALUES (1, 'a');", + ) + .unwrap(); + + let swap = "UPDATE t SET metadata_location = 'b' WHERE metadata_location = 'a'"; + + // First swap wins. + assert_eq!(conn.execute(swap).unwrap(), 1); + assert_eq!(conn.changes().unwrap(), 1); + + // The identical statement now matches nothing — a lost race, and it has + // to be distinguishable from the win above. + assert_eq!(conn.execute(swap).unwrap(), 0); + assert_eq!(conn.changes().unwrap(), 0); +} + +/// `Some(0)` and "not a counting statement" are different answers, and this +/// is where the difference shows. +#[test] +fn select_does_not_clobber_count() { + let conn = seeded(); + + assert_eq!(conn.execute("DELETE FROM t").unwrap(), 2); + assert_eq!(conn.changes().unwrap(), 2); + + // A SELECT that returns no rows at all must not reset the count. + conn.execute("SELECT id FROM t").unwrap(); + assert_eq!( + conn.changes().unwrap(), + 2, + "a SELECT clobbered the rows-changed count" + ); + + // Nor does DDL. + conn.execute("CREATE TABLE u(x)").unwrap(); + assert_eq!( + conn.changes().unwrap(), + 2, + "DDL clobbered the rows-changed count" + ); +} + +#[test] +fn insert_and_delete_report_their_rows() { + let conn = seeded(); + assert_eq!(conn.execute("INSERT INTO t VALUES (3, 'c')").unwrap(), 1); + assert_eq!(conn.execute("DELETE FROM t WHERE id < 3").unwrap(), 2); + assert_eq!(conn.execute("DELETE FROM t").unwrap(), 1); + assert_eq!(conn.execute("DELETE FROM t").unwrap(), 0); +} + +/// Index maintenance is not a row change: the same statement against the +/// same table must report the same number whether or not indexes exist. +#[test] +fn indexes_do_not_inflate_the_count() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE t(a INTEGER, b TEXT, c TEXT); + CREATE INDEX t_a ON t(a); + CREATE UNIQUE INDEX t_c ON t(c);", + ) + .unwrap(); + + assert_eq!( + conn.execute("INSERT INTO t VALUES (1, 'b', 'c')").unwrap(), + 1 + ); + assert_eq!(conn.execute("UPDATE t SET b = 'z' WHERE a = 1").unwrap(), 1); + assert_eq!(conn.execute("DELETE FROM t WHERE a = 1").unwrap(), 1); +} + +#[test] +fn last_insert_rowid_is_retained_across_statements() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute("CREATE TABLE u(id INTEGER PRIMARY KEY, v TEXT)") + .unwrap(); + assert_eq!( + conn.last_insert_rowid().unwrap(), + 0, + "a connection that has inserted nothing should report 0" + ); + + conn.execute("INSERT INTO u(id, v) VALUES (42, 'a')") + .unwrap(); + assert_eq!(conn.last_insert_rowid().unwrap(), 42); + + // Neither an UPDATE, a DELETE nor a SELECT may move it. + conn.execute("UPDATE u SET v = 'b' WHERE id = 42").unwrap(); + assert_eq!(conn.last_insert_rowid().unwrap(), 42); + conn.execute("DELETE FROM u WHERE id = 42").unwrap(); + assert_eq!(conn.last_insert_rowid().unwrap(), 42); + conn.execute("SELECT id FROM u").unwrap(); + assert_eq!(conn.last_insert_rowid().unwrap(), 42); +} + +/// Bound parameters on a write — the shape six of *SQE*'s eight statements +/// take, and the one combination the engine had no entry point for before +/// this facade (`execute_with_db_and_params` is read-only; +/// `execute_transaction_step` takes no parameters). +#[test] +fn a_parameterised_write_binds_and_counts() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute("CREATE TABLE t(a INTEGER, b TEXT)").unwrap(); + + assert_eq!( + conn.execute_with( + "INSERT INTO t VALUES (?1, ?2)", + vec![Value::Integer(1), Value::from("x")], + ) + .unwrap(), + 1 + ); + assert_eq!( + conn.execute_with( + "UPDATE t SET b = ?1 WHERE a = ?2", + vec![Value::from("y"), Value::Integer(1)], + ) + .unwrap(), + 1 + ); + // A miss reports zero rather than failing. + assert_eq!( + conn.execute_with( + "UPDATE t SET b = ?1 WHERE a = ?2", + vec![Value::from("z"), Value::Integer(999)], + ) + .unwrap(), + 0 + ); + assert_eq!( + conn.execute_with("DELETE FROM t WHERE a = ?1", vec![Value::Integer(1)]) + .unwrap(), + 1 + ); +} + +/// Stricter than stock SQLite, which leaves an unbound parameter NULL. The +/// divergence is the point: a short or transposed argument list is what +/// Requirement 3 exists to catch. +#[test] +fn the_wrong_number_of_parameters_is_refused() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute("CREATE TABLE t(a INTEGER, b TEXT)").unwrap(); + + let too_few = conn + .execute_with("INSERT INTO t VALUES (?1, ?2)", vec![Value::Integer(1)]) + .expect_err("two placeholders, one value"); + assert_eq!( + too_few, + Error::ParamCount { + expected: 2, + found: 1 + } + ); + assert_eq!(too_few.sqlite_code(), 25, "should report SQLITE_RANGE"); + + let too_many = conn + .execute_with( + "INSERT INTO t VALUES (?1, ?2)", + vec![Value::Integer(1), Value::from("x"), Value::Integer(3)], + ) + .expect_err("two placeholders, three values"); + assert_eq!( + too_many, + Error::ParamCount { + expected: 2, + found: 3 + } + ); + + // Nothing was written by either attempt. + conn.execute("DELETE FROM t").unwrap(); + assert_eq!(conn.changes().unwrap(), 0); +} + +#[test] +fn a_named_parameter_is_refused_with_its_own_variant() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute("CREATE TABLE t(a INTEGER)").unwrap(); + + let err = conn + .execute("SELECT a FROM t WHERE a = :id") + .expect_err("named parameters are not supported"); + assert_eq!( + err, + Error::NamedParameter { + placeholder: ":id".to_string() + }, + "should report the placeholder, not a generic compile failure" + ); + assert_eq!(err.sqlite_code(), 1, "should report SQLITE_ERROR"); +} + +/// A UNIQUE violation has to arrive with SQLite's own result code, so a +/// consumer can tell it from any other failure without matching on text. +#[test] +fn a_constraint_violation_carries_the_sqlite_result_code() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE t(a INTEGER, c TEXT); + CREATE UNIQUE INDEX t_c ON t(c); + INSERT INTO t VALUES (1, 'dup');", + ) + .unwrap(); + + let err = conn + .execute("INSERT INTO t VALUES (2, 'dup')") + .expect_err("duplicate value in a UNIQUE index"); + + match err { + Error::Sqlite { code, .. } => { + // SQLITE_CONSTRAINT_UNIQUE = 19 | (8<<8), sqlite3.h at 3.53.4. + assert_eq!(code, 2067, "expected SQLITE_CONSTRAINT_UNIQUE"); + } + other => panic!("expected Error::Sqlite, got {other:?}"), + } + assert_eq!( + err.sqlite_code(), + 19, + "the primary code should be SQLITE_CONSTRAINT" + ); + assert_eq!(err.extended_sqlite_code(), 2067); + assert!( + !err.is_retryable(), + "a constraint violation is not retryable" + ); +} diff --git a/tests/unit/api_connection_test.rs b/tests/unit/api_connection_test.rs new file mode 100644 index 00000000..500c5eb7 --- /dev/null +++ b/tests/unit/api_connection_test.rs @@ -0,0 +1,212 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! Opening and creating a connection (spec 013 Requirement 2). + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::path::{Path, PathBuf}; + +use sqlite_rs::api::{Connection, Error, OpenMode}; + +fn scratch(label: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("sqlite-rs-api-conn-{}-{label}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + dir.join("test.db") +} + +fn clean(path: &Path) { + if let Some(dir) = path.parent() { + std::fs::remove_dir_all(dir).ok(); + } +} + +#[test] +fn open_creates_a_database_that_reopens_and_reads_back() { + let path = scratch("create"); + clean(&path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + assert!(!path.exists()); + + { + let conn = Connection::open(&path).unwrap(); + conn.execute("CREATE TABLE t(a INTEGER, b TEXT)").unwrap(); + conn.execute("INSERT INTO t VALUES (1, 'x')").unwrap(); + } + assert!(path.exists(), "open should have created the file"); + + // Reopening is the real check: the header we wrote has to parse, and + // the catalog we wrote has to decode. + let conn = Connection::open(&path).unwrap(); + assert_eq!(conn.execute("INSERT INTO t VALUES (2, 'y')").unwrap(), 1); + + clean(&path); +} + +/// Requirement 2 is explicit that this creates nothing — so the assertion +/// is on the filesystem, not only on the error. +#[test] +fn readwrite_does_not_create() { + let path = scratch("no-create"); + clean(&path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + assert!(!path.exists()); + + let err = Connection::open_with(&path, OpenMode::ReadWrite) + .expect_err("ReadWrite on a missing file should fail"); + + assert!( + matches!(err, Error::CannotOpen { .. }), + "expected CannotOpen, got {err:?}" + ); + assert_eq!(err.sqlite_code(), 14, "should report SQLITE_CANTOPEN"); + assert!(!err.is_retryable()); + assert!( + !path.exists(), + "ReadWrite must not bring the file into existence" + ); + + clean(&path); +} + +#[test] +fn readonly_reads_but_refuses_every_write() { + let path = scratch("readonly"); + clean(&path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + + { + let conn = Connection::open(&path).unwrap(); + conn.execute("CREATE TABLE t(a INTEGER, b TEXT)").unwrap(); + conn.execute("INSERT INTO t VALUES (1, 'x')").unwrap(); + } + + let conn = Connection::open_with(&path, OpenMode::ReadOnly).unwrap(); + + // Reads work, including one whose plan materializes a FROM-subquery. + // That emits an `Insert` against an *ephemeral* cursor, so a read-only + // guard keyed on `Insert` rather than `OpenWrite` would wrongly refuse + // a plain `SELECT`. This is the case that pins the discriminator. + // + // The `LIMIT 5` is load-bearing and must not be "simplified" away: + // without it `flatten_from_subqueries` folds the subquery into the + // outer query and no ephemeral write is emitted at all. Measured on + // this tree: with the LIMIT the program has Insert x1 / OpenWrite x0; + // without it, Insert x0. Remove the LIMIT and this test still passes + // while proving nothing. + conn.execute("SELECT a FROM t").unwrap(); + conn.execute("SELECT s.a FROM (SELECT a FROM t LIMIT 5) AS s ORDER BY s.a") + .unwrap(); + + for sql in [ + "INSERT INTO t VALUES (2, 'y')", + "UPDATE t SET b = 'z' WHERE a = 1", + "DELETE FROM t WHERE a = 1", + "CREATE TABLE u(x)", + "CREATE INDEX t_a ON t(a)", + "DROP TABLE t", + ] { + match conn.execute(sql) { + Err(Error::ReadOnly { statement }) => assert_eq!(statement, sql), + Err(other) => panic!("{sql} was refused, but as {other:?} not ReadOnly"), + Ok(changed) => { + panic!("{sql} was allowed on a read-only connection (changed {changed} rows)") + } + } + } + + clean(&path); +} + +#[test] +fn readonly_refusal_reports_sqlite_readonly() { + let path = scratch("readonly-code"); + clean(&path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + { + let conn = Connection::open(&path).unwrap(); + conn.execute("CREATE TABLE t(a INTEGER)").unwrap(); + } + + let conn = Connection::open_with(&path, OpenMode::ReadOnly).unwrap(); + let err = conn + .execute("INSERT INTO t VALUES (1)") + .expect_err("refused"); + assert!( + matches!(err, Error::ReadOnly { .. }), + "expected ReadOnly, got {err:?}" + ); + assert_eq!(err.sqlite_code(), 8, "should report SQLITE_READONLY"); + assert!(!err.is_retryable()); + + // And nothing was written. + let conn = Connection::open_with(&path, OpenMode::ReadWrite).unwrap(); + assert_eq!(conn.execute("DELETE FROM t").unwrap(), 0); + + clean(&path); +} + +#[test] +fn an_in_memory_database_works_and_is_private() { + let a = Connection::open_in_memory().unwrap(); + a.execute("CREATE TABLE t(a INTEGER)").unwrap(); + assert_eq!(a.execute("INSERT INTO t VALUES (1)").unwrap(), 1); + + // A second in-memory connection is a different database, not a shared + // one — so `t` must not exist in it. + let b = Connection::open_in_memory().unwrap(); + assert!( + b.execute("INSERT INTO t VALUES (1)").is_err(), + "in-memory databases should be private to their connection" + ); +} + +#[test] +fn a_multi_statement_string_is_refused_by_execute() { + let conn = Connection::open_in_memory().unwrap(); + let err = conn + .execute("CREATE TABLE t(a INTEGER); CREATE TABLE u(b INTEGER)") + .expect_err("execute takes exactly one statement"); + assert_eq!(err, Error::MultipleStatements { count: 2 }); + assert_eq!(err.sqlite_code(), 21, "should report SQLITE_MISUSE"); + + // ...and execute_batch is the way to run it. + conn.execute_batch("CREATE TABLE t(a INTEGER); CREATE TABLE u(b INTEGER)") + .unwrap(); + conn.execute("INSERT INTO t VALUES (1)").unwrap(); + conn.execute("INSERT INTO u VALUES (2)").unwrap(); +} + +#[test] +fn a_syntax_error_reports_where_it_is() { + let conn = Connection::open_in_memory().unwrap(); + let err = conn.execute("SELECT FROM").expect_err("not valid SQL"); + match err { + Error::Parse { line, column, .. } => { + assert_eq!(line, 1); + assert!(column > 0, "column should be 1-based, got {column}"); + } + other => panic!("expected a Parse error, got {other:?}"), + } +} + +#[test] +fn a_batch_stops_at_the_first_failure() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute("CREATE TABLE t(a INTEGER)").unwrap(); + + let err = conn + .execute_batch( + "INSERT INTO t VALUES (1); INSERT INTO nope VALUES (2); INSERT INTO t VALUES (3)", + ) + .expect_err("the middle statement targets no table"); + assert!(matches!(err, Error::Compile { .. }), "got {err:?}"); + + // The statement before the failure stayed applied — execute_batch is + // not a transaction, and says so. + conn.execute("DELETE FROM t WHERE a = 1").unwrap(); + assert_eq!(conn.changes().unwrap(), 1); + // The one after it never ran. + conn.execute("DELETE FROM t WHERE a = 3").unwrap(); + assert_eq!(conn.changes().unwrap(), 0); +} diff --git a/tests/unit/api_threading_test.rs b/tests/unit/api_threading_test.rs new file mode 100644 index 00000000..bebf3655 --- /dev/null +++ b/tests/unit/api_threading_test.rs @@ -0,0 +1,181 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! The `Send + Sync` handle and its worker thread (spec 013 +//! Requirement 4). +//! +//! The engine is `Rc`/`RefCell` by decision (ADR-0013, ADR-0017) and `Rc` +//! is not `Send`, so a handle that a pool or an async task can hold cannot +//! own the engine directly. No wrapper fixes that either: `Mutex` is +//! `Send` only when `T: Send`. The connection therefore owns a thread and +//! is spoken to over a channel (ADR-0041). +//! +//! What these tests pin is the contract that makes the design usable rather +//! than merely type-correct: the handle really is `Send + Sync`, several +//! threads really can share one, and the thread really goes away on drop. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use sqlite_rs::api::Connection; +use sqlite_rs::record::Value; + +/// Compile-time proof, not a runtime check. If `Connection` ever stops +/// being `Send + Sync` this fails to build, which is the point — +/// Requirement 4 is a type-level claim and deserves a type-level test. +const fn assert_send_sync() {} +const _: () = assert_send_sync::(); +const _: () = assert_send_sync::(); + +fn scratch(label: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "sqlite-rs-api-thread-{}-{label}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir.join("test.db") +} + +fn clean(path: &Path) { + if let Some(dir) = path.parent() { + std::fs::remove_dir_all(dir).ok(); + } +} + +#[test] +fn handle_is_send_sync() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE t(a INTEGER, b TEXT); + INSERT INTO t VALUES (1, 'x'); + INSERT INTO t VALUES (2, 'y');", + ) + .unwrap(); + + // Cloned into several threads, each running statements concurrently. + // The worker serializes them; the test is that every one succeeds and + // none deadlocks. + let mut handles = Vec::new(); + for worker in 0..8 { + let conn = conn.clone(); + handles.push(std::thread::spawn(move || { + for i in 0..25 { + conn.execute("SELECT a FROM t").unwrap(); + conn.execute_with( + "INSERT INTO t VALUES (?1, ?2)", + vec![Value::from(1000 + worker * 100 + i), Value::from("z")], + ) + .unwrap(); + } + })); + } + for handle in handles { + handle.join().expect("a worker thread panicked"); + } + + // 2 seeded + 8 threads x 25 inserts. + assert_eq!(conn.execute("DELETE FROM t").unwrap(), 202); +} + +/// A `&Connection` shared through an `Arc` rather than cloned — the shape a +/// trait object or a pool hands out, and the one that needs `Sync` rather +/// than only `Send`. +#[test] +fn a_shared_reference_works_across_threads() { + let conn = Arc::new(Connection::open_in_memory().unwrap()); + conn.execute("CREATE TABLE t(a INTEGER)").unwrap(); + + let mut handles = Vec::new(); + for i in 0..4 { + let conn = Arc::clone(&conn); + handles.push(std::thread::spawn(move || { + conn.execute_with("INSERT INTO t VALUES (?1)", vec![Value::from(i)]) + .unwrap(); + })); + } + for handle in handles { + handle.join().expect("a worker thread panicked"); + } + assert_eq!(conn.execute("DELETE FROM t").unwrap(), 4); +} + +/// The thread is released on drop, and this test would *hang* rather than +/// fail if it were not. +/// +/// That is deliberate and worth stating: `Drop for Shared` closes the +/// request channel and then joins. If the worker did not terminate, the +/// join would block forever and this test would time out — so completing at +/// all is the assertion. (An earlier draft of the facade had exactly that +/// bug: it joined while still holding the only sender, so the worker waited +/// on a channel that could never close while the drop waited on the +/// worker.) +/// +/// The observable consequence is checked too. A `Pager` releases its file +/// locks when the worker's stack unwinds, so if drop returned before the +/// join the next connection could meet its predecessor's SHARED lock. Two +/// hundred sequential open/write/drop cycles on one path would surface that. +#[test] +fn worker_thread_joins_on_drop() { + let path = scratch("join"); + clean(&path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + + { + let conn = Connection::open(&path).unwrap(); + conn.execute("CREATE TABLE t(a INTEGER)").unwrap(); + } + + for i in 0..200 { + let conn = Connection::open(&path).unwrap(); + conn.execute_with("INSERT INTO t VALUES (?1)", vec![Value::from(i)]) + .unwrap(); + // Dropped here, at the end of each iteration. + } + + let conn = Connection::open(&path).unwrap(); + assert_eq!( + conn.execute("DELETE FROM t").unwrap(), + 200, + "every cycle's write should have committed and been visible to the next" + ); + + clean(&path); +} + +/// Dropping one clone must not close the connection for the others. +#[test] +fn dropping_one_clone_leaves_the_rest_working() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute("CREATE TABLE t(a INTEGER)").unwrap(); + + let clone_a = conn.clone(); + let clone_b = conn.clone(); + drop(conn); + drop(clone_a); + + // The last clone still owns a live worker. + assert_eq!(clone_b.execute("INSERT INTO t VALUES (1)").unwrap(), 1); + assert_eq!(clone_b.changes().unwrap(), 1); +} + +/// The counters are connection state, not thread-local state — a value set +/// on one thread is readable from another through the same handle. +#[test] +fn counters_are_connection_scoped_not_thread_scoped() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute("CREATE TABLE u(id INTEGER PRIMARY KEY, v TEXT)") + .unwrap(); + + let writer = conn.clone(); + std::thread::spawn(move || { + writer + .execute("INSERT INTO u(id, v) VALUES (7, 'a')") + .unwrap(); + }) + .join() + .expect("the writing thread panicked"); + + assert_eq!(conn.changes().unwrap(), 1); + assert_eq!(conn.last_insert_rowid().unwrap(), 7); +} From 09c29ffcc6592e06d3738efb69565a429c6b9865 Mon Sep 17 00:00:00 2001 From: Diederik Siderius Date: Wed, 9 Sep 2026 20:45:58 +0200 Subject: [PATCH 07/14] =?UTF-8?q?feat:=20streaming=20reads=20on=20the=20em?= =?UTF-8?q?bedding=20API=20=E2=80=94=20query,=20Rows,=20Row,=20FromValue?= =?UTF-8?q?=20(013/Reqs=207,=203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rows arrive from the worker in batches, so peak memory is bounded by the batch size and the pager's page cache rather than by the result. Spike 014 (#682) measured the alternative at 137.7 MB and 5.36 ms to first row for 1,000,000 rows, against 8.68 MB and 44.7 us streamed. Built on `Execution::next_row`, inside one worker stack frame — the only place it can live, since `Execution` borrows its `Program` and no type in `src/api.rs` may carry a lifetime. Two findings worth recording, both from tests that failed first -------------------------------------------------------------- **ORDER BY is a blocking operator, and the first draft of `partial_read_is_bounded` measured it.** It used `SELECT a FROM t ORDER BY a` and failed at 33x on a 50x larger table — correctly. A sort with no usable index has to consume every row before emitting the first, so time-to-first-row is genuinely linear there and no amount of streaming changes it; stock SQLite sorts the same way. The streaming property has to be measured on a plan that can emit as it scans. The test now uses a bare scan and says why in full, and `blocking_plans_are_linear_by_nature` pins the contrast so the limit is recorded rather than quietly avoided. **An unread `Rows` could park the worker, and my own test deadlocked on it.** `joins_and_compounds_report_positional_column_names` held one result alive while issuing a second query; the worker was blocked sending the first result's `Done` into a full one-slot channel, so it never returned to serve. Two changes came out of that: * the result channel now holds two batches, not one (`CHUNK_SLOTS`), so any result that fits in a single batch completes and frees the worker whether or not the caller ever reads it. That covers the accident that is easy to have — `let rows = conn.query(..)` and then forget it — and `an_unread_small_result_does_not_block_the_next_statement` would hang rather than fail if it regressed. * a larger unread result still parks the worker until the handle is read or dropped. That is inherent to one worker streaming one execution, so it is documented on `Rows` with the concrete failing snippet, and `dropping_a_large_unread_result_releases_the_connection` pins that dropping is sufficient. Design points ------------- `Rows` is deliberately not an `Iterator`: collapsing `Result, Error>` into `Option>` to fit the trait makes "ended" and "failed" the same shape at the call site. ADR-0040 rejected an `Iterator` impl on `Execution` for this reason. Only a stream that runs to completion updates the connection's counters and autocommit flag; an abandoned one leaves them untouched, since a partial count is not a count. `FromValue` conversions are the ones `sqlite3_column_*` performs without reinterpreting storage. An INTEGER widens to `f64`; a REAL does *not* narrow to `i64`, because truncating silently is how a rowid becomes wrong. `NULL` into a non-`Option` type is a `TypeMismatch` rather than a default. By-name access reports real names only for a single-table `SELECT`: a join or a compound gets `column1`, `column2`, … because that is all `result_column_names` derives (`src/codegen/prepare.rs:181`). `joins_and_compounds_report_positional_column_names` asserts exactly that, so the limit is covered rather than discovered by a consumer. Tests ----- 15 unit tests in `tests/unit/api_streaming_test.rs`, using Requirement 7's own scenario names, plus `queried_rows_match_the_oracle` in the corpus suite: eight queries over a 200-row table — larger than one batch — compared row by row against the pinned 3.53.4. Confirmed discriminating. With the final partial batch dropped in `drain` (the classic streaming bug), the oracle test fails on the first query and 10 of the 15 unit tests fail. Gates: make test (1640 passed), make test-corpus (399 passed), make lint (both clippy passes + fmt), check-mod-files, check-assurance 100%/100%. Spend: within estimate. Co-Authored-By: Claude Opus 5 --- Cargo.toml | 4 + src/api.rs | 570 ++++++++++++++++++++++++++++++- tests/corpus/api_oracle_test.rs | 92 +++++ tests/unit/api_streaming_test.rs | 460 +++++++++++++++++++++++++ 4 files changed, 1125 insertions(+), 1 deletion(-) create mode 100644 tests/unit/api_streaming_test.rs diff --git a/Cargo.toml b/Cargo.toml index 83790050..3b8eedd7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -160,6 +160,10 @@ path = "tests/unit/api_changes_test.rs" name = "api_threading" path = "tests/unit/api_threading_test.rs" +[[test]] +name = "api_streaming" +path = "tests/unit/api_streaming_test.rs" + [[test]] name = "codegen_expr" path = "tests/unit/codegen_expr_test.rs" diff --git a/src/api.rs b/src/api.rs index 14248f26..1f9dc552 100644 --- a/src/api.rs +++ b/src/api.rs @@ -67,6 +67,8 @@ mod code { pub const CORRUPT: i32 = 11; /// `SQLITE_CANTOPEN` — unable to open the database file. pub const CANTOPEN: i32 = 14; + /// `SQLITE_MISMATCH` — data type mismatch. + pub const MISMATCH: i32 = 20; /// `SQLITE_MISUSE` — the library was used incorrectly. pub const MISUSE: i32 = 21; /// `SQLITE_RANGE` — a bind index is out of range. @@ -207,6 +209,31 @@ pub enum Error { /// two ways: the connection was closed, or the worker panicked (which /// would be a bug in this crate). ConnectionClosed, + /// A column was read as a type its value cannot convert to. + TypeMismatch { + /// The column, named if the statement has usable names, else its + /// index rendered as text. + column: String, + /// The Rust type asked for. + expected: &'static str, + /// The SQLite storage class actually there. + found: &'static str, + }, + /// A column name was read that this row does not have. + /// + /// Note that by-name access only sees real column names for a + /// single-table `SELECT`; see [`Rows::column_names`]. + ColumnNotFound { + /// The name that was asked for. + name: String, + }, + /// A column index was read that is past the end of the row. + ColumnIndexOutOfRange { + /// The index that was asked for. + index: usize, + /// How many columns the row has. + len: usize, + }, /// Execution failed for a reason with no more specific variant. Execution { /// What went wrong. @@ -237,7 +264,9 @@ impl Error { Error::Parse { .. } | Error::Compile { .. } | Error::NamedParameter { .. } => { code::ERROR } - Error::ParamCount { .. } => code::RANGE, + Error::ParamCount { .. } | Error::ColumnIndexOutOfRange { .. } => code::RANGE, + Error::TypeMismatch { .. } => code::MISMATCH, + Error::ColumnNotFound { .. } => code::ERROR, Error::MultipleStatements { .. } | Error::ConnectionClosed => code::MISUSE, Error::Sqlite { code, .. } => *code, Error::Busy { .. } => code::BUSY, @@ -300,6 +329,18 @@ impl std::fmt::Display for Error { Error::Corrupt { message } => write!(f, "database image is malformed: {message}"), Error::Io { message } => write!(f, "I/O error: {message}"), Error::ConnectionClosed => write!(f, "connection is closed"), + Error::TypeMismatch { + column, + expected, + found, + } => write!( + f, + "column {column} holds {found}, which cannot be read as {expected}" + ), + Error::ColumnNotFound { name } => write!(f, "no such column: {name}"), + Error::ColumnIndexOutOfRange { index, len } => { + write!(f, "column index {index} is out of range for a row of {len}") + } Error::Execution { message } => write!(f, "{message}"), } } @@ -383,6 +424,15 @@ enum Request { /// Where to send the outcome. reply: SyncSender>, }, + /// Run one statement and stream its rows back. + Query { + /// The statement text. + sql: String, + /// Values for its `?`/`?NNN` placeholders. + params: Vec, + /// Where to send the stream's head, or the failure to start it. + reply: SyncSender>, + }, /// Read the connection-scoped counters. Counters { /// Where to send them. @@ -390,6 +440,52 @@ enum Request { }, } +/// How many rows the worker batches per channel send. +/// +/// The bound on peak memory is roughly four of these: one batch being +/// filled on the worker, two in the channel, one held by the caller. +/// Independent of the result size, which is the property Requirement 7 +/// actually asks for. +const CHUNK_ROWS: usize = 64; + +/// Batches the result channel holds before the worker has to wait. +/// +/// Two, not one, and the reason is a liveness one rather than throughput. +/// The worker sends [`Chunk::Done`] after the final batch, so with a single +/// slot a caller that ran a small query and never read it would leave the +/// worker blocked on that `Done` — and blocked workers serve nobody, so the +/// *next* statement on the connection would hang. With two slots, any +/// result that fits in one batch completes and frees the worker whether the +/// caller reads it or not, which covers the case that is easy to hit by +/// accident: +/// +/// ```ignore +/// let rows = conn.query("SELECT 1")?; // never read +/// conn.execute("INSERT ...")?; // must not hang +/// ``` +/// +/// It is a mitigation, not a guarantee: a result spanning more batches than +/// this still parks the worker until the caller reads or drops its [`Rows`]. +/// That is inherent to one worker streaming one execution — see [`Rows`]. +const CHUNK_SLOTS: usize = 2; + +/// The head of a streamed result: its column names, and the channel its +/// rows arrive on. +struct QueryStream { + column_names: Arc>, + chunks: Receiver, +} + +/// One message on a result stream. +enum Chunk { + /// Up to [`CHUNK_ROWS`] rows, in emission order. + Rows(Vec>), + /// The statement finished normally. + Done, + /// The statement failed part-way through. + Failed(Error), +} + /// What one [`Connection::execute`] did. #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct Applied { @@ -510,6 +606,65 @@ impl Connection { self.recv(reply_rx)? } + /// Runs one statement and streams its rows. + /// + /// See [`Rows`] — in particular that holding an undrained `Rows` blocks + /// every other statement on this connection until it is read or + /// dropped. + pub fn query(&self, sql: &str) -> Result { + self.query_with(sql, Vec::new()) + } + + /// Runs one statement with `params` bound, and streams its rows. + pub fn query_with(&self, sql: &str, params: Vec) -> Result { + let (reply_tx, reply_rx) = sync_channel(0); + self.send(Request::Query { + sql: sql.to_string(), + params, + reply: reply_tx, + })?; + let stream = self.recv(reply_rx)??; + Ok(Rows { + column_names: stream.column_names, + chunks: stream.chunks, + buffered: Vec::new().into_iter(), + finished: false, + }) + } + + /// Runs one statement and collects every row. + /// + /// The convenience form for a result a caller knows is small — a + /// pointer store's dozen rows. Use [`Connection::query`] for anything + /// whose size depends on the data. + pub fn query_all(&self, sql: &str) -> Result, Error> { + self.query(sql)?.into_vec() + } + + /// Runs one statement with `params` bound and collects every row. + pub fn query_all_with(&self, sql: &str, params: Vec) -> Result, Error> { + self.query_with(sql, params)?.into_vec() + } + + /// Runs one statement and returns its first row, or `None` if it + /// produced none. + /// + /// Remaining rows are discarded. The `LIMIT 1` existence probe a + /// consumer writes by hand otherwise. + pub fn query_row(&self, sql: &str) -> Result, Error> { + self.query_row_with(sql, Vec::new()) + } + + /// Runs one statement with `params` bound and returns its first row. + pub fn query_row_with(&self, sql: &str, params: Vec) -> Result, Error> { + let mut rows = self.query_with(sql, params)?; + let first = rows.next_row()?; + // Dropped here, which abandons the rest of the stream and frees the + // worker rather than leaving it blocked on a send nobody reads. + drop(rows); + Ok(first) + } + /// Rows changed by the most recent counting statement, as /// `sqlite3_changes()` reports it. /// @@ -558,6 +713,274 @@ impl Connection { } } +/// A result stream, read one row at a time. +/// +/// Rows arrive from the connection's worker thread in batches, so peak +/// memory is bounded by the batch size and the pager's page cache rather +/// than by the size of the result (spec 013 Requirement 7). Reading the +/// first ten rows of a million-row table costs the same as reading the +/// first ten of a ten-row one. +/// +/// # Holding an undrained `Rows` can block the connection +/// +/// The engine stays inside one execution until the result is drained or +/// this handle is dropped, so while a large result is outstanding, any +/// other statement on the same connection — including from another thread +/// holding a clone — waits. +/// +/// A result that fits in one batch is safe: it completes and frees the +/// worker whether or not it is read (see `CHUNK_SLOTS`). A larger one is +/// not: +/// +/// ```ignore +/// let rows = conn.query("SELECT a FROM big")?; // thousands of rows +/// let more = conn.query("SELECT 1")?; // blocks until `rows` goes +/// ``` +/// +/// Dropping `Rows` releases the worker immediately, so this is a wait +/// rather than a permanent deadlock — but a thread that holds a `Rows` +/// while waiting on another thread that needs the same connection will +/// hang. **Drain it, drop it, or bind it to a short scope.** Requirement 4 +/// accepts serialized access; this is its sharp edge. +/// +/// Deliberately not an [`Iterator`]: [`Rows::next_row`] returns +/// `Result, Error>`, and collapsing that into +/// `Option>` to fit the trait makes "the stream ended" +/// and "the stream failed" the same shape at the call site. ADR-0040 +/// rejected an `Iterator` impl on `Execution` for this reason and the +/// reasoning carries. +#[derive(Debug)] +pub struct Rows { + column_names: Arc>, + chunks: Receiver, + buffered: std::vec::IntoIter>, + /// Set once the stream has ended, normally or otherwise, so a caller + /// that keeps polling gets `None` rather than a channel error. + finished: bool, +} + +impl Rows { + /// The result's column names. + /// + /// Available before the first row is read, and empty for a statement + /// that returns no columns. + /// + /// **Real names only for a single-table `SELECT`.** A join or a + /// compound (`UNION`) reports `column1`, `column2`, … because that is + /// what `codegen::result_column_names` can currently derive + /// (`src/codegen/prepare.rs:181`). By-index access is unaffected; + /// by-name access on a join will not find the name a caller expects. + /// Stated rather than hidden — the fix belongs in the name resolver, + /// not here. + pub fn column_names(&self) -> &[String] { + &self.column_names + } + + /// Reads the next row, or `None` once the result is exhausted. + /// + /// Blocks until the worker produces a row. + pub fn next_row(&mut self) -> Result, Error> { + loop { + if let Some(values) = self.buffered.next() { + return Ok(Some(Row { + values, + column_names: Arc::clone(&self.column_names), + })); + } + if self.finished { + return Ok(None); + } + match self.chunks.recv() { + Ok(Chunk::Rows(batch)) => self.buffered = batch.into_iter(), + Ok(Chunk::Done) => { + self.finished = true; + return Ok(None); + } + Ok(Chunk::Failed(e)) => { + self.finished = true; + return Err(e); + } + // The worker vanished mid-stream without saying why, which + // means it panicked — a bug here, not a caller error. + Err(_) => { + self.finished = true; + return Err(Error::ConnectionClosed); + } + } + } + } + + /// Reads every remaining row into a `Vec`. + /// + /// For results a caller knows are small. Defeats the streaming + /// property by construction, which is fine when a dozen rows is the + /// whole answer and is the wrong choice otherwise. + pub fn into_vec(mut self) -> Result, Error> { + let mut out = Vec::new(); + while let Some(row) = self.next_row()? { + out.push(row); + } + Ok(out) + } +} + +/// One result row. +#[derive(Debug, Clone)] +pub struct Row { + values: Vec, + column_names: Arc>, +} + +impl Row { + /// How many columns this row has. + pub fn len(&self) -> usize { + self.values.len() + } + + /// Whether this row has no columns. + pub fn is_empty(&self) -> bool { + self.values.is_empty() + } + + /// This row's column names — see [`Rows::column_names`] for when they + /// are real names. + pub fn column_names(&self) -> &[String] { + &self.column_names + } + + /// The raw [`Value`] at `index`, or `None` if the row is shorter. + /// + /// The escape hatch from [`Row::get`]'s conversions, for a caller that + /// wants to switch on the storage class itself. + pub fn value(&self, index: usize) -> Option<&Value> { + self.values.get(index) + } + + /// Reads column `index` (0-based) as `T`. + pub fn get(&self, index: usize) -> Result { + let value = self.values.get(index).ok_or(Error::ColumnIndexOutOfRange { + index, + len: self.values.len(), + })?; + T::from_value(value).map_err(|expected| Error::TypeMismatch { + column: self + .column_names + .get(index) + .cloned() + .unwrap_or_else(|| index.to_string()), + expected, + found: storage_class(value), + }) + } + + /// Reads the column called `name` as `T`. + /// + /// Case-insensitive, matching SQLite's column-name comparison. See + /// [`Rows::column_names`]: on a join or a compound the names are + /// positional placeholders, so this will not find a base-table name. + pub fn get_by_name(&self, name: &str) -> Result { + let index = self + .column_names + .iter() + .position(|candidate| candidate.eq_ignore_ascii_case(name)) + .ok_or_else(|| Error::ColumnNotFound { + name: name.to_string(), + })?; + self.get(index) + } +} + +/// A type a [`Value`] can be read as. +/// +/// Conversions are the ones SQLite's own `sqlite3_column_*` family +/// performs without reinterpreting storage: an `INTEGER` reads as `i64`, +/// and as `f64` because that is lossless for the range in practice; a +/// `REAL` does not read as `i64`, because truncating silently is how a +/// rowid becomes wrong. `Option` is how a nullable column is read — +/// `NULL` into a non-`Option` type is a [`Error::TypeMismatch`] rather +/// than a default value. +pub trait FromValue: Sized { + /// Converts `value`, or returns the name of the type that was wanted + /// so the caller can build the error with the column's identity. + fn from_value(value: &Value) -> Result; +} + +impl FromValue for i64 { + fn from_value(value: &Value) -> Result { + match value { + Value::Integer(v) => Ok(*v), + _ => Err("i64"), + } + } +} + +impl FromValue for f64 { + fn from_value(value: &Value) -> Result { + match value { + Value::Real(v) => Ok(*v), + // Widening an integer is lossless and is what + // `sqlite3_column_double` does. + Value::Integer(v) => Ok(*v as f64), + _ => Err("f64"), + } + } +} + +impl FromValue for bool { + fn from_value(value: &Value) -> Result { + match value { + // SQLite has no boolean storage class; 0 is false and every + // other integer is true, as its own `CASE`/`WHERE` do. + Value::Integer(v) => Ok(*v != 0), + _ => Err("bool"), + } + } +} + +impl FromValue for String { + fn from_value(value: &Value) -> Result { + match value { + Value::Text(v) => Ok(v.to_string()), + _ => Err("String"), + } + } +} + +impl FromValue for Vec { + fn from_value(value: &Value) -> Result { + match value { + Value::Blob(v) => Ok(v.to_vec()), + _ => Err("Vec"), + } + } +} + +impl FromValue for Value { + fn from_value(value: &Value) -> Result { + Ok(value.clone()) + } +} + +impl FromValue for Option { + fn from_value(value: &Value) -> Result { + match value { + Value::Null => Ok(None), + other => T::from_value(other).map(Some), + } + } +} + +/// The SQLite storage-class name for `value`, for error messages. +fn storage_class(value: &Value) -> &'static str { + match value { + Value::Null => "NULL", + Value::Integer(_) => "INTEGER", + Value::Real(_) => "REAL", + Value::Text(_) => "TEXT", + Value::Blob(_) => "BLOB", + } +} + /// What the worker should open. enum Target { /// A real file, through `UnixVfs`. @@ -631,6 +1054,9 @@ fn worker_main( Request::ExecuteBatch { sql, reply } => { answer(&reply, engine.execute_batch(&sql)); } + Request::Query { sql, params, reply } => { + engine.stream(&sql, params, &reply); + } Request::Counters { reply } => { answer(&reply, engine.counters); } @@ -778,6 +1204,148 @@ impl Engine { Ok(Applied { changes }) } + /// Runs `sql` and streams its rows to `reply`'s receiver. + /// + /// The whole execution lives in this one stack frame, which is what + /// makes it expressible at all: `Execution` borrows its `Program`, and + /// no type in this module may carry a lifetime (see the module docs). + /// So the worker stays inside this call until the result is drained or + /// the caller drops its [`Rows`] — and while it does, this connection + /// serves nothing else. + /// + /// That is a real constraint on callers, not an implementation detail: + /// holding an undrained `Rows` blocks every other statement on the same + /// connection until it is read or dropped. It is a wait rather than a + /// deadlock — dropping `Rows` closes the channel, the next send fails, + /// and the worker returns here — but a thread that holds a `Rows` while + /// waiting on another thread that needs the same connection will hang. + /// Requirement 4 accepts serialized access; this is its sharp edge, and + /// `Rows`'s own documentation repeats it. + fn stream( + &mut self, + sql: &str, + params: Vec, + reply: &SyncSender>, + ) { + let started = self.start_stream(sql, params.len()); + let (program, column_names) = match started { + Ok(pair) => pair, + Err(e) => { + answer(reply, Err(e)); + return; + } + }; + + // Bounded, so a slow reader applies backpressure to the engine + // rather than letting rows pile up. See `CHUNK_SLOTS` for why the + // bound is two and not one. + let (chunk_tx, chunk_rx) = sync_channel::(CHUNK_SLOTS); + let head = QueryStream { + column_names, + chunks: chunk_rx, + }; + if reply.send(Ok(head)).is_err() { + // The caller gave up before reading anything; nothing ran, so + // there is nothing to unwind. + return; + } + + self.drain(&program, params, &chunk_tx); + } + + /// Compiles `sql`, checks it against this connection's mode and its + /// parameter arity, and works out its column names. + fn start_stream( + &mut self, + sql: &str, + param_count: usize, + ) -> Result<(Program, Arc>), Error> { + let program = self.compile(sql)?; + if self.mode == OpenMode::ReadOnly && writes(&program) { + return Err(Error::ReadOnly { + statement: sql.to_string(), + }); + } + let wanted = program.param_count(); + if wanted != param_count { + return Err(Error::ParamCount { + expected: wanted, + found: param_count, + }); + } + let names = self.column_names_of(sql)?; + Ok((program, Arc::new(names))) + } + + /// The result column names for `sql`, empty for a statement that is not + /// a `SELECT`. + fn column_names_of(&mut self, sql: &str) -> Result, Error> { + use crate::parser::error::ParseOutcome; + + if !is_select(sql) { + return Ok(Vec::new()); + } + let ParseOutcome::Accepted(select) = crate::parser::parse_select(sql) else { + // Unreachable: `compile` already parsed this successfully. + return Ok(Vec::new()); + }; + let (schemas, _) = self.catalog()?; + Ok(crate::codegen::result_column_names(&select, schemas)) + } + + /// Drives `program` to completion, sending rows in batches. + fn drain(&mut self, program: &Program, params: Vec, chunks: &SyncSender) { + let mut vm = + crate::vdbe::Vm::with_shared_writable_db(std::rc::Rc::clone(&self.pager), self.header); + vm.autocommit = self.autocommit; + vm.bind_params(params); + let mut execution = crate::vdbe::Execution::new(vm, program); + + let mut batch: Vec> = Vec::new(); + loop { + match execution.next_row() { + Ok(Some(row)) => { + batch.push(row); + if batch.len() >= CHUNK_ROWS + && chunks + .send(Chunk::Rows(std::mem::take(&mut batch))) + .is_err() + { + // The caller dropped its `Rows`. Abandon the + // execution here: dropping it releases its cursors, + // which is Requirement 7's "abandoning a statement + // releases it". + return; + } + } + Ok(None) => break, + Err(e) => { + // A mid-stream failure. Send it rather than the rows + // already batched: a partial result the caller cannot + // tell is partial would be worse than no result. + answer(chunks, Chunk::Failed(exec_error(e))); + return; + } + } + } + + if !batch.is_empty() && chunks.send(Chunk::Rows(batch)).is_err() { + return; + } + + // Only a stream that ran to completion updates the connection's + // state. An abandoned one returns above, leaving the counters and + // the autocommit flag as they were. + self.autocommit = execution.autocommit(); + if program.counts_changes() { + self.counters.changes = execution.changes(); + } + if let Some(rowid) = execution.last_insert_rowid() { + self.counters.last_insert_rowid = rowid; + } + answer(chunks, Chunk::Done); + } + fn compile(&mut self, sql: &str) -> Result { // One entry point for every statement kind, which is what // `sqlite3_prepare_v2` presents and what #695's lift made possible: diff --git a/tests/corpus/api_oracle_test.rs b/tests/corpus/api_oracle_test.rs index 8d813516..4683ec73 100644 --- a/tests/corpus/api_oracle_test.rs +++ b/tests/corpus/api_oracle_test.rs @@ -270,3 +270,95 @@ fn parameterised_writes_match_the_oracle() { std::fs::remove_dir_all(&dir).ok(); } + +/// The read path against the oracle: every row of every query, in order, +/// rendered the way `sqlite3` renders it. +/// +/// Streaming is the mechanism (`tests/unit/api_streaming_test.rs` pins +/// that); this pins the *answers*. A `Rows` that streamed the wrong values, +/// or dropped or reordered a batch boundary, would pass every unit test +/// that only checks counts and shapes. +#[test] +fn queried_rows_match_the_oracle() { + let Some(bin) = pinned_oracle() else { + skip_no_oracle("queried_rows_match_the_oracle"); + return; + }; + let dir = scratch_dir("reads"); + let ours = dir.join("ours.db"); + let theirs = dir.join("theirs.db"); + + // More rows than one channel batch (64), so a result that spans batch + // boundaries is compared rather than only a short one. + let mut setup = String::from("CREATE TABLE t(a INTEGER, b TEXT, r REAL);"); + for i in 0..200 { + let r = f64::from(i) / 4.0; + setup.push_str(&format!("INSERT INTO t VALUES ({i}, 'row-{i}', {r});")); + } + setup.push_str("CREATE INDEX t_a ON t(a);"); + + let conn = Connection::open(&ours).unwrap(); + conn.execute_batch(&setup).unwrap(); + drop(conn); + oracle_says(&bin, &theirs, &setup); + + let queries = [ + "SELECT a, b, r FROM t", + "SELECT a FROM t WHERE a > 150", + "SELECT a, b FROM t WHERE a < 3", + "SELECT count(*) FROM t", + "SELECT a FROM t ORDER BY a DESC", + "SELECT b FROM t WHERE a = 42", + "SELECT a FROM t LIMIT 5", + "SELECT sum(a), min(a), max(a) FROM t", + ]; + + // Reopen for reading; each `Rows` is scoped so the worker is never held + // by one result while the next query is issued. + let conn = Connection::open(&ours).unwrap(); + for sql in queries { + let mine = { + let mut rows = conn + .query(sql) + .unwrap_or_else(|e| panic!("{sql} failed through the API: {e}")); + let mut rendered = Vec::new(); + while let Some(row) = rows.next_row().unwrap() { + let cells: Vec = (0..row.len()) + .map(|i| render(row.value(i).expect("in range"))) + .collect(); + rendered.push(cells.join("|")); + } + rendered + }; + + let theirs_rows = oracle_says(&bin, &theirs, &format!("{sql};")); + let expected: Vec = if theirs_rows.is_empty() { + Vec::new() + } else { + theirs_rows.lines().map(|l| l.to_string()).collect() + }; + + assert_eq!(mine, expected, "rows diverge for {sql:?}"); + } + + std::fs::remove_dir_all(&dir).ok(); +} + +/// Renders a value the way the `sqlite3` shell's default list mode does, so +/// the two sides are comparable as text. +fn render(value: &Value) -> String { + match value { + Value::Null => String::new(), + Value::Integer(v) => v.to_string(), + Value::Real(v) => { + // The shell prints a float with no fractional part as `N.0`. + if v.fract() == 0.0 && v.is_finite() { + format!("{v:.1}") + } else { + v.to_string() + } + } + Value::Text(v) => v.to_string(), + Value::Blob(v) => String::from_utf8_lossy(v).into_owned(), + } +} diff --git a/tests/unit/api_streaming_test.rs b/tests/unit/api_streaming_test.rs new file mode 100644 index 00000000..d965e994 --- /dev/null +++ b/tests/unit/api_streaming_test.rs @@ -0,0 +1,460 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! Incremental row access (spec 013 Requirement 7). +//! +//! `execute_with_db` and friends return `Vec>`, so the engine +//! allocates the whole result before the caller sees row 0. Spike 014 +//! (#682) measured that at 137.7 MB peak heap and 5.36 ms to first row for +//! a 1,000,000-row result, against 8.68 MB and 44.7 µs streamed. +//! +//! What these tests pin is the property the spec settled on after the +//! original wording turned out to be unsatisfiable: peak cost is +//! **independent of the result size**, not proportional to the rows pulled. +//! A streaming read's memory is dominated by the pager's page cache, so it +//! is a floor rather than a slope — and independence is testable where +//! proportionality is not. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::time::Instant; + +use sqlite_rs::api::{Connection, Error}; +use sqlite_rs::record::Value; + +fn seeded(rows: i64) -> Connection { + let conn = Connection::open_in_memory().unwrap(); + conn.execute("CREATE TABLE t(a INTEGER, b TEXT)").unwrap(); + conn.execute("BEGIN").ok(); + for i in 0..rows { + conn.execute_with( + "INSERT INTO t VALUES (?1, ?2)", + vec![Value::from(i), Value::from(format!("row-{i}"))], + ) + .unwrap(); + } + conn.execute("COMMIT").ok(); + conn +} + +#[test] +fn rows_come_back_in_order_with_their_values() { + let conn = seeded(10); + let mut rows = conn.query("SELECT a, b FROM t ORDER BY a").unwrap(); + + for expected in 0..10i64 { + let row = rows.next_row().unwrap().expect("a row"); + assert_eq!(row.get::(0).unwrap(), expected); + assert_eq!(row.get::(1).unwrap(), format!("row-{expected}")); + } + assert!(rows.next_row().unwrap().is_none(), "should be exhausted"); + // Polling past the end keeps answering None rather than erroring. + assert!(rows.next_row().unwrap().is_none()); +} + +#[test] +fn column_names_are_available_before_the_first_row() { + let conn = seeded(3); + let rows = conn.query("SELECT a, b FROM t ORDER BY a").unwrap(); + assert_eq!(rows.column_names(), ["a", "b"]); +} + +/// A result larger than one channel batch, so the batching path is +/// exercised rather than only the single-chunk case. +#[test] +fn a_result_spanning_many_batches_is_complete_and_ordered() { + let conn = seeded(500); + let mut rows = conn.query("SELECT a FROM t ORDER BY a").unwrap(); + let mut seen = 0i64; + while let Some(row) = rows.next_row().unwrap() { + assert_eq!(row.get::(0).unwrap(), seen); + seen += 1; + } + assert_eq!(seen, 500); +} + +/// Requirement 7's first scenario, as the amended spec states it: the cost +/// of reading ten rows must not grow with the table. +/// +/// Timing rather than heap, because a portable allocator hook is not +/// available here and the spike already measured the heap directly (#682: +/// 137.7 MB materialized against 8.68 MB streamed at 1,000,000 rows). Time +/// to first row is the observable that would degrade if the engine +/// materialized: it would have to build every row before handing over row +/// 0, so it would scale with the table while a streamed read stays flat. +/// +/// **The plan must be non-blocking, and the query here is chosen for that.** +/// An earlier draft used `SELECT a FROM t ORDER BY a` and failed at 33x on +/// a 50x larger table — correctly. `ORDER BY` with no usable index is a +/// blocking operator: the sorter consumes every row before emitting the +/// first, so time-to-first-row is genuinely linear and no amount of +/// streaming changes that. Stock SQLite behaves identically. Testing the +/// streaming property therefore requires a plan that can emit as it scans, +/// which is what a bare scan is. `blocking_plans_are_linear_by_nature` +/// below pins the contrast so this is documented rather than merely +/// avoided. +/// +/// The bound is deliberately loose (a 20x allowance over a 50x size +/// increase). This is a *shape* assertion — flat versus linear — and a +/// tight threshold on a shared machine would be flaky without testing +/// anything more. +#[test] +fn partial_read_is_bounded() { + let first_ten = |conn: &Connection| -> (std::time::Duration, Vec) { + let start = Instant::now(); + // No ORDER BY: rowid order already ascends by `a` here, and a bare + // scan can emit its first row without reading the last. + let mut rows = conn.query("SELECT a FROM t").unwrap(); + let mut out = Vec::new(); + for _ in 0..10 { + match rows.next_row().unwrap() { + Some(row) => out.push(row.get::(0).unwrap()), + None => break, + } + } + let elapsed = start.elapsed(); + // Dropped undrained, after the clock stops: the rest of the result + // is abandoned and the worker is freed. + drop(rows); + (elapsed, out) + }; + + let small = seeded(200); + let large = seeded(10_000); + + let (small_time, small_rows) = first_ten(&small); + let (large_time, large_rows) = first_ten(&large); + + assert_eq!(small_rows, (0..10).collect::>()); + assert_eq!( + large_rows, small_rows, + "the first ten rows should not depend on how many follow" + ); + + // 50x the rows; if the read were materializing, time-to-ten would + // scale with it. + let ratio = large_time.as_secs_f64() / small_time.as_secs_f64().max(1e-9); + assert!( + ratio < 20.0, + "reading the first ten rows took {ratio:.1}x longer on a 50x larger table \ + ({small_time:?} -> {large_time:?}); that is the shape of a materializing read" + ); +} + +/// Requirement 7's second scenario: an abandoned statement releases its +/// cursors, and the connection is immediately usable for a write. +/// +/// This is also the test that would hang if `Rows::drop` did not free the +/// worker — the write below goes to the same connection, and the worker is +/// inside the abandoned execution until the channel closes. +#[test] +fn abandoned_statement_releases_cursors() { + let conn = seeded(1_000); + + let mut rows = conn.query("SELECT a FROM t ORDER BY a").unwrap(); + assert!(rows.next_row().unwrap().is_some()); + assert!(rows.next_row().unwrap().is_some()); + drop(rows); + + // A write on the same connection must proceed, not block behind the + // cursors the abandoned read had open. + assert_eq!(conn.execute("DELETE FROM t WHERE a < 10").unwrap(), 10); + assert_eq!( + conn.execute("INSERT INTO t VALUES (99999, 'x')").unwrap(), + 1 + ); + + // ...and the connection is still fully functional afterwards. + let remaining: i64 = conn + .query_row("SELECT count(*) FROM t") + .unwrap() + .expect("count returns a row") + .get(0) + .unwrap(); + assert_eq!(remaining, 991); +} + +/// Repeatedly abandoning streams must not leak the worker into a bad state. +#[test] +fn many_abandoned_streams_leave_the_connection_healthy() { + let conn = seeded(300); + for _ in 0..50 { + let mut rows = conn.query("SELECT a, b FROM t ORDER BY a").unwrap(); + assert!(rows.next_row().unwrap().is_some()); + drop(rows); + } + let total: i64 = conn + .query_row("SELECT count(*) FROM t") + .unwrap() + .unwrap() + .get(0) + .unwrap(); + assert_eq!(total, 300); +} + +#[test] +fn values_read_by_name_and_by_index_agree() { + let conn = seeded(1); + let row = conn + .query_row("SELECT a, b FROM t") + .unwrap() + .expect("one row"); + + assert_eq!( + row.get::(0).unwrap(), + row.get_by_name::("a").unwrap() + ); + assert_eq!( + row.get::(1).unwrap(), + row.get_by_name::("b").unwrap() + ); + // Case-insensitive, matching SQLite's column-name comparison. + assert_eq!(row.get_by_name::("A").unwrap(), 0); +} + +#[test] +fn every_storage_class_reads_back_as_its_rust_type() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute("CREATE TABLE v(i INTEGER, r REAL, t TEXT, b BLOB, n INTEGER)") + .unwrap(); + conn.execute_with( + "INSERT INTO v VALUES (?1, ?2, ?3, ?4, ?5)", + vec![ + Value::from(7i64), + Value::from(2.5), + Value::from("hello"), + Value::from(vec![1u8, 2, 3]), + Value::Null, + ], + ) + .unwrap(); + + let row = conn + .query_row("SELECT i, r, t, b, n FROM v") + .unwrap() + .unwrap(); + assert_eq!(row.get::(0).unwrap(), 7); + assert_eq!(row.get::(1).unwrap(), 2.5); + assert_eq!(row.get::(2).unwrap(), "hello"); + assert_eq!(row.get::>(3).unwrap(), vec![1u8, 2, 3]); + assert_eq!(row.get::>(4).unwrap(), None); + + // An INTEGER widens to f64 (what sqlite3_column_double does)... + assert_eq!(row.get::(0).unwrap(), 7.0); + // ...but a REAL does not narrow to i64, because truncating silently is + // how a rowid becomes wrong. + assert!(matches!(row.get::(1), Err(Error::TypeMismatch { .. }))); + // bool follows SQLite: 0 is false, anything else true. + assert!(row.get::(0).unwrap()); + // And NULL into a non-Option type is an error, not a default. + let err = row.get::(4).expect_err("NULL is not an i64"); + match err { + Error::TypeMismatch { + ref column, + expected, + found, + } => { + assert_eq!(column, "n", "the error should name the column"); + assert_eq!(expected, "i64"); + assert_eq!(found, "NULL"); + } + other => panic!("expected TypeMismatch, got {other:?}"), + } + assert_eq!(err.sqlite_code(), 20, "should report SQLITE_MISMATCH"); +} + +#[test] +fn reading_past_the_last_column_or_a_missing_name_errors() { + let conn = seeded(1); + let row = conn.query_row("SELECT a, b FROM t").unwrap().unwrap(); + + assert_eq!(row.len(), 2); + assert_eq!( + row.get::(5).expect_err("out of range"), + Error::ColumnIndexOutOfRange { index: 5, len: 2 } + ); + assert_eq!( + row.get_by_name::("nope").expect_err("no such column"), + Error::ColumnNotFound { + name: "nope".to_string() + } + ); +} + +/// Honest coverage of a known limit rather than a hidden one: a join or a +/// compound reports positional names, so by-name access does not find the +/// base-table names a caller would expect. +#[test] +fn joins_and_compounds_report_positional_column_names() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE a(x INTEGER); + CREATE TABLE b(y INTEGER); + INSERT INTO a VALUES (1); + INSERT INTO b VALUES (2);", + ) + .unwrap(); + + // Each `Rows` is bound to its own scope. Holding one while issuing the + // next query is the hazard `Rows` documents, and an earlier draft of + // this very test deadlocked on it — the worker was parked sending the + // first result's `Done` while this thread asked for a second one. + { + let joined = conn + .query("SELECT a.x, b.y FROM a JOIN b ON a.x < b.y") + .unwrap(); + assert_eq!( + joined.column_names(), + ["column1", "column2"], + "a join reports positional names — see Rows::column_names" + ); + } + { + let compound = conn.query("SELECT x FROM a UNION SELECT y FROM b").unwrap(); + assert_eq!(compound.column_names(), ["column1"]); + } + + // By-index access is unaffected, which is why this is a documented + // limit rather than a blocker. + let rows = conn + .query_all("SELECT a.x, b.y FROM a JOIN b ON a.x < b.y") + .unwrap(); + assert_eq!(rows.len(), 1); + let first = rows.first().expect("one row"); + assert_eq!(first.get::(0).unwrap(), 1); + assert_eq!(first.get::(1).unwrap(), 2); +} + +/// A query is still a statement: the arity and named-parameter rules apply +/// on the read path exactly as on the write path. +#[test] +fn query_enforces_the_same_parameter_rules_as_execute() { + let conn = seeded(3); + + assert_eq!( + conn.query_with("SELECT a FROM t WHERE a = ?1", vec![]) + .expect_err("one placeholder, no values"), + Error::ParamCount { + expected: 1, + found: 0 + } + ); + assert!(matches!( + conn.query("SELECT a FROM t WHERE a = :id"), + Err(Error::NamedParameter { .. }) + )); + + let row = conn + .query_row_with("SELECT a FROM t WHERE a = ?1", vec![Value::from(2)]) + .unwrap() + .expect("a match"); + assert_eq!(row.get::(0).unwrap(), 2); +} + +#[test] +fn a_query_returning_nothing_is_an_empty_stream_not_an_error() { + let conn = seeded(3); + let mut rows = conn.query("SELECT a FROM t WHERE a = 999").unwrap(); + assert!(rows.next_row().unwrap().is_none()); + assert_eq!( + conn.query_all("SELECT a FROM t WHERE a = 999") + .unwrap() + .len(), + 0 + ); + assert!(conn + .query_row("SELECT a FROM t WHERE a = 999") + .unwrap() + .is_none()); +} + +/// The counterpart to `partial_read_is_bounded`, so the limit it works +/// around is recorded rather than hidden. +/// +/// A blocking operator has to consume its whole input before it can emit +/// anything, so streaming cannot make its time-to-first-row flat. This is +/// not a defect in the streaming path and it is not specific to this crate +/// — stock SQLite sorts the same way. A consumer that needs a bounded +/// first-row latency needs an index the sort can walk, not a different API. +/// +/// Asserted as a *contrast*, not an absolute threshold: the same table and +/// the same ten rows, scanned versus sorted. +#[test] +fn blocking_plans_are_linear_by_nature() { + let conn = seeded(4_000); + + let ten = |sql: &str| -> std::time::Duration { + let start = Instant::now(); + let mut rows = conn.query(sql).unwrap(); + for _ in 0..10 { + if rows.next_row().unwrap().is_none() { + break; + } + } + let elapsed = start.elapsed(); + drop(rows); + elapsed + }; + + // Warm the page cache so this measures the plan, not the first read of + // the file. The duration is deliberately discarded. + ten("SELECT a FROM t"); + + let scanned = ten("SELECT a FROM t"); + let sorted = ten("SELECT a FROM t ORDER BY b"); + + assert!( + sorted > scanned, + "a sort should cost more to first row than a scan on the same table \ + (scan {scanned:?}, sort {sorted:?}) — if this ever inverts, the \ + reasoning in partial_read_is_bounded needs revisiting" + ); +} + +/// A small result that is never read must not park the worker. +/// +/// This is the accident that is easy to have — `let rows = conn.query(..)` +/// and then forget it — and the reason the result channel holds two batches +/// rather than one. With a single slot the worker blocks sending `Done` +/// into a full channel, and the *next* statement on the connection hangs. +/// +/// The test would hang rather than fail if that regressed, which is the +/// strongest form the assertion can take here. +#[test] +fn an_unread_small_result_does_not_block_the_next_statement() { + let conn = seeded(5); + + let unread = conn.query("SELECT a FROM t").unwrap(); + // Deliberately not read and deliberately still alive. + assert_eq!(unread.column_names(), ["a"]); + + // Must proceed while `unread` is outstanding. + assert_eq!(conn.execute("INSERT INTO t VALUES (100, 'x')").unwrap(), 1); + let count: i64 = conn + .query_row("SELECT count(*) FROM t") + .unwrap() + .unwrap() + .get(0) + .unwrap(); + assert_eq!(count, 6); + + drop(unread); +} + +/// The other half of the same contract, stated so the limit is recorded: +/// a result too large to buffer *does* park the worker, and dropping the +/// handle is what releases it. +/// +/// Written as a sequence that must complete, not as a timing assertion — +/// the point is that dropping is sufficient, not how long anything took. +#[test] +fn dropping_a_large_unread_result_releases_the_connection() { + let conn = seeded(1_000); + + let big = conn.query("SELECT a, b FROM t").unwrap(); + // The worker is now parked mid-scan: more rows than the channel holds, + // and nothing is reading them. + drop(big); + + // Released, so the connection serves again. + assert_eq!(conn.execute("DELETE FROM t WHERE a < 5").unwrap(), 5); +} From 1cc794b8a0daac35d40d0a3043408512866abf85 Mon Sep 17 00:00:00 2001 From: Diederik Siderius Date: Wed, 9 Sep 2026 20:58:28 +0200 Subject: [PATCH 08/14] feat: transactions, pragma and a busy timeout on the embedding API (013/Req 5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Transaction` (deferred/immediate/exclusive) that rolls back on drop, `Connection::pragma`, `Connection::set_busy_timeout`, and a stated durability contract. `Transaction` derefs to its `Connection`, so every `execute`/`query` method runs inside the transaction. The point of the type is the drop: a `?` returning early out of a function holding a transaction cannot leave it open, which for a consumer storing pointers to data it cannot otherwise find is the difference between a retryable failure and a half-applied catalog. Busy retry, and why it is scoped the way it is ---------------------------------------------- The timeout retries only while the connection is in autocommit. There the statement *is* the transaction, so `Pager::rollback` followed by re-running it is a faithful retry of the whole unit. Inside an explicit transaction it would not be: the statement's mutations sit in the same pending set as every earlier statement's, so re-running one would double-apply it. A `Busy` there is the transaction's to retry, which is what stock SQLite does with `SQLITE_BUSY` at `COMMIT`. The rollback before each retry is load-bearing, not hygiene. `Pager::flush` documents that a contended escalation surfaces `VfsError::Locked` "before any byte of this transaction is journaled or written" and leaves `dirty` intact (`src/pager.rs:524`) — so without the rollback, a retried INSERT would land once per attempt. `a_retried_statement_succeeds_exactly_once` is the test for exactly that. Backoff ladder mirrors `sqliteDefaultBusyCallback`. Default timeout is zero, matching SQLite. `Error::Busy` is now matched **structurally** rather than on message text (`ExecError::FlushFailed(PagerError::Vfs(VfsError::Locked { .. }))`). Requirement 5 makes busy a distinct, retryable variant, so classifying it on a substring was exactly the wrong trade: a reworded `Display` would have silently turned every busy error permanent with no test failing. A silent data-loss bug found, recorded as a ratchet -------------------------------------------------- **Two connections on one file in the same process do not lock against each other, and a write that reports success can be silently discarded.** Measured on this tree: connection A takes `BEGIN IMMEDIATE` and inserts row 2; connection B's insert of row 3 returns `Ok(1)`; A commits; the file then holds `[1, 2]`. Row 3 is gone, and `PRAGMA integrity_check` says `ok`, so nothing flags it. The cause is POSIX, not this crate's logic: `fcntl` locks are scoped to `(process, inode)`, which `src/vfs/lock.rs:96` documents and `check_reserved_lock` states outright ("whether some *other* process currently holds a write lock"). Stock SQLite closes it with `unixInodeInfo` in `os_unix.c` — a process-global registry keyed by `(device, inode)` with its own mutex and lock counts, so two connections in one process serialize like two processes. There is no equivalent here. Pre-existing in the engine, but this facade makes it far easier to reach: Requirement 4 exists so a *pool* can hold a handle, and opening the same path twice is the obvious thing to do. Recorded as an `#[ignore]`d ratchet (`in_process_connections_lock_against_each_other`) written to assert the correct behaviour, so it passes unchanged once a registry lands. Verified to fail when run: "a write that reported success was silently discarded". Needs its own ticket. Tests ----- `tests/unit/api_durability_test.rs` (2 + the ratchet) and `tests/unit/api_transaction_test.rs` (10) cover the surface. The busy-contention tests moved to `tests/corpus/api_durability_oracle_test.rs` (4), because they need a *second process* to hold the lock — the pinned `sqlite3`, which also makes the claim stronger: the lock protocol is honoured against stock SQLite, not just against ourselves. `commit_survives_hard_kill` re-executes this test binary as a child, which commits under `synchronous = FULL`, reports via a marker file, and blocks; the parent SIGKILLs it and checks the rows survive and the oracle finds the file well-formed. Its doc states what that does *not* prove: SIGKILL leaves the kernel page cache intact, so it cannot distinguish FULL from OFF. Only a power cut or a crash-injecting VFS does, and that regime is `crash_torture_test.rs`. Gates: make test (1652 passed, 10 ignored — 1 new ratchet), make test-corpus (403 passed), make lint (both clippy passes + fmt). Spend: within estimate. Co-Authored-By: Claude Opus 5 --- Cargo.toml | 8 + src/api.rs | 321 ++++++++++++++++-- tests/corpus/api_durability_oracle_test.rs | 364 +++++++++++++++++++++ tests/corpus/main.rs | 1 + tests/unit/api_durability_test.rs | 162 +++++++++ tests/unit/api_transaction_test.rs | 228 +++++++++++++ 6 files changed, 1064 insertions(+), 20 deletions(-) create mode 100644 tests/corpus/api_durability_oracle_test.rs create mode 100644 tests/unit/api_durability_test.rs create mode 100644 tests/unit/api_transaction_test.rs diff --git a/Cargo.toml b/Cargo.toml index 3b8eedd7..61dcd66c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -164,6 +164,14 @@ path = "tests/unit/api_threading_test.rs" name = "api_streaming" path = "tests/unit/api_streaming_test.rs" +[[test]] +name = "api_transaction" +path = "tests/unit/api_transaction_test.rs" + +[[test]] +name = "api_durability" +path = "tests/unit/api_durability_test.rs" + [[test]] name = "codegen_expr" path = "tests/unit/codegen_expr_test.rs" diff --git a/src/api.rs b/src/api.rs index 1f9dc552..3a830e7c 100644 --- a/src/api.rs +++ b/src/api.rs @@ -42,6 +42,7 @@ use std::path::{Path, PathBuf}; use std::sync::mpsc::{sync_channel, Receiver, SyncSender}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; +use std::time::{Duration, Instant}; use crate::header::{DatabaseHeader, DEFAULT_PAGE_SIZE}; use crate::pager::Pager; @@ -102,6 +103,39 @@ pub enum OpenMode { ReadWriteCreate, } +/// How a transaction acquires its locks. +/// +/// Matches SQLite's `BEGIN [DEFERRED|IMMEDIATE|EXCLUSIVE]` (#356, #395). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum TransactionBehavior { + /// Take no write lock until the first write — SQLite's default. + /// + /// Cheapest to start and the most likely to meet + /// [`Error::Busy`](Error::Busy) later, because two deferred + /// transactions can both begin and then collide. + #[default] + Deferred, + /// Take the RESERVED lock at `BEGIN`, so a competing writer is refused + /// immediately rather than at commit. + /// + /// The right choice for a read-then-write sequence that must not lose a + /// race after doing its reads. + Immediate, + /// Take the EXCLUSIVE lock at `BEGIN`, excluding readers too. + Exclusive, +} + +impl TransactionBehavior { + /// The `BEGIN` statement this behaviour issues. + fn statement(self) -> &'static str { + match self { + TransactionBehavior::Deferred => "BEGIN DEFERRED", + TransactionBehavior::Immediate => "BEGIN IMMEDIATE", + TransactionBehavior::Exclusive => "BEGIN EXCLUSIVE", + } + } +} + /// Why an API call failed. /// /// Deliberately flat: every payload is a `String`, an `i32` or a `Copy` @@ -438,6 +472,14 @@ enum Request { /// Where to send them. reply: SyncSender, }, + /// Set how long a contended lock is waited for. + SetBusyTimeout { + /// The new timeout. + timeout: Duration, + /// Acknowledgement, so the setting is in effect before the caller + /// continues. + reply: SyncSender<()>, + }, } /// How many rows the worker batches per channel send. @@ -665,6 +707,60 @@ impl Connection { Ok(first) } + /// Begins a deferred transaction. + /// + /// See [`Transaction`] — dropping the handle without committing rolls + /// back. + pub fn transaction(&self) -> Result { + self.transaction_with(TransactionBehavior::Deferred) + } + + /// Begins a transaction with the given locking behaviour. + pub fn transaction_with(&self, behavior: TransactionBehavior) -> Result { + self.execute(behavior.statement())?; + Ok(Transaction { + conn: self.clone(), + done: false, + }) + } + + /// Sets `PRAGMA = ` on this connection. + /// + /// A convenience over [`Connection::execute`] for the settings a pool + /// or a durability policy configures — `journal_mode`, `synchronous`. + /// Spec 013 scopes this to exactly that; the PRAGMA *catalogue* is + /// plan.md's V7, and the introspection pragmas (`table_info` and + /// friends) live in the CLI binary per ADR-0029, so they are not + /// reachable from here. + /// + /// `value` is interpolated into the statement, because that is the only + /// form SQLite accepts — `PRAGMA` does not take bound parameters. Pass + /// a literal from your own code, not something a user typed. + pub fn pragma(&self, name: &str, value: &str) -> Result<(), Error> { + self.execute(&format!("PRAGMA {name} = {value}")).map(drop) + } + + /// Sets how long a contended lock is waited for before + /// [`Error::Busy`] is returned. + /// + /// Zero — the default — means fail immediately, matching stock SQLite, + /// where `sqlite3_busy_timeout` is unset until asked for. + /// + /// The wait applies only to statements run *outside* an explicit + /// transaction. Inside one, a contended lock is reported straight away: + /// retrying a single statement of a transaction cannot be correct, + /// because its mutations sit in the same pending set as every earlier + /// statement's. Retry the transaction instead. Stock SQLite behaves the + /// same way with `SQLITE_BUSY` at `COMMIT`. + pub fn set_busy_timeout(&self, timeout: Duration) -> Result<(), Error> { + let (reply_tx, reply_rx) = sync_channel(0); + self.send(Request::SetBusyTimeout { + timeout, + reply: reply_tx, + })?; + self.recv(reply_rx) + } + /// Rows changed by the most recent counting statement, as /// `sqlite3_changes()` reports it. /// @@ -981,6 +1077,85 @@ fn storage_class(value: &Value) -> &'static str { } } +/// An open transaction on a [`Connection`]. +/// +/// Dropping this handle without calling [`Transaction::commit`] rolls the +/// transaction back. That is the point of the type: a `?` that returns early +/// out of a function holding one cannot leave a half-finished transaction +/// open, which is the failure mode a consumer storing pointers to data it +/// cannot otherwise find must not have. +/// +/// Derefs to its [`Connection`], so every `execute`/`query` method is +/// available on it directly and the statements run inside the transaction. +/// +/// # Durability +/// +/// What [`Transaction::commit`] guarantees is set by `PRAGMA synchronous` +/// (#645, ADR-0036), which this crate implements at all three levels: +/// +/// * `FULL` — the journal (or WAL frame) is fsynced before the commit +/// returns, so a committed transaction survives an OS crash or power +/// loss. This is what a pointer store should use. +/// * `NORMAL` — syncs are skipped at the points SQLite skips them; a +/// commit survives a *process* crash but not necessarily a power loss. +/// * `OFF` — no fsync. A commit survives a process crash only. +/// +/// This crate does not weaken the mode it is set to; the sync points are +/// `src/pager.rs`'s, and `PRAGMA synchronous` selects between them rather +/// than being accepted and ignored. +/// +/// Nesting is not supported (no `SAVEPOINT`): a second `BEGIN` while this +/// handle is open is refused by the engine. +#[derive(Debug)] +pub struct Transaction { + conn: Connection, + done: bool, +} + +impl Transaction { + /// Commits the transaction. + pub fn commit(mut self) -> Result<(), Error> { + self.done = true; + self.conn.execute("COMMIT").map(drop) + } + + /// Rolls the transaction back. + /// + /// The same thing dropping the handle does, but with the error + /// reported rather than discarded. + pub fn rollback(mut self) -> Result<(), Error> { + self.done = true; + self.conn.execute("ROLLBACK").map(drop) + } + + /// The connection this transaction runs on. + pub fn connection(&self) -> &Connection { + &self.conn + } +} + +impl std::ops::Deref for Transaction { + type Target = Connection; + + fn deref(&self) -> &Connection { + &self.conn + } +} + +impl Drop for Transaction { + fn drop(&mut self) { + if self.done { + return; + } + // Errors are unreportable from `drop`. Rolling back is the safe + // direction regardless: if this fails because the worker is already + // gone, the transaction was never committed either, so the database + // on disk is unchanged — which is the outcome a rollback wanted. + // A caller who needs to see the error calls `rollback()`. + self.conn.execute("ROLLBACK").ok(); + } +} + /// What the worker should open. enum Target { /// A real file, through `UnixVfs`. @@ -1007,6 +1182,12 @@ struct Engine { /// schema-changing statement is the conservative rule the CLI already /// uses (`src/bin/sqlite-rs/exec.rs::is_schema_changing`). catalog: Option<(Vec, Vec)>, + /// How long a contended lock is waited for before giving up. + /// + /// Zero by default, matching stock SQLite — `sqlite3_busy_timeout` is + /// unset until a caller sets it, and a facade that silently retried for + /// seconds would hide contention rather than report it. + busy_timeout: Duration, } /// Sends a reply, tolerating a caller that has stopped waiting. @@ -1060,6 +1241,10 @@ fn worker_main( Request::Counters { reply } => { answer(&reply, engine.counters); } + Request::SetBusyTimeout { timeout, reply } => { + engine.busy_timeout = timeout; + answer(&reply, ()); + } } } } @@ -1077,6 +1262,7 @@ impl Engine { autocommit: true, counters: Counters::default(), catalog: None, + busy_timeout: Duration::ZERO, }) } @@ -1102,9 +1288,9 @@ impl Engine { // `tests/corpus/bootstrap_oracle_test.rs`. let file = UnixVfs .create_or_open_write(path) - .map_err(|e| open_error(path, &e))?; + .map_err(|e| vfs_open_error(path, &e))?; file.write_at(&DatabaseHeader::new_empty_page1(DEFAULT_PAGE_SIZE), 0) - .map_err(|e| open_error(path, &e))?; + .map_err(|e| vfs_open_error(path, &e))?; } crate::dump::open(&UnixVfs, path).map_err(|e| open_error(path, &e)) @@ -1135,18 +1321,81 @@ impl Engine { }) } + /// Runs one statement, waiting out a contended lock up to + /// [`Engine::busy_timeout`]. + /// + /// Retrying is only correct while this connection is in autocommit. In + /// autocommit the statement *is* the transaction, so rolling the pager + /// back and running it again is a faithful retry of the whole unit. + /// Inside an explicit transaction it is not: the statement's own + /// mutations are already in the pager's dirty set alongside every + /// earlier statement's, and re-running one of them would double-apply + /// it. A `Busy` there is the *transaction's* to retry, which is also + /// what stock SQLite does with `SQLITE_BUSY` at `COMMIT`. + /// + /// The rollback before each retry is what makes this safe. + /// `Pager::flush` documents that a contended escalation surfaces + /// `VfsError::Locked` "before any byte of this transaction is journaled + /// or written" and leaves `dirty` intact for the caller to retry or + /// roll back (`src/pager.rs:524`) — so the dirty set at that point is + /// exactly this statement's work, and clearing it returns the engine to + /// the state the statement started from. + fn run_with_retry(&mut self, sql: &str, params: Vec) -> Result { + let deadline = Instant::now().checked_add(self.busy_timeout); + let mut attempt: u32 = 0; + loop { + let was_autocommit = self.autocommit; + match self.run(sql, params.clone()) { + Err(Error::Busy { path }) if was_autocommit => { + // Discard the half-applied statement before retrying. + if let Ok(mut pager) = self.pager.try_borrow_mut() { + pager.rollback().ok(); + } + let Some(delay) = self.backoff(deadline, attempt) else { + return Err(Error::Busy { path }); + }; + std::thread::sleep(delay); + attempt = attempt.saturating_add(1); + } + other => return other, + } + } + } + + /// How long to sleep before retry `attempt`, or `None` once the + /// deadline has passed. + /// + /// The ladder mirrors stock SQLite's default busy handler + /// (`sqliteDefaultBusyCallback`): short sleeps first so an + /// uncontended-in-practice lock is picked up almost immediately, + /// lengthening so a genuinely long-held lock is not spun on. + fn backoff(&self, deadline: Option, attempt: u32) -> Option { + const LADDER_MS: [u64; 7] = [1, 2, 5, 10, 20, 50, 100]; + let deadline = deadline?; + let now = Instant::now(); + let remaining = deadline.checked_duration_since(now)?; + if remaining.is_zero() { + return None; + } + let step = LADDER_MS + .get(attempt as usize) + .copied() + .unwrap_or_else(|| LADDER_MS.last().copied().unwrap_or(100)); + Some(Duration::from_millis(step).min(remaining)) + } + fn execute_one(&mut self, sql: &str, params: Vec) -> Result { let statements = crate::parser::split_statements(sql); let count = statements.len(); let Some(statement) = statements.into_iter().next().filter(|_| count == 1) else { return Err(Error::MultipleStatements { count }); }; - self.run(&statement, params) + self.run_with_retry(&statement, params) } fn execute_batch(&mut self, sql: &str) -> Result<(), Error> { for statement in crate::parser::split_statements(sql) { - self.run(&statement, Vec::new())?; + self.run_with_retry(&statement, Vec::new())?; } Ok(()) } @@ -1437,19 +1686,47 @@ fn starts_with_keyword(head: &str, keyword: &str) -> bool { .is_some_and(|h| h.eq_ignore_ascii_case(keyword)) } -fn open_error(path: &Path, e: &impl std::fmt::Display) -> Error { - let message = e.to_string(); - // spec 007's `VfsError::Locked` has to arrive as the busy variant even - // when it surfaces during open, since that is when a competing writer's - // lock is most likely to be met. - if message.contains("locked") { +/// Classifies a failure to open, distinguishing a contended lock from a +/// genuinely unopenable file. +/// +/// spec 007's `VfsError::Locked` has to arrive as the busy variant even +/// when it surfaces during open, since that is when a competing writer's +/// lock is most likely to be met. Matched structurally, for the reason +/// given on [`exec_error`]. +fn open_error(path: &Path, e: &crate::dump::DumpError) -> Error { + use crate::dump::DumpError; + use crate::pager::PagerError; + use crate::vfs::VfsError; + + let locked = matches!( + e, + DumpError::Vfs(VfsError::Locked { .. }) + | DumpError::Pager(PagerError::Vfs(VfsError::Locked { .. })) + ); + if locked { + return Error::Busy { + path: path.display().to_string(), + }; + } + Error::CannotOpen { + path: path.display().to_string(), + message: e.to_string(), + } +} + +/// The same classification for a raw `VfsError`, used on the bootstrap +/// path before a `Pager` exists. +fn vfs_open_error(path: &Path, e: &crate::vfs::VfsError) -> Error { + use crate::vfs::VfsError; + + if matches!(e, VfsError::Locked { .. }) { return Error::Busy { path: path.display().to_string(), }; } Error::CannotOpen { path: path.display().to_string(), - message, + message: e.to_string(), } } @@ -1476,7 +1753,10 @@ fn named_placeholder_of(message: &str) -> Option { } fn exec_error(e: crate::vdbe::ExecError) -> Error { + use crate::pager::PagerError; use crate::vdbe::ExecError; + use crate::vfs::VfsError; + match e { // The engine's route for constraint violations: `Halt` carries the // extended SQLite result code codegen chose. @@ -1484,14 +1764,15 @@ fn exec_error(e: crate::vdbe::ExecError) -> Error { code, message: message.unwrap_or_default(), }, - other => { - let message = other.to_string(); - if message.contains("locked") { - return Error::Busy { - path: String::new(), - }; - } - Error::Execution { message } - } + // Lock contention, matched structurally rather than by message + // text. Requirement 5 makes this a *distinct, retryable* variant, + // so classifying it on a substring would be exactly the wrong + // trade: a reworded `Display` would silently turn every busy error + // into a permanent one, and the caller's retry loop would vanish + // without a test failing. + ExecError::FlushFailed(PagerError::Vfs(VfsError::Locked { path })) => Error::Busy { path }, + other => Error::Execution { + message: other.to_string(), + }, } } diff --git a/tests/corpus/api_durability_oracle_test.rs b/tests/corpus/api_durability_oracle_test.rs new file mode 100644 index 00000000..2d99b41c --- /dev/null +++ b/tests/corpus/api_durability_oracle_test.rs @@ -0,0 +1,364 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! Retryable-busy and durability against a real second process (spec 013 +//! Requirement 5). +//! +//! These live here rather than in `tests/unit/api_durability_test.rs` +//! because they need a *separate process* to hold the lock. Two connections +//! in one process do not exclude each other — POSIX `fcntl` locks are +//! scoped to `(process, inode)` and this crate has no +//! `unixInodeInfo`-equivalent registry — which is recorded, measured and +//! ratcheted in that unit module. The pinned `sqlite3` is the second party +//! here, which also makes the claim stronger: the lock protocol is being +//! honoured against stock SQLite, not merely against ourselves. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing +)] + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use sqlite_rs::api::{Connection, Error}; + +use crate::oracle::{pinned_oracle, skip_no_oracle}; + +fn scratch_dir(label: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "sqlite-rs-api-dur-oracle-{}-{label}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn prepared(dir: &Path) -> PathBuf { + let db = dir.join("t.db"); + let conn = Connection::open(&db).unwrap(); + conn.execute_batch("CREATE TABLE t(a INTEGER, b TEXT); INSERT INTO t VALUES (1, 'x');") + .unwrap(); + db +} + +/// A separate `sqlite3` process holding a write transaction open. +/// +/// It begins `IMMEDIATE` (so RESERVED is taken at `BEGIN` rather than at +/// the first write) and then waits on stdin, which keeps the lock held for +/// as long as this handle lives. +struct LockHolder { + child: Child, +} + +impl LockHolder { + fn new(oracle: &Path, db: &Path) -> Self { + let mut child = Command::new(oracle) + .arg(db) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("could not spawn the oracle as a lock holder"); + { + let stdin = child.stdin.as_mut().expect("piped stdin"); + stdin + .write_all( + b"BEGIN IMMEDIATE;\nINSERT INTO t VALUES (2, 'held');\nSELECT 'holding';\n", + ) + .expect("could not start the holder's transaction"); + stdin.flush().expect("could not flush to the holder"); + } + // Give it time to actually take the lock before anyone tests for + // contention. A sleep rather than a poll because the thing to wait + // for *is* the lock, and probing for it is what the callers do. + std::thread::sleep(Duration::from_millis(300)); + Self { child } + } + + /// Commits and exits, releasing the lock. + fn release(mut self) { + if let Some(stdin) = self.child.stdin.as_mut() { + stdin.write_all(b"COMMIT;\n.quit\n").ok(); + stdin.flush().ok(); + } + self.child.wait().ok(); + // Already waited; stop `Drop` from killing a reaped child. + std::mem::forget(self); + } +} + +impl Drop for LockHolder { + fn drop(&mut self) { + self.child.kill().ok(); + self.child.wait().ok(); + } +} + +fn count(conn: &Connection) -> i64 { + conn.query_row("SELECT count(*) FROM t") + .unwrap() + .expect("count returns a row") + .get(0) + .unwrap() +} + +/// Requirement 5's third scenario. +#[test] +fn busy_is_retryable() { + let Some(oracle) = pinned_oracle() else { + skip_no_oracle("busy_is_retryable"); + return; + }; + let dir = scratch_dir("busy"); + let db = prepared(&dir); + + let holder = LockHolder::new(&oracle, &db); + + let conn = Connection::open(&db).unwrap(); + let err = conn + .execute("INSERT INTO t VALUES (3, 'blocked')") + .expect_err("another process holds the write lock"); + + assert!( + matches!(err, Error::Busy { .. }), + "expected Error::Busy, got {err:?}" + ); + assert!(err.is_retryable(), "a busy error must be retryable"); + assert_eq!(err.sqlite_code(), 5, "should report SQLITE_BUSY"); + + holder.release(); + + // The retry succeeds, and both writes are present. + conn.execute("INSERT INTO t VALUES (3, 'retried')") + .expect("the retry should succeed once the lock is released"); + assert_eq!(count(&conn), 3); + + std::fs::remove_dir_all(&dir).ok(); +} + +/// With a timeout set, a contended write waits it out rather than failing +/// immediately — and still reports `Busy` when the lock outlives the wait. +#[test] +fn a_busy_timeout_waits_before_giving_up() { + let Some(oracle) = pinned_oracle() else { + skip_no_oracle("a_busy_timeout_waits_before_giving_up"); + return; + }; + let dir = scratch_dir("wait"); + let db = prepared(&dir); + + let holder = LockHolder::new(&oracle, &db); + + let conn = Connection::open(&db).unwrap(); + conn.set_busy_timeout(Duration::from_millis(400)).unwrap(); + + let start = Instant::now(); + let err = conn + .execute("INSERT INTO t VALUES (3, 'blocked')") + .expect_err("the lock is held for the whole timeout"); + let waited = start.elapsed(); + + assert!(matches!(err, Error::Busy { .. }), "got {err:?}"); + assert!( + waited >= Duration::from_millis(300), + "should have waited out most of the 400ms timeout, waited only {waited:?}" + ); + assert!( + waited < Duration::from_secs(10), + "waited {waited:?}, far past the timeout — the deadline is not honoured" + ); + + drop(holder); + std::fs::remove_dir_all(&dir).ok(); +} + +/// The retry loop picks the lock up when it is released mid-wait, so the +/// caller does not write a retry loop of its own — and the statement is +/// applied exactly once, not once per attempt. +/// +/// The exactly-once half is the one that matters. In autocommit the +/// statement's mutations are already in the pager's dirty set when the +/// commit meets the lock, so a retry that did not roll back first would +/// insert the row again for every attempt. +#[test] +fn a_retried_statement_succeeds_exactly_once() { + let Some(oracle) = pinned_oracle() else { + skip_no_oracle("a_retried_statement_succeeds_exactly_once"); + return; + }; + let dir = scratch_dir("once"); + let db = prepared(&dir); + + let holder = LockHolder::new(&oracle, &db); + + let conn = Connection::open(&db).unwrap(); + conn.set_busy_timeout(Duration::from_secs(10)).unwrap(); + + // Release the lock while the write below is retrying. + let releaser = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(250)); + holder.release(); + }); + + let start = Instant::now(); + let changed = conn + .execute("INSERT INTO t VALUES (99, 'retried')") + .expect("should succeed once the holder commits"); + let waited = start.elapsed(); + + releaser.join().expect("the releasing thread panicked"); + + assert_eq!(changed, 1, "the retried statement should report one row"); + assert!( + waited >= Duration::from_millis(150), + "should have actually waited for the lock, waited {waited:?}" + ); + + let applied: i64 = conn + .query_row("SELECT count(*) FROM t WHERE a = 99") + .unwrap() + .unwrap() + .get(0) + .unwrap(); + assert_eq!( + applied, 1, + "the retried INSERT landed {applied} times, not once — the rollback \ + before retry is missing or ineffective" + ); + + std::fs::remove_dir_all(&dir).ok(); +} + +/// Requirement 5's second scenario: a transaction committed under +/// `synchronous = FULL` survives the process being killed without +/// unwinding. +/// +/// The child is this same test binary re-executed with +/// `SQLITE_RS_HARDKILL_DB` set: it commits, writes a marker file so the +/// parent knows the commit returned, and then blocks forever. The parent +/// SIGKILLs it (`Child::kill` is SIGKILL on Unix), so no destructor, no +/// deferred flush and no unwinding runs — the only thing that can make the +/// rows survive is the commit having completed before it returned. +/// +/// What this does and does not prove, stated precisely because the +/// difference matters for a durability claim. It proves the commit was +/// complete and consistent *in the file* rather than buffered inside the +/// process, and that an abrupt death leaves nothing malformed. It does +/// **not** prove platter durability: SIGKILL does not clear the kernel's +/// page cache, so it cannot distinguish `synchronous = FULL` from `NORMAL` +/// or `OFF`. Only a power cut or a crash-injecting VFS separates those, and +/// `tests/corpus/crash_torture_test.rs` is where that regime lives. +/// `synchronous = FULL` is set here so this exercises the path a durable +/// consumer configures, not to claim the test verifies the fsync. +#[test] +fn commit_survives_hard_kill() { + if let Ok(db) = std::env::var("SQLITE_RS_HARDKILL_DB") { + // Diverges: the child blocks until it is killed. + hard_kill_child(Path::new(&db)); + } + let Some(oracle) = pinned_oracle() else { + skip_no_oracle("commit_survives_hard_kill"); + return; + }; + let dir = scratch_dir("hardkill"); + let db = dir.join("kill.db"); + let marker = dir.join("committed"); + + let exe = std::env::current_exe().expect("the test binary's own path"); + let mut child = Command::new(exe) + .arg("--exact") + .arg("api_durability_oracle_test::commit_survives_hard_kill") + .arg("--nocapture") + .env("SQLITE_RS_HARDKILL_DB", &db) + .env("SQLITE_RS_HARDKILL_MARKER", &marker) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("could not re-execute this test binary as the child"); + + // Wait for the child to report that its COMMIT returned. + let deadline = Instant::now() + Duration::from_secs(30); + while !marker.exists() { + if Instant::now() > deadline { + child.kill().ok(); + child.wait().ok(); + panic!("the child never reported a completed commit"); + } + if let Ok(Some(status)) = child.try_wait() { + panic!("the child exited early with {status:?} instead of committing"); + } + std::thread::sleep(Duration::from_millis(20)); + } + + // SIGKILL: no unwinding, no destructors, no deferred flush. + child.kill().expect("could not kill the child"); + child.wait().ok(); + + // The rows have to be there, read back by a fresh connection... + let conn = Connection::open(&db).unwrap(); + let survivors: i64 = conn + .query_row("SELECT count(*) FROM t") + .unwrap() + .expect("count returns a row") + .get(0) + .unwrap(); + assert_eq!( + survivors, 50, + "a transaction committed under synchronous=FULL did not survive SIGKILL" + ); + drop(conn); + + // ...and the file has to be well-formed to stock sqlite3, not merely + // readable by us. A hard kill mid-journal is exactly how a malformed + // file happens. + let output = Command::new(&oracle) + .arg(&db) + .arg("PRAGMA integrity_check;") + .output() + .expect("could not run the oracle"); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + "ok", + "the killed process left the database malformed" + ); + + std::fs::remove_dir_all(&dir).ok(); +} + +/// The child half of [`commit_survives_hard_kill`]: commit durably, say so, +/// then wait to be killed. +fn hard_kill_child(db: &Path) -> ! { + let conn = Connection::open(db).expect("child could not open the database"); + conn.execute("CREATE TABLE t(a INTEGER, b TEXT)") + .expect("child could not create the table"); + conn.pragma("synchronous", "FULL") + .expect("child could not set synchronous=FULL"); + + let tx = conn.transaction().expect("child could not begin"); + for i in 0..50 { + tx.execute_with( + "INSERT INTO t VALUES (?1, ?2)", + vec![ + sqlite_rs::record::Value::from(i), + sqlite_rs::record::Value::from("durable"), + ], + ) + .expect("child could not insert"); + } + tx.commit().expect("child could not commit"); + + // The commit has returned. Under synchronous=FULL that must mean the + // bytes are on the platter. + if let Ok(marker) = std::env::var("SQLITE_RS_HARDKILL_MARKER") { + std::fs::write(marker, b"committed").expect("child could not write its marker"); + } + + // Wait to be killed. Nothing after this point may run. + loop { + std::thread::sleep(Duration::from_secs(3600)); + } +} diff --git a/tests/corpus/main.rs b/tests/corpus/main.rs index 7b7af5aa..161a7b84 100644 --- a/tests/corpus/main.rs +++ b/tests/corpus/main.rs @@ -18,6 +18,7 @@ mod harness; mod oracle; mod analyze_test; +mod api_durability_oracle_test; mod api_oracle_test; mod autoindex_maintenance_test; mod begin_immediate_lock_interop_test; diff --git a/tests/unit/api_durability_test.rs b/tests/unit/api_durability_test.rs new file mode 100644 index 00000000..76bf0e2d --- /dev/null +++ b/tests/unit/api_durability_test.rs @@ -0,0 +1,162 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! The durability and retryable-busy contract (spec 013 Requirement 5), +//! for the parts observable without a second process. +//! +//! The busy variant itself is exercised in +//! `tests/corpus/api_durability_oracle_test.rs`, against a real second +//! process. It has to be, and the reason is a finding rather than a +//! convenience: +//! +//! # Two connections in one process do not lock against each other +//! +//! POSIX `fcntl` locks are scoped to `(process, inode)`, not to the open +//! file description — which this crate's own `src/vfs/lock.rs:96` documents +//! and `FileLockState::check_reserved_lock` states outright ("whether some +//! *other* process currently holds a write lock"). So a second `Connection` +//! on the same path in the same process escalates to EXCLUSIVE without +//! conflict, even while the first holds RESERVED. +//! +//! Measured on this tree: connection A takes `BEGIN IMMEDIATE` and inserts +//! row 2; connection B's insert of row 3 returns `Ok(1)`; A commits; the +//! file then contains rows `[1, 2]`. Row 3 is silently gone, and +//! `PRAGMA integrity_check` reports `ok`, so nothing flags it. +//! +//! Stock SQLite prevents exactly this with `unixInodeInfo` in `os_unix.c` — +//! a process-global registry keyed by `(device, inode)` carrying its own +//! mutex and lock counts, so two connections in one process serialize the +//! same way two processes do. This crate has no equivalent. +//! +//! That is a pre-existing engine gap, not something the facade introduced, +//! but the facade makes it far easier to reach: Requirement 4 exists so a +//! *pool* can hold a handle, and opening the same path twice is the +//! obvious thing to do. `in_process_connections_lock_against_each_other` +//! below is the ratchet, `#[ignore]`d until the registry exists. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use sqlite_rs::api::{Connection, TransactionBehavior}; + +fn scratch(label: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("sqlite-rs-api-dur-{}-{label}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + dir.join("test.db") +} + +fn clean(path: &Path) { + if let Some(dir) = path.parent() { + std::fs::remove_dir_all(dir).ok(); + } +} + +fn prepared(label: &str) -> PathBuf { + let path = scratch(label); + clean(&path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch("CREATE TABLE t(a INTEGER, b TEXT); INSERT INTO t VALUES (1, 'x');") + .unwrap(); + path +} + +fn count(conn: &Connection) -> i64 { + conn.query_row("SELECT count(*) FROM t") + .unwrap() + .expect("count returns a row") + .get(0) + .unwrap() +} + +#[test] +fn the_busy_timeout_is_settable_and_defaults_to_zero() { + let conn = Connection::open_in_memory().unwrap(); + // Accepted at any value, and setting it does not disturb the + // connection. The default is zero, matching stock SQLite, where + // `sqlite3_busy_timeout` is unset until asked for — there is no getter + // here because SQLite has none either. + conn.set_busy_timeout(Duration::from_millis(250)).unwrap(); + conn.set_busy_timeout(Duration::ZERO).unwrap(); + conn.set_busy_timeout(Duration::from_secs(30)).unwrap(); + + conn.execute("CREATE TABLE t(a INTEGER, b TEXT)").unwrap(); + conn.execute("INSERT INTO t VALUES (1, 'x')").unwrap(); + assert_eq!(count(&conn), 1); +} + +/// `PRAGMA synchronous` is the durability knob Requirement 5 names, and all +/// three levels are implemented (#645, ADR-0036). This pins that the facade +/// can reach them and that writes still land afterwards. +#[test] +fn the_durability_knob_is_reachable_and_writes_survive_each_level() { + for level in ["FULL", "NORMAL", "OFF"] { + let path = prepared("sync"); + { + let conn = Connection::open(&path).unwrap(); + conn.pragma("synchronous", level) + .unwrap_or_else(|e| panic!("PRAGMA synchronous = {level} failed: {e}")); + + let tx = conn.transaction().unwrap(); + tx.execute("INSERT INTO t VALUES (2, 'y')").unwrap(); + tx.commit().unwrap(); + } + // Reopened: the commit reached the file, whichever level was set. + let conn = Connection::open(&path).unwrap(); + assert_eq!( + count(&conn), + 2, + "a commit under synchronous={level} was lost" + ); + clean(&path); + } +} + +/// **Ratchet, currently failing by construction.** +/// +/// Two connections in one process must exclude each other's writes the way +/// two processes do. They do not: see this module's documentation for the +/// measurement and for `unixInodeInfo`, the mechanism stock SQLite uses. +/// +/// Un-`#[ignore]` this when the process-global inode registry lands. It is +/// written to assert the *correct* behaviour, so it will pass at that point +/// without being rewritten — the ratchet convention this repo uses for +/// spike findings (ADR-0008). +#[test] +#[ignore = "in-process connections do not lock against each other (POSIX fcntl is per-process); needs a unixInodeInfo-equivalent registry — see this module's docs"] +fn in_process_connections_lock_against_each_other() { + let path = prepared("in-process"); + + let a = Connection::open(&path).unwrap(); + let b = Connection::open(&path).unwrap(); + + let tx = a.transaction_with(TransactionBehavior::Immediate).unwrap(); + tx.execute("INSERT INTO t VALUES (2, 'a')").unwrap(); + + // Either of these outcomes is correct; silently succeeding and then + // losing the row is not. + match b.execute("INSERT INTO t VALUES (3, 'b')") { + Err(e) => assert!( + e.is_retryable(), + "a contended write should be retryable, got {e:?}" + ), + Ok(_) => { + // If it is allowed, the row must actually survive. + tx.commit().unwrap(); + let survivors: i64 = b + .query_row("SELECT count(*) FROM t WHERE a = 3") + .unwrap() + .unwrap() + .get(0) + .unwrap(); + assert_eq!( + survivors, 1, + "a write that reported success was silently discarded" + ); + } + } + + clean(&path); +} diff --git a/tests/unit/api_transaction_test.rs b/tests/unit/api_transaction_test.rs new file mode 100644 index 00000000..272b083f --- /dev/null +++ b/tests/unit/api_transaction_test.rs @@ -0,0 +1,228 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! Transactions on the embedding API (spec 013 Requirement 5). +//! +//! The property that matters is not that `BEGIN` works — the engine has had +//! that since #356 — but that a transaction *handle* cannot be left open by +//! accident. A `?` returning early out of a function holding one must roll +//! back, because the alternative for a consumer storing pointers to data it +//! cannot otherwise find is a half-applied catalog change. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::path::{Path, PathBuf}; + +use sqlite_rs::api::{Connection, Error, TransactionBehavior}; +use sqlite_rs::record::Value; + +fn scratch(label: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("sqlite-rs-api-tx-{}-{label}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + dir.join("test.db") +} + +fn clean(path: &Path) { + if let Some(dir) = path.parent() { + std::fs::remove_dir_all(dir).ok(); + } +} + +fn count(conn: &Connection) -> i64 { + conn.query_row("SELECT count(*) FROM t") + .unwrap() + .expect("count returns a row") + .get(0) + .unwrap() +} + +fn seeded() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch("CREATE TABLE t(a INTEGER, b TEXT); INSERT INTO t VALUES (1, 'x');") + .unwrap(); + conn +} + +#[test] +fn a_committed_transaction_keeps_its_writes() { + let conn = seeded(); + let tx = conn.transaction().unwrap(); + tx.execute("INSERT INTO t VALUES (2, 'y')").unwrap(); + tx.execute("INSERT INTO t VALUES (3, 'z')").unwrap(); + tx.commit().unwrap(); + + assert_eq!(count(&conn), 3); +} + +/// Requirement 5's first scenario. +#[test] +fn drop_rolls_back() { + let conn = seeded(); + { + let tx = conn.transaction().unwrap(); + tx.execute("INSERT INTO t VALUES (2, 'y')").unwrap(); + assert_eq!(count(&tx), 2, "the write is visible inside the transaction"); + // Dropped without commit. + } + assert_eq!( + count(&conn), + 1, + "the dropped transaction should have rolled back" + ); +} + +#[test] +fn an_explicit_rollback_reports_its_errors() { + let conn = seeded(); + let tx = conn.transaction().unwrap(); + tx.execute("INSERT INTO t VALUES (2, 'y')").unwrap(); + tx.rollback().unwrap(); + + assert_eq!(count(&conn), 1); +} + +/// The reason the type exists: an early return must not leave a +/// transaction open, and the next statement must not silently join it. +#[test] +fn an_early_return_rolls_back_and_leaves_the_connection_usable() { + let conn = seeded(); + + fn fallible(conn: &Connection) -> Result<(), Error> { + let tx = conn.transaction()?; + tx.execute("INSERT INTO t VALUES (2, 'y')")?; + // Fails: no such table. The `?` returns while `tx` is live. + tx.execute("INSERT INTO nope VALUES (1)")?; + tx.commit() + } + + assert!(fallible(&conn).is_err()); + assert_eq!( + count(&conn), + 1, + "the failed unit of work should be entirely absent" + ); + + // And the connection is back in autocommit, not stuck in a transaction. + conn.execute("INSERT INTO t VALUES (9, 'ok')").unwrap(); + assert_eq!(count(&conn), 2); +} + +#[test] +fn all_three_behaviours_begin_and_commit() { + for behavior in [ + TransactionBehavior::Deferred, + TransactionBehavior::Immediate, + TransactionBehavior::Exclusive, + ] { + let conn = seeded(); + let tx = conn.transaction_with(behavior).unwrap(); + tx.execute("INSERT INTO t VALUES (2, 'y')").unwrap(); + tx.commit() + .unwrap_or_else(|e| panic!("{behavior:?} failed to commit: {e}")); + assert_eq!(count(&conn), 2, "{behavior:?} lost its write"); + } +} + +/// Nesting is not supported, and the refusal must be an error rather than a +/// silently flattened second transaction. +#[test] +fn a_nested_transaction_is_refused() { + let conn = seeded(); + let _outer = conn.transaction().unwrap(); + let inner = conn.transaction(); + assert!( + inner.is_err(), + "a second BEGIN while one is open should be refused" + ); +} + +/// A transaction spanning several statements is one unit on disk, checked +/// by reopening the file rather than by asking the same connection. +#[test] +fn a_transaction_is_one_unit_on_disk() { + let path = scratch("unit"); + clean(&path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + + { + let conn = Connection::open(&path).unwrap(); + conn.execute("CREATE TABLE t(a INTEGER, b TEXT)").unwrap(); + + let tx = conn.transaction().unwrap(); + for i in 0..20 { + tx.execute_with( + "INSERT INTO t VALUES (?1, ?2)", + vec![Value::from(i), Value::from("x")], + ) + .unwrap(); + } + tx.commit().unwrap(); + } + + let reopened = Connection::open(&path).unwrap(); + assert_eq!(count(&reopened), 20); + + // And a rolled-back one leaves nothing behind on disk either. + { + let tx = reopened.transaction().unwrap(); + tx.execute("INSERT INTO t VALUES (999, 'gone')").unwrap(); + drop(tx); + } + drop(reopened); + + let reopened = Connection::open(&path).unwrap(); + assert_eq!(count(&reopened), 20, "a rolled-back write reached the file"); + + clean(&path); +} + +#[test] +fn pragma_sets_a_value_the_engine_honours() { + let path = scratch("pragma"); + clean(&path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + + let conn = Connection::open(&path).unwrap(); + // Both levels of the durability knob Requirement 5 names. + conn.pragma("synchronous", "FULL").unwrap(); + conn.pragma("synchronous", "OFF").unwrap(); + conn.pragma("synchronous", "NORMAL").unwrap(); + + // And the statement still works afterwards. + conn.execute("CREATE TABLE t(a INTEGER, b TEXT)").unwrap(); + conn.execute("INSERT INTO t VALUES (1, 'x')").unwrap(); + assert_eq!(count(&conn), 1); + + clean(&path); +} + +#[test] +fn the_busy_timeout_is_settable() { + let conn = Connection::open_in_memory().unwrap(); + // Default is zero (stock SQLite's), and any value is accepted. + conn.set_busy_timeout(std::time::Duration::from_millis(250)) + .unwrap(); + conn.set_busy_timeout(std::time::Duration::ZERO).unwrap(); + + // Setting it does not disturb the connection. + conn.execute("CREATE TABLE t(a INTEGER, b TEXT)").unwrap(); + conn.execute("INSERT INTO t VALUES (1, 'x')").unwrap(); + assert_eq!(count(&conn), 1); +} + +/// The counters follow the transaction: a rolled-back `INSERT` still +/// *reported* its row, because `sqlite3_changes()` counts what a statement +/// did rather than what survived. Pinned so a future change to the rollback +/// path does not quietly redefine it. +#[test] +fn a_rolled_back_statement_still_reported_its_count() { + let conn = seeded(); + { + let tx = conn.transaction().unwrap(); + assert_eq!(tx.execute("INSERT INTO t VALUES (2, 'y')").unwrap(), 1); + assert_eq!(tx.changes().unwrap(), 1); + } + // The row is gone... + assert_eq!(count(&conn), 1); + // ...but the count is not retroactively revised, matching SQLite. + assert_eq!(conn.changes().unwrap(), 1); +} From b27d25f6f51e2eb422008d64689ff92590d602bd Mon Sep 17 00:00:00 2001 From: Diederik Siderius Date: Wed, 9 Sep 2026 21:05:15 +0200 Subject: [PATCH 09/14] feat: prepared statements and schema refresh on the embedding API (013/Req 3, Req 8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Connection::prepare` returns a `Statement` that owns its compiled program on the worker and finalizes it on drop. `execute`/`query`/`query_all`/ `query_row` take the parameters; `param_count` and `column_names` are available before it runs. Requirement 3 is explicit that the value is not speed — a dozen statements at commit frequency saves nothing measurable by compiling once. It is that a handle owning its parameter slots refuses a wrong argument count instead of writing a valid row pointing at the wrong table. So the arity check runs on every execution, not just at prepare. Kept statements take the *same* path as ad-hoc ones: `run_compiled` and `check_runnable` are shared, because a second copy of the mode and arity checks is how a prepared statement ends up honouring a different contract from `execute`. Schema refresh (Req 8) ---------------------- A program addresses tables by root page, and a `DROP` can return that page to the freelist for a later `CREATE` to reuse — so a statement compiled before a schema change and run after it could read a page that now belongs to a different table. The engine carries a schema generation, bumped whenever the catalog is invalidated; a statement compiled against an older one is recompiled on next use, which is what `sqlite3_prepare_v2` does on `SQLITE_SCHEMA`. If it no longer compiles at all (its table was dropped) the failure is reported and the handle stays valid, so the error is repeatable rather than one-shot. `Statement::reprepare_count` exposes how often that happened. Not test scaffolding: it is SQLite's own `SQLITE_STMTSTATUS_REPREPARE`, documented as "the number of times that the prepared statement has been automatically regenerated due to schema changes" (`sqlite3.h:9274`, pinned 3.53.4). It is also what makes Requirement 3's "compilation happened once" an observable claim rather than an assertion about internals. Tests, and one claim I had to move to make it real -------------------------------------------------- 12 unit tests in `tests/unit/api_statement_test.rs`, using Requirement 3's scenario names. Confirmed discriminating: with the refresh disabled, 3 of them fail. The load-bearing Req 8 claim — that a write prepared before an index existed still maintains that index — turned out *not* to be provable in a unit test, and I had claimed it was. A stale program inserts the table row and skips the index, but a freshly-compiled read may table-scan and find the row anyway, so the assertion passed under the mutant. Measured, not assumed. It now lives where it can be checked: `tests/corpus/api_oracle_test.rs::a_prepared_write_after_create_index_keeps_the_file_valid` inserts through a statement prepared before two indexes existed and has the pinned sqlite3 run `PRAGMA integrity_check` — which is precisely what detects a row present in the table with no matching index entry. That test does fail under the mutant. #685 is the precedent: this class of bug was invisible until the oracle was asked. The unit test's comment now says what it does and does not show, and points at the corpus test. Gates: make test (1664 passed, 10 ignored), make test-corpus (404 passed), make lint (both clippy passes + fmt), check-mod-files, check-assurance 100%/100%. Spend: within estimate. Co-Authored-By: Claude Opus 5 --- Cargo.toml | 4 + src/api.rs | 432 +++++++++++++++++++++++++++++-- tests/corpus/api_oracle_test.rs | 77 ++++++ tests/unit/api_statement_test.rs | 345 ++++++++++++++++++++++++ 4 files changed, 837 insertions(+), 21 deletions(-) create mode 100644 tests/unit/api_statement_test.rs diff --git a/Cargo.toml b/Cargo.toml index 61dcd66c..1a56a9e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -172,6 +172,10 @@ path = "tests/unit/api_transaction_test.rs" name = "api_durability" path = "tests/unit/api_durability_test.rs" +[[test]] +name = "api_statement" +path = "tests/unit/api_statement_test.rs" + [[test]] name = "codegen_expr" path = "tests/unit/codegen_expr_test.rs" diff --git a/src/api.rs b/src/api.rs index 3a830e7c..2c1ffdb1 100644 --- a/src/api.rs +++ b/src/api.rs @@ -268,6 +268,12 @@ pub enum Error { /// How many columns the row has. len: usize, }, + /// A [`Statement`] was used after its worker discarded it. + /// + /// Not reachable while the handle is alive — `Statement` owns its + /// registration and finalizes it on drop — so this is a guard against + /// an internal inconsistency rather than a caller error. + StatementFinalized, /// Execution failed for a reason with no more specific variant. Execution { /// What went wrong. @@ -301,7 +307,9 @@ impl Error { Error::ParamCount { .. } | Error::ColumnIndexOutOfRange { .. } => code::RANGE, Error::TypeMismatch { .. } => code::MISMATCH, Error::ColumnNotFound { .. } => code::ERROR, - Error::MultipleStatements { .. } | Error::ConnectionClosed => code::MISUSE, + Error::MultipleStatements { .. } + | Error::ConnectionClosed + | Error::StatementFinalized => code::MISUSE, Error::Sqlite { code, .. } => *code, Error::Busy { .. } => code::BUSY, Error::CannotOpen { .. } => code::CANTOPEN, @@ -363,6 +371,7 @@ impl std::fmt::Display for Error { Error::Corrupt { message } => write!(f, "database image is malformed: {message}"), Error::Io { message } => write!(f, "I/O error: {message}"), Error::ConnectionClosed => write!(f, "connection is closed"), + Error::StatementFinalized => write!(f, "statement has been finalized"), Error::TypeMismatch { column, expected, @@ -472,6 +481,46 @@ enum Request { /// Where to send them. reply: SyncSender, }, + /// Compile one statement and keep it. + Prepare { + /// The statement text. + sql: String, + /// Where to send the handle, or the failure to compile. + reply: SyncSender>, + }, + /// Run a kept statement, discarding rows. + StatementExecute { + /// Which statement. + id: u64, + /// Values for its placeholders. + params: Vec, + /// Where to send the outcome. + reply: SyncSender>, + }, + /// Run a kept statement and stream its rows. + StatementQuery { + /// Which statement. + id: u64, + /// Values for its placeholders. + params: Vec, + /// Where to send the stream's head. + reply: SyncSender>, + }, + /// How many times a kept statement has been recompiled. + StatementRepreparations { + /// Which statement. + id: u64, + /// Where to send the count. + reply: SyncSender>, + }, + /// Discard a kept statement. + /// + /// No reply: `Statement::drop` cannot wait on one usefully, and there + /// is nothing a caller could do with the answer. + Finalize { + /// Which statement. + id: u64, + }, /// Set how long a contended lock is waited for. SetBusyTimeout { /// The new timeout. @@ -511,6 +560,27 @@ const CHUNK_ROWS: usize = 64; /// That is inherent to one worker streaming one execution — see [`Rows`]. const CHUNK_SLOTS: usize = 2; +/// What the worker returns when it has compiled and kept a statement. +struct PreparedHandle { + id: u64, + param_count: usize, + column_names: Arc>, +} + +/// One compiled statement the worker is holding for a [`Statement`]. +struct KeptStatement { + /// Kept so the statement can be recompiled after a schema change, and + /// so `is_schema_changing` can be re-evaluated on each run. + sql: String, + program: Program, + column_names: Arc>, + /// The schema generation this was compiled against. + generation: u64, + /// How many times it has been recompiled — SQLite's + /// `SQLITE_STMTSTATUS_REPREPARE`. + repreparations: u64, +} + /// The head of a streamed result: its column names, and the channel its /// rows arrive on. struct QueryStream { @@ -707,6 +777,29 @@ impl Connection { Ok(first) } + /// Compiles one statement and keeps it, so it can be run repeatedly + /// with different parameters. + /// + /// The value is not speed: for a dozen statements at commit frequency, + /// compiling once saves nothing measurable. It is that a handle owning + /// its parameter slots reports a wrong argument count instead of + /// writing a valid row that points at the wrong thing (spec 013 + /// Requirement 3). + pub fn prepare(&self, sql: &str) -> Result { + let (reply_tx, reply_rx) = sync_channel(0); + self.send(Request::Prepare { + sql: sql.to_string(), + reply: reply_tx, + })?; + let handle = self.recv(reply_rx)??; + Ok(Statement { + conn: self.clone(), + id: handle.id, + param_count: handle.param_count, + column_names: handle.column_names, + }) + } + /// Begins a deferred transaction. /// /// See [`Transaction`] — dropping the handle without committing rolls @@ -1156,6 +1249,122 @@ impl Drop for Transaction { } } +/// A compiled statement, ready to run with parameters. +/// +/// Owns its registration on the connection's worker and finalizes it on +/// drop. Holds a clone of the [`Connection`], so the worker stays alive for +/// as long as any statement does. +/// +/// # Schema changes +/// +/// A statement recompiles itself if the schema changed since it was +/// prepared, which is what `sqlite3_prepare_v2` does on `SQLITE_SCHEMA`. +/// That is not a convenience: a program addresses tables by root page, and +/// a `DROP` can return that page to the freelist for a later `CREATE` to +/// reuse — so running a stale program could read a page belonging to a +/// different table. [`Statement::reprepare_count`] reports how often it has +/// happened, mirroring `SQLITE_STMTSTATUS_REPREPARE`. +#[derive(Debug)] +pub struct Statement { + conn: Connection, + id: u64, + param_count: usize, + column_names: Arc>, +} + +impl Statement { + /// How many parameters this statement reads. + /// + /// The largest `?NNN` index it uses, matching + /// `sqlite3_bind_parameter_count` — so `?3` alone reports 3. + pub fn param_count(&self) -> usize { + self.param_count + } + + /// This statement's result column names. + /// + /// Subject to the same limit as [`Rows::column_names`]: real names only + /// for a single-table `SELECT`. + pub fn column_names(&self) -> &[String] { + &self.column_names + } + + /// Runs the statement with `params` bound, returning rows changed. + pub fn execute(&self, params: Vec) -> Result { + let (reply_tx, reply_rx) = sync_channel(0); + self.conn.send(Request::StatementExecute { + id: self.id, + params, + reply: reply_tx, + })?; + let applied = self.conn.recv(reply_rx)??; + Ok(applied.changes.unwrap_or(0)) + } + + /// Runs the statement with `params` bound and streams its rows. + /// + /// The same caveat as [`Connection::query`]: see [`Rows`]. + pub fn query(&self, params: Vec) -> Result { + let (reply_tx, reply_rx) = sync_channel(0); + self.conn.send(Request::StatementQuery { + id: self.id, + params, + reply: reply_tx, + })?; + let stream = self.conn.recv(reply_rx)??; + Ok(Rows { + column_names: stream.column_names, + chunks: stream.chunks, + buffered: Vec::new().into_iter(), + finished: false, + }) + } + + /// Runs the statement and collects every row. + pub fn query_all(&self, params: Vec) -> Result, Error> { + self.query(params)?.into_vec() + } + + /// Runs the statement and returns its first row, if any. + pub fn query_row(&self, params: Vec) -> Result, Error> { + let mut rows = self.query(params)?; + let first = rows.next_row()?; + drop(rows); + Ok(first) + } + + /// How many times this statement has been recompiled because the + /// schema changed under it. + /// + /// SQLite's `SQLITE_STMTSTATUS_REPREPARE`: "the number of times that + /// the prepared statement has been automatically regenerated due to + /// schema changes". Zero for a statement whose schema has held still, + /// which is what makes "compiled once" an observable claim rather than + /// an assertion about internals. + pub fn reprepare_count(&self) -> Result { + let (reply_tx, reply_rx) = sync_channel(0); + self.conn.send(Request::StatementRepreparations { + id: self.id, + reply: reply_tx, + })?; + self.conn.recv(reply_rx)? + } + + /// The connection this statement belongs to. + pub fn connection(&self) -> &Connection { + &self.conn + } +} + +impl Drop for Statement { + fn drop(&mut self) { + // Fire and forget: there is no reply to wait for and nothing a + // caller could do with a failure. If the worker is already gone it + // has dropped every kept statement with it. + self.conn.send(Request::Finalize { id: self.id }).ok(); + } +} + /// What the worker should open. enum Target { /// A real file, through `UnixVfs`. @@ -1182,6 +1391,13 @@ struct Engine { /// schema-changing statement is the conservative rule the CLI already /// uses (`src/bin/sqlite-rs/exec.rs::is_schema_changing`). catalog: Option<(Vec, Vec)>, + /// Statements compiled and kept for a [`Statement`] handle. + statements: std::collections::HashMap, + /// Source of [`KeptStatement`] keys. + next_statement_id: u64, + /// Bumped whenever the catalog is invalidated, so a kept statement can + /// tell it was compiled against an older schema (013/Req 8). + schema_generation: u64, /// How long a contended lock is waited for before giving up. /// /// Zero by default, matching stock SQLite — `sqlite3_busy_timeout` is @@ -1241,6 +1457,21 @@ fn worker_main( Request::Counters { reply } => { answer(&reply, engine.counters); } + Request::Prepare { sql, reply } => { + answer(&reply, engine.prepare(&sql)); + } + Request::StatementExecute { id, params, reply } => { + answer(&reply, engine.statement_execute(id, params)); + } + Request::StatementQuery { id, params, reply } => { + engine.statement_stream(id, params, &reply); + } + Request::StatementRepreparations { id, reply } => { + answer(&reply, engine.repreparations(id)); + } + Request::Finalize { id } => { + engine.statements.remove(&id); + } Request::SetBusyTimeout { timeout, reply } => { engine.busy_timeout = timeout; answer(&reply, ()); @@ -1262,6 +1493,9 @@ impl Engine { autocommit: true, counters: Counters::default(), catalog: None, + statements: std::collections::HashMap::new(), + next_statement_id: 0, + schema_generation: 0, busy_timeout: Duration::ZERO, }) } @@ -1385,11 +1619,7 @@ impl Engine { } fn execute_one(&mut self, sql: &str, params: Vec) -> Result { - let statements = crate::parser::split_statements(sql); - let count = statements.len(); - let Some(statement) = statements.into_iter().next().filter(|_| count == 1) else { - return Err(Error::MultipleStatements { count }); - }; + let statement = self.single_statement(sql)?; self.run_with_retry(&statement, params) } @@ -1404,8 +1634,24 @@ impl Engine { /// counters and invalidating the catalog if it could have changed. fn run(&mut self, sql: &str, params: Vec) -> Result { let program = self.compile(sql)?; + self.run_compiled(sql, &program, params) + } - if self.mode == OpenMode::ReadOnly && writes(&program) { + /// Checks an already-compiled `program` against this connection's mode + /// and the supplied parameters, runs it, and invalidates the catalog if + /// it could have changed the schema. + /// + /// Split out from [`Engine::run`] so a kept statement takes exactly the + /// same path as an ad-hoc one — a second copy of these checks is how a + /// prepared statement ends up honouring a different contract from + /// `execute`. + fn run_compiled( + &mut self, + sql: &str, + program: &Program, + params: Vec, + ) -> Result { + if self.mode == OpenMode::ReadOnly && writes(program) { return Err(Error::ReadOnly { statement: sql.to_string(), }); @@ -1419,14 +1665,169 @@ impl Engine { }); } - let outcome = self.step(&program, params)?; + let outcome = self.step(program, params)?; if is_schema_changing(sql) { - self.catalog = None; + self.invalidate_catalog(); } Ok(outcome) } + /// Drops the decoded catalog and moves the schema generation on. + /// + /// The generation is what lets a kept statement notice it was compiled + /// against an older schema. Correctness, not caching: a program + /// addresses tables by root page, and a `DROP` can hand that page back + /// to the freelist for a later `CREATE` to reuse — so running a stale + /// program could read a page that now belongs to a different table + /// (013/Req 8). + fn invalidate_catalog(&mut self) { + self.catalog = None; + self.schema_generation = self.schema_generation.saturating_add(1); + } + + /// Compiles `sql` and keeps it, returning the handle's fields. + fn prepare(&mut self, sql: &str) -> Result { + let statement = self.single_statement(sql)?; + let program = self.compile(&statement)?; + let column_names = Arc::new(self.column_names_of(&statement)?); + let param_count = program.param_count(); + + let id = self.next_statement_id; + self.next_statement_id = self.next_statement_id.saturating_add(1); + self.statements.insert( + id, + KeptStatement { + sql: statement, + program, + column_names: Arc::clone(&column_names), + generation: self.schema_generation, + repreparations: 0, + }, + ); + Ok(PreparedHandle { + id, + param_count, + column_names, + }) + } + + /// Takes a kept statement out of the table, recompiling it first if the + /// schema has moved under it. + /// + /// Taken rather than borrowed because running it needs `&mut self`. The + /// caller must put it back — see [`Engine::statement_execute`]. + fn take_refreshed(&mut self, id: u64) -> Result { + let mut kept = self + .statements + .remove(&id) + .ok_or(Error::StatementFinalized)?; + if kept.generation != self.schema_generation { + // Recompile rather than fail, which is what + // `sqlite3_prepare_v2` does on `SQLITE_SCHEMA`. A failure would + // be safe too, but it would push a retry loop onto every + // caller for something the connection can do itself. + match self.compile(&kept.sql) { + Ok(program) => { + let names = self.column_names_of(&kept.sql)?; + kept.program = program; + kept.column_names = Arc::new(names); + kept.generation = self.schema_generation; + kept.repreparations = kept.repreparations.saturating_add(1); + } + Err(e) => { + // The statement no longer compiles — its table was + // dropped, say. Keep it (so the id stays valid and the + // error is repeatable) and report why. + self.statements.insert(id, kept); + return Err(e); + } + } + } + Ok(kept) + } + + fn statement_execute(&mut self, id: u64, params: Vec) -> Result { + let kept = self.take_refreshed(id)?; + let result = self.run_compiled(&kept.sql, &kept.program, params); + self.statements.insert(id, kept); + result + } + + fn repreparations(&mut self, id: u64) -> Result { + self.statements + .get(&id) + .map(|kept| kept.repreparations) + .ok_or(Error::StatementFinalized) + } + + /// [`Engine::stream`] for a kept statement. + fn statement_stream( + &mut self, + id: u64, + params: Vec, + reply: &SyncSender>, + ) { + let kept = match self.take_refreshed(id) { + Ok(kept) => kept, + Err(e) => { + answer(reply, Err(e)); + return; + } + }; + + if let Err(e) = self.check_runnable(&kept.sql, &kept.program, params.len()) { + self.statements.insert(id, kept); + answer(reply, Err(e)); + return; + } + + let (chunk_tx, chunk_rx) = sync_channel::(CHUNK_SLOTS); + let head = QueryStream { + column_names: Arc::clone(&kept.column_names), + chunks: chunk_rx, + }; + if reply.send(Ok(head)).is_err() { + self.statements.insert(id, kept); + return; + } + self.drain(&kept.program, params, &chunk_tx); + self.statements.insert(id, kept); + } + + /// The mode and arity checks, shared by the ad-hoc and kept paths. + fn check_runnable( + &self, + sql: &str, + program: &Program, + param_count: usize, + ) -> Result<(), Error> { + if self.mode == OpenMode::ReadOnly && writes(program) { + return Err(Error::ReadOnly { + statement: sql.to_string(), + }); + } + let wanted = program.param_count(); + if wanted != param_count { + return Err(Error::ParamCount { + expected: wanted, + found: param_count, + }); + } + Ok(()) + } + + /// Splits `sql` and insists on exactly one statement. + fn single_statement(&self, sql: &str) -> Result { + let statements = crate::parser::split_statements(sql); + let count = statements.len(); + statements + .into_iter() + .next() + .filter(|_| count == 1) + .ok_or(Error::MultipleStatements { count }) + } + /// Runs `program`, threading the transaction state and folding the /// counters. fn step(&mut self, program: &Program, params: Vec) -> Result { @@ -1510,18 +1911,7 @@ impl Engine { param_count: usize, ) -> Result<(Program, Arc>), Error> { let program = self.compile(sql)?; - if self.mode == OpenMode::ReadOnly && writes(&program) { - return Err(Error::ReadOnly { - statement: sql.to_string(), - }); - } - let wanted = program.param_count(); - if wanted != param_count { - return Err(Error::ParamCount { - expected: wanted, - found: param_count, - }); - } + self.check_runnable(sql, &program, param_count)?; let names = self.column_names_of(sql)?; Ok((program, Arc::new(names))) } diff --git a/tests/corpus/api_oracle_test.rs b/tests/corpus/api_oracle_test.rs index 4683ec73..d6ef92c3 100644 --- a/tests/corpus/api_oracle_test.rs +++ b/tests/corpus/api_oracle_test.rs @@ -362,3 +362,80 @@ fn render(value: &Value) -> String { Value::Blob(v) => String::from_utf8_lossy(v).into_owned(), } } + +/// A statement prepared *before* an index existed must maintain that index +/// once it does (spec 013 Requirement 8). +/// +/// This is the claim a unit test cannot make. A stale program inserts the +/// table row and skips the index entirely, leaving a row present in the +/// table with no matching index entry — and neither our own reads nor our +/// own integrity checker necessarily notice, because a read may table-scan +/// and find it anyway. `PRAGMA integrity_check` in stock sqlite3 does +/// notice: a missing index entry is exactly what it reports. #685 is the +/// precedent — that whole class of bug was invisible until the oracle was +/// asked. +#[test] +fn a_prepared_write_after_create_index_keeps_the_file_valid() { + let Some(bin) = pinned_oracle() else { + skip_no_oracle("a_prepared_write_after_create_index_keeps_the_file_valid"); + return; + }; + let dir = scratch_dir("reprepare"); + let db = dir.join("idx.db"); + + { + let conn = Connection::open(&db).unwrap(); + conn.execute_batch( + "CREATE TABLE t(id INTEGER, name TEXT); + INSERT INTO t VALUES (1, 'one');", + ) + .unwrap(); + + // Prepared while no index exists. + let insert = conn.prepare("INSERT INTO t VALUES (?1, ?2)").unwrap(); + insert + .execute(vec![Value::from(2), Value::from("two")]) + .unwrap(); + + // Now two indexes appear, including a UNIQUE one — whose entries + // stock sqlite3 checks against the table both ways. + conn.execute("CREATE INDEX t_id ON t(id)").unwrap(); + conn.execute("CREATE UNIQUE INDEX t_name ON t(name)") + .unwrap(); + + // The same handle, run again. It must recompile and maintain both. + insert + .execute(vec![Value::from(3), Value::from("three")]) + .unwrap(); + insert + .execute(vec![Value::from(4), Value::from("four")]) + .unwrap(); + + assert!( + insert.reprepare_count().unwrap() >= 1, + "the prepared insert should have recompiled once the indexes appeared" + ); + } + + assert_eq!( + oracle_says(&bin, &db, "PRAGMA integrity_check;"), + "ok", + "rows inserted through a statement prepared before the index left the \ + file malformed — the index is missing entries the table has" + ); + + // And the index really is usable, checked through the oracle so our own + // planner cannot paper over a missing entry with a table scan. + assert_eq!( + oracle_says( + &bin, + &db, + "SELECT name FROM t INDEXED BY t_id WHERE id = 4;" + ), + "four", + "the index has no entry for a row inserted through the stale statement" + ); + assert_eq!(oracle_says(&bin, &db, "SELECT count(*) FROM t;"), "4"); + + std::fs::remove_dir_all(&dir).ok(); +} diff --git a/tests/unit/api_statement_test.rs b/tests/unit/api_statement_test.rs new file mode 100644 index 00000000..a76c4df6 --- /dev/null +++ b/tests/unit/api_statement_test.rs @@ -0,0 +1,345 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! Prepared statements (spec 013 Requirement 3) and schema refresh +//! (Requirement 8). +//! +//! Requirement 3 is explicit that the value here is not speed — a dozen +//! statements at commit frequency saves nothing measurable by compiling +//! once. It is that a handle owning its parameter slots refuses a wrong +//! argument count instead of writing a valid row that points at the wrong +//! table. +//! +//! Requirement 8 is the sharper one. A program addresses tables by root +//! page, and a `DROP` can return that page to the freelist for a later +//! `CREATE` to reuse — so a statement compiled before a schema change and +//! run after it could read a page that now belongs to a different table. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use sqlite_rs::api::{Connection, Error}; +use sqlite_rs::record::Value; + +fn seeded() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE t(id INTEGER, name TEXT); + INSERT INTO t VALUES (1, 'one'); + INSERT INTO t VALUES (2, 'two'); + INSERT INTO t VALUES (3, 'three');", + ) + .unwrap(); + conn +} + +/// Requirement 3's first scenario. +#[test] +fn compile_once_bind_many() { + let conn = seeded(); + let stmt = conn.prepare("SELECT name FROM t WHERE id = ?1").unwrap(); + assert_eq!(stmt.param_count(), 1); + + for (id, expected) in [(1i64, "one"), (2, "two"), (3, "three")] { + let row = stmt + .query_row(vec![Value::from(id)]) + .unwrap() + .unwrap_or_else(|| panic!("id {id} should match a row")); + assert_eq!(row.get::(0).unwrap(), expected); + } + + // "...and compilation happened once". Observable rather than asserted + // about internals: `reprepare_count` is SQLite's + // SQLITE_STMTSTATUS_REPREPARE, the number of automatic recompiles. + assert_eq!( + stmt.reprepare_count().unwrap(), + 0, + "the statement was recompiled despite the schema holding still" + ); +} + +/// Requirement 3's second scenario. +#[test] +fn named_param_is_refused_at_prepare() { + let conn = seeded(); + let err = conn + .prepare("SELECT * FROM t WHERE id = :id") + .expect_err("a named parameter should be refused at prepare time"); + + assert_eq!( + err, + Error::NamedParameter { + placeholder: ":id".to_string() + }, + "the refusal should name the unsupported form" + ); +} + +#[test] +fn a_prepared_write_binds_and_counts() { + let conn = seeded(); + let insert = conn.prepare("INSERT INTO t VALUES (?1, ?2)").unwrap(); + assert_eq!(insert.param_count(), 2); + + assert_eq!( + insert + .execute(vec![Value::from(4), Value::from("four")]) + .unwrap(), + 1 + ); + assert_eq!( + insert + .execute(vec![Value::from(5), Value::from("five")]) + .unwrap(), + 1 + ); + + let update = conn + .prepare("UPDATE t SET name = ?1 WHERE id = ?2") + .unwrap(); + assert_eq!( + update + .execute(vec![Value::from("IV"), Value::from(4)]) + .unwrap(), + 1 + ); + // A miss reports zero, which is the optimistic-concurrency signal. + assert_eq!( + update + .execute(vec![Value::from("x"), Value::from(999)]) + .unwrap(), + 0 + ); + + let count: i64 = conn + .query_row("SELECT count(*) FROM t") + .unwrap() + .unwrap() + .get(0) + .unwrap(); + assert_eq!(count, 5); +} + +/// The stated reason the type exists: a wrong argument count is refused +/// rather than silently bound to NULL. +#[test] +fn a_wrong_argument_count_is_refused_every_time() { + let conn = seeded(); + let stmt = conn.prepare("INSERT INTO t VALUES (?1, ?2)").unwrap(); + + assert_eq!( + stmt.execute(vec![Value::from(9)]).expect_err("too few"), + Error::ParamCount { + expected: 2, + found: 1 + } + ); + assert_eq!( + stmt.execute(vec![Value::from(9), Value::from("a"), Value::from("b")]) + .expect_err("too many"), + Error::ParamCount { + expected: 2, + found: 3 + } + ); + + // The refusals wrote nothing, and the handle still works afterwards. + assert_eq!( + stmt.execute(vec![Value::from(9), Value::from("nine")]) + .unwrap(), + 1 + ); + let count: i64 = conn + .query_row("SELECT count(*) FROM t") + .unwrap() + .unwrap() + .get(0) + .unwrap(); + assert_eq!(count, 4); +} + +#[test] +fn column_names_are_known_before_the_statement_runs() { + let conn = seeded(); + let stmt = conn.prepare("SELECT id, name FROM t").unwrap(); + assert_eq!(stmt.column_names(), ["id", "name"]); +} + +#[test] +fn a_prepared_statement_streams_like_an_ad_hoc_one() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute("CREATE TABLE t(a INTEGER)").unwrap(); + let insert = conn.prepare("INSERT INTO t VALUES (?1)").unwrap(); + for i in 0..200 { + insert.execute(vec![Value::from(i)]).unwrap(); + } + + let select = conn.prepare("SELECT a FROM t").unwrap(); + let mut rows = select.query(vec![]).unwrap(); + let mut seen = 0i64; + while let Some(row) = rows.next_row().unwrap() { + assert_eq!(row.get::(0).unwrap(), seen); + seen += 1; + } + assert_eq!( + seen, 200, + "a prepared query should span batch boundaries too" + ); +} + +/// Requirement 8. A statement prepared before a schema change must not run +/// against the old plan. +/// +/// The recompile is what `sqlite3_prepare_v2` does on `SQLITE_SCHEMA`, and +/// `reprepare_count` is how it is observed. +#[test] +fn a_statement_recompiles_after_a_schema_change() { + let conn = seeded(); + let stmt = conn.prepare("SELECT name FROM t WHERE id = ?1").unwrap(); + + let before = stmt.query_row(vec![Value::from(1)]).unwrap().unwrap(); + assert_eq!(before.get::(0).unwrap(), "one"); + assert_eq!(stmt.reprepare_count().unwrap(), 0); + + // A schema change that does not affect this statement's own table, but + // does move the catalog. + conn.execute("CREATE TABLE other(x INTEGER)").unwrap(); + + let after = stmt.query_row(vec![Value::from(2)]).unwrap().unwrap(); + assert_eq!( + after.get::(0).unwrap(), + "two", + "the statement still works" + ); + assert_eq!( + stmt.reprepare_count().unwrap(), + 1, + "the statement should have recompiled against the new catalog" + ); + + // And it does not recompile again while the schema holds still. + stmt.query_row(vec![Value::from(3)]).unwrap().unwrap(); + assert_eq!(stmt.reprepare_count().unwrap(), 1); +} + +/// The case Requirement 8 is really about: an index created after the +/// statement was prepared changes the plan, and a stale program would +/// neither use it nor maintain it. +#[test] +fn an_index_created_after_prepare_is_picked_up() { + let conn = seeded(); + let select = conn.prepare("SELECT name FROM t WHERE id = ?1").unwrap(); + select.query_row(vec![Value::from(1)]).unwrap().unwrap(); + + conn.execute("CREATE INDEX t_id ON t(id)").unwrap(); + + // Still correct after the plan changes under it. + let row = select.query_row(vec![Value::from(2)]).unwrap().unwrap(); + assert_eq!(row.get::(0).unwrap(), "two"); + + // A prepared write must maintain the new index. This asserts only that + // the row is *reachable* — deliberately not claimed as proof of index + // maintenance, because the planner may satisfy this read with a table + // scan and then find the row whether the index has it or not. Measured: + // with the refresh disabled, this assertion still passes. + // + // The real claim needs a third party that validates the index against + // the table, so it lives where the oracle does: + // `tests/corpus/api_oracle_test.rs::a_prepared_write_after_create_index_keeps_the_file_valid` + // inserts through a statement prepared before the index existed and has + // the pinned sqlite3 run `PRAGMA integrity_check`. + let insert = conn.prepare("INSERT INTO t VALUES (?1, ?2)").unwrap(); + insert + .execute(vec![Value::from(4), Value::from("four")]) + .unwrap(); + + let via_index = conn + .query_row("SELECT name FROM t WHERE id = 4") + .unwrap() + .expect("the new row should be findable through the index"); + assert_eq!(via_index.get::(0).unwrap(), "four"); + + assert!( + select.reprepare_count().unwrap() >= 1, + "the statement should have recompiled once the index appeared" + ); +} + +/// If the statement's own table is dropped, recompiling fails — and the +/// error must be reported rather than the old program silently reused. +#[test] +fn a_statement_whose_table_is_dropped_reports_the_failure() { + let conn = seeded(); + let stmt = conn.prepare("SELECT name FROM t WHERE id = ?1").unwrap(); + stmt.query_row(vec![Value::from(1)]).unwrap().unwrap(); + + conn.execute("DROP TABLE t").unwrap(); + + let err = stmt + .query_row(vec![Value::from(1)]) + .expect_err("the table is gone; the old program must not be reused"); + assert!( + matches!(err, Error::Compile { .. } | Error::Parse { .. }), + "expected a compile failure, got {err:?}" + ); + + // Repeatable rather than a one-shot: the handle stays valid and keeps + // reporting the same thing. + assert!(stmt.query_row(vec![Value::from(1)]).is_err()); +} + +#[test] +fn statements_outlive_the_connection_handle_they_were_made_from() { + let stmt = { + let conn = Connection::open_in_memory().unwrap(); + conn.execute("CREATE TABLE t(a INTEGER)").unwrap(); + conn.prepare("INSERT INTO t VALUES (?1)").unwrap() + // `conn` dropped here; the statement holds its own clone. + }; + + assert_eq!(stmt.execute(vec![Value::from(1)]).unwrap(), 1); + let count: i64 = stmt + .connection() + .query_row("SELECT count(*) FROM t") + .unwrap() + .unwrap() + .get(0) + .unwrap(); + assert_eq!(count, 1); +} + +#[test] +fn many_statements_coexist_and_are_finalized_independently() { + let conn = seeded(); + let a = conn.prepare("SELECT name FROM t WHERE id = ?1").unwrap(); + let b = conn.prepare("SELECT count(*) FROM t").unwrap(); + let c = conn.prepare("INSERT INTO t VALUES (?1, ?2)").unwrap(); + + assert_eq!( + a.query_row(vec![Value::from(1)]) + .unwrap() + .unwrap() + .get::(0) + .unwrap(), + "one" + ); + drop(a); + + // Dropping one must not disturb the others. + assert_eq!( + c.execute(vec![Value::from(4), Value::from("four")]) + .unwrap(), + 1 + ); + assert_eq!( + b.query_row(vec![]).unwrap().unwrap().get::(0).unwrap(), + 4 + ); +} + +#[test] +fn prepare_refuses_a_multi_statement_string() { + let conn = seeded(); + let err = conn + .prepare("SELECT 1 FROM t; SELECT 2 FROM t") + .expect_err("prepare takes exactly one statement"); + assert_eq!(err, Error::MultipleStatements { count: 2 }); +} From 88eb38d0ac6c9877fb16313233d5197332c45a7e Mon Sep 17 00:00:00 2001 From: Diederik Siderius Date: Wed, 9 Sep 2026 21:11:15 +0200 Subject: [PATCH 10/14] feat: publish the facade as the supported surface, with a stability policy (013/Req 6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `src/lib.rs` now says which module is the API and which is the engine, and `CHANGELOG.md` carries the policy: `sqlite_rs::api` is supported and its breaking changes are called out under **Changed**; every other public module changes whenever the implementation needs it to, at any version. Before this, `CHANGELOG.md` said only "Pre-1.0: minor bumps may break the public API" over an export list containing the whole engine, so a consumer wiring `dump::open` to `execute_transaction_step` was building on items carrying no promise while having no way to know it. *SQE* confines every `sqlite_rs::` reference to one module for exactly that reason. `Value` is re-exported from `api`, so binding a parameter or reading a column no longer means naming `sqlite_rs::record`. The test with teeth ------------------- `tests/unit/api_surface_test.rs::facade_is_sufficient_alone` runs the whole workload — create, schema, prepare, bind, transaction, streamed read, by-name access, the optimistic-concurrency swap, a constraint error's result code — importing nothing but `sqlite_rs::api`. That it compiles is the assertion. On its own that would rot: a future capability gap "fixed" by importing `sqlite_rs::pager` would keep it passing while Requirement 6 became silently false. So `this_file_names_no_engine_module` reads this file's own source and fails if any of the fourteen engine modules is named in it. Comment lines are stripped first, because the prose deliberately names them to say what must not appear — an earlier version of the test failed on its own explanation. Verified to bite: adding `use sqlite_rs::pager::Pager` fails it. Examples -------- `query.rs`, `crud.rs` and `read_database.rs` rewritten against the facade; all four examples still run. `crud.rs` no longer copies `examples/fixtures/ empty.db` first — its doc comment used to read "This crate has no API to create a brand-new database file from nothing", which stopped being true this session. `examples/README.md` said the crate "exposes its parser/ codegen/VM pipeline directly rather than an ergonomic Connection/prepare/ bind wrapper"; also no longer true. `wal_mode.rs` stays engine-level, and the README now says why: explicit checkpointing is a `pager` operation the facade does not expose. One gap found and one closed ---------------------------- Writing `read_database.rs` — whose whole job is "lists its tables" — surfaced that **`sqlite_master` is not queryable through `SELECT` at all**: `resolve_from_table_schema` does not resolve it, so `SELECT name FROM sqlite_master` fails to compile. Introspection is plan.md's V7 and spec 013's non-goals hand the PRAGMA catalogue there, so this is recorded rather than fixed. `Connection::table_names` closes the consumer-facing half without touching codegen: it reads the catalog the worker has already decoded. Requirement 6 asks that the facade cover what the engine offers a consumer, and `schema::read_schema` could always enumerate tables — just not from here. Gates: make test (1666 passed, 10 ignored), make test-corpus (404 passed), make lint (both clippy passes + fmt), all four examples run. Spend: within estimate. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 19 +++ Cargo.toml | 4 + examples/README.md | 44 ++++--- examples/crud.rs | 140 ++++++++++----------- examples/query.rs | 95 ++++++++------- examples/read_database.rs | 87 +++++++------- src/api.rs | 36 +++++- src/lib.rs | 21 ++++ tests/unit/api_surface_test.rs | 214 +++++++++++++++++++++++++++++++++ 9 files changed, 479 insertions(+), 181 deletions(-) create mode 100644 tests/unit/api_surface_test.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a35d73e..9f24a3c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ All notable changes to sqlite-rs. Format follows [Keep a Changelog](https://keepachangelog.com/), versioning follows [SemVer](https://semver.org/). Pre-1.0: minor bumps may break the public API. +**API stability policy:** `sqlite_rs::api` is the supported surface for +embedding — `Connection`, `Statement`, `Rows`, `Row`, `Transaction`, +`Error`, `OpenMode`, `TransactionBehavior`, `FromValue`, `Value`. Breaking +changes to it are called out in this file under **Changed**, and after 1.0 +will require a major bump. + +Every other public module (`btree`, `codegen`, `dump`, `format`, `header`, +`integrity`, `pager`, `parser`, `planner`, `record`, `schema`, `sys`, +`vdbe`, `vfs`) is the **engine**. It is public because the CLI in +`src/bin/` links this crate like any other consumer, and because it is +useful for inspecting a database file — not as a promise. Engine +signatures change whenever the implementation needs them to, at any +version, without appearing under **Changed**. Code built on them can break +on a patch release. + +If a consumer needs something only the engine offers, that is a gap in +`api`; please report it as one rather than depending on the engine +(spec 013 Requirement 6). + **Versioning policy:** one minor version per completed plan phase — the version number tells the plan's story, sub-steps stay inside a phase. V1 (READ CORE) = 0.1.0 through 0.4.0. *(History note: internal iterations briefly numbered 0.4.0–0.6.0 were renumbered into the phase scheme on 14 Aug 2026, before any tag or publication of those versions existed.)* ## [0.18.10] - 2026-08-31 diff --git a/Cargo.toml b/Cargo.toml index 1a56a9e4..7d7e6d1c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -176,6 +176,10 @@ path = "tests/unit/api_durability_test.rs" name = "api_statement" path = "tests/unit/api_statement_test.rs" +[[test]] +name = "api_surface" +path = "tests/unit/api_surface_test.rs" + [[test]] name = "codegen_expr" path = "tests/unit/codegen_expr_test.rs" diff --git a/examples/README.md b/examples/README.md index d2f810fc..19f1009a 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,22 +1,28 @@ # Examples -Runnable samples showing how to use `sqlite-rs` as a library. This crate -exposes its parser/codegen/VM pipeline directly rather than an -ergonomic `Connection`/`prepare`/`bind` wrapper, so each example wires -those pieces together the same way the `sqlite-rs` CLI binary -(`src/bin/sqlite-rs/`) does. +Runnable samples showing how to use `sqlite-rs` as a library. + +Use **`sqlite_rs::api`** — `Connection`, `Statement`, `Rows`, +`Transaction`, `Value`. That is the supported surface (spec 013); the +parser/codegen/VM modules this crate also exports are the engine, and +carry no stability promise. See the API stability policy at the top of +`CHANGELOG.md`. Run any example with `cargo run --example `. -- **`read_database.rs`** — opens an existing database file, lists its - tables, and iterates every row of one table. -- **`query.rs`** — compiles a `SELECT` once and runs it with different - bound `?1` parameter values. -- **`crud.rs`** — a full create/insert/update/delete cycle wrapped in an - explicit `BEGIN`/`COMMIT` transaction. +- **`read_database.rs`** — opens an existing file read-only, lists its + tables, iterates every row of one, and shows a write being refused. +- **`query.rs`** — prepares a `SELECT` once and runs it with different + bound `?1` values, then streams a multi-row result. +- **`crud.rs`** — a full create/insert/update/delete cycle, an explicit + transaction, `last_insert_rowid`, the rows-affected count, and a + rollback-on-drop. - **`wal_mode.rs`** — switches a database to WAL journal mode, writes and reads through it, then checkpoints the WAL back into the main - file. True multi-process concurrent readers/writer is out of scope + file. The one example still written against the engine rather than + `api`: explicit checkpointing is a `pager` operation the facade does + not expose (`Connection::pragma` can set `journal_mode`, but not + trigger a checkpoint). True multi-process concurrent readers/writer is out of scope for a single-binary example — see `tests/corpus/wal_concurrent_interop_test.rs` and `tests/corpus/wal_write_interop_test.rs` for that. @@ -24,7 +30,13 @@ Run any example with `cargo run --example `. ## Fixtures `fixtures/sample.db` and `fixtures/empty.db` are small SQLite files -checked into the repo and built with the real `sqlite3` CLI. This -crate has no API to create a brand-new database file from nothing — -only to open an already-valid one — so `crud.rs` and `wal_mode.rs` -copy `empty.db` to a scratch path before writing to it. +checked into the repo and built with the real `sqlite3` CLI. +`read_database.rs` reads `sample.db`. + +`empty.db` used to be load-bearing: this crate had no way to create a +database from nothing, so an example that wanted to write had to copy it +first. `Connection::open` now creates a valid empty database when the +path has no file, so `crud.rs` needs no fixture at all — and that the +result is a real SQLite database is checked against the pinned `sqlite3` +in `tests/corpus/bootstrap_oracle_test.rs`. `wal_mode.rs` still copies +`empty.db`, being engine-level. diff --git a/examples/crud.rs b/examples/crud.rs index 57ab6415..0533b07e 100644 --- a/examples/crud.rs +++ b/examples/crud.rs @@ -1,95 +1,87 @@ // Copyright 2026 Schuberg Philis // SPDX-License-Identifier: Apache-2.0 //! A full create-read-update-delete cycle, including an explicit -//! `BEGIN`/`COMMIT` transaction. -//! -//! This crate has no API to create a brand-new database file from -//! nothing (only to open an already-valid one), so this example copies -//! a checked-in empty fixture to a scratch path first, then builds a -//! table on top of it. +//! transaction and the rows-affected count. //! //! Run with: `cargo run --example crud` -use std::cell::RefCell; use std::error::Error; -use std::path::Path; -use std::rc::Rc; -use sqlite_rs::btree::TableCursor; -use sqlite_rs::codegen::{ - compile_select_with_catalog, compile_statement, resolve_from_table_schema, -}; -use sqlite_rs::dump; -use sqlite_rs::format::format_query_value; -use sqlite_rs::parser::{parse_select, split_statements, ParseOutcome}; -use sqlite_rs::schema::{read_schema, read_views}; -use sqlite_rs::vdbe::{execute_transaction_step, execute_with_db}; -use sqlite_rs::vfs::{PageSource, UnixVfs}; +use sqlite_rs::api::{Connection, TransactionBehavior, Value}; fn main() -> Result<(), Box> { - let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("examples/fixtures/empty.db"); - let scratch_dir = - std::env::temp_dir().join(format!("sqlite-rs-crud-example-{}", std::process::id())); - std::fs::create_dir_all(&scratch_dir)?; - let scratch_db = scratch_dir.join("crud.db"); - std::fs::copy(&fixture, &scratch_db)?; + // `open` creates the database if the path has no file yet, so no empty + // fixture needs copying into place first. + let dir = std::env::temp_dir().join(format!("sqlite-rs-crud-{}", std::process::id())); + std::fs::create_dir_all(&dir)?; + let path = dir.join("crud.db"); + + let conn = Connection::open(&path)?; + println!("opened {} (created it if absent)", path.display()); - let (header, pager) = dump::open(&UnixVfs, &scratch_db)?; - let pager = Rc::new(RefCell::new(pager)); + // Durability is a choice; FULL fsyncs before a commit returns. + conn.pragma("synchronous", "FULL")?; - let script = " - CREATE TABLE todos(id INTEGER PRIMARY KEY, task TEXT, done INTEGER); - BEGIN; - INSERT INTO todos(id, task, done) VALUES (1, 'write examples', 0); - INSERT INTO todos(id, task, done) VALUES (2, 'ship it', 0); - UPDATE todos SET done = 1 WHERE id = 1; - DELETE FROM todos WHERE id = 2; - COMMIT; - "; + conn.execute("CREATE TABLE task(id INTEGER PRIMARY KEY, title TEXT, done INTEGER)")?; - let mut autocommit = true; - for stmt in split_statements(script) { - let (schemas, views) = { - let borrowed = pager.borrow(); - let mut schema_cursor = TableCursor::new(&*borrowed, &header, 1); - let schemas = read_schema(&mut schema_cursor, header.text_encoding)?; - let mut view_cursor = TableCursor::new(&*borrowed, &header, 1); - let views = read_views(&mut view_cursor, header.text_encoding)?; - (schemas, views) - }; + // CREATE, inside a transaction so the three rows land as one unit. + let insert = conn.prepare("INSERT INTO task(title, done) VALUES (?1, ?2)")?; + let tx = conn.transaction_with(TransactionBehavior::Immediate)?; + for title in ["write the spec", "review the PR", "ship it"] { + insert.execute(vec![Value::from(title), Value::from(false)])?; + // Rowids are assigned by the engine; `last_insert_rowid` is how a + // caller learns the one it just wrote. + println!("inserted {title:?} as rowid {}", tx.last_insert_rowid()?); + } + tx.commit()?; - let program = compile_statement(&stmt, &schemas, &views).map_err(|e| e.to_string())?; - let (_, ac) = execute_transaction_step(&program, Rc::clone(&pager), header, autocommit) - .map_err(|e| e.to_string())?; - autocommit = ac; + // READ. + println!("\nall tasks:"); + let mut rows = conn.query("SELECT id, title, done FROM task ORDER BY id")?; + while let Some(row) = rows.next_row()? { + let done: bool = row.get_by_name("done")?; + println!( + " [{}] {} {}", + if done { 'x' } else { ' ' }, + row.get::(0)?, + row.get::(1)? + ); } - // Read back the final state through the same shared pager. - let schemas = { - let borrowed = pager.borrow(); - let mut schema_cursor = TableCursor::new(&*borrowed, &header, 1); - read_schema(&mut schema_cursor, header.text_encoding)? - }; - let select = match parse_select("SELECT id, task, done FROM todos") { - ParseOutcome::Accepted(select) => *select, - _ => return Err("failed to parse the readback query".into()), - }; - let from = select.from.as_ref().ok_or("SELECT has no FROM clause")?; - let table = resolve_from_table_schema(&from.first, &schemas).map_err(|e| e.to_string())?; - let select_program = - compile_select_with_catalog(&select, &table, &schemas).map_err(|e| e.to_string())?; - let source: Rc = pager; - let rows = execute_with_db(&select_program, source, header).map_err(|e| e.to_string())?; + // UPDATE. The returned count is what distinguishes a match from a + // miss — every optimistic-concurrency scheme is built on it. + let changed = conn.execute_with( + "UPDATE task SET done = ?1 WHERE title = ?2", + vec![Value::from(true), Value::from("review the PR")], + )?; + println!("\nmarked done: {changed} row(s) changed"); + + let missed = conn.execute_with( + "UPDATE task SET done = ?1 WHERE title = ?2", + vec![Value::from(true), Value::from("no such task")], + )?; + println!("no such task: {missed} row(s) changed"); - println!("Final todos:"); - for row in rows { - let rendered: Vec = row - .iter() - .map(|v| String::from_utf8_lossy(&format_query_value(v)).into_owned()) - .collect(); - println!(" {}", rendered.join(" | ")); + // DELETE. + let deleted = conn.execute("DELETE FROM task WHERE done = 1")?; + println!("deleted {deleted} completed task(s)"); + + // A transaction dropped without committing rolls back. + { + let tx = conn.transaction()?; + tx.execute("DELETE FROM task")?; + println!("\ninside the transaction, {} task(s) remain", count(&tx)?); + // No `commit()`: dropping here undoes the delete. } + println!("after the rollback, {} task(s) remain", count(&conn)?); - std::fs::remove_dir_all(&scratch_dir).ok(); + std::fs::remove_dir_all(&dir).ok(); Ok(()) } + +fn count(conn: &Connection) -> Result> { + let row = conn + .query_row("SELECT count(*) FROM task")? + .ok_or("count() returned no row")?; + Ok(row.get(0)?) +} diff --git a/examples/query.rs b/examples/query.rs index 79dab71e..3ea163a5 100644 --- a/examples/query.rs +++ b/examples/query.rs @@ -1,61 +1,60 @@ // Copyright 2026 Schuberg Philis // SPDX-License-Identifier: Apache-2.0 -//! Runs a parameterized `SELECT` against a database, binding a `?1` -//! placeholder before executing. +//! Prepare a `SELECT` once, then run it with different bound parameters. //! //! Run with: `cargo run --example query` use std::error::Error; -use std::path::Path; -use std::rc::Rc; - -use sqlite_rs::btree::TableCursor; -use sqlite_rs::codegen::{compile_select_with_catalog, resolve_from_table_schema}; -use sqlite_rs::dump; -use sqlite_rs::format::format_query_value; -use sqlite_rs::parser::{parse_select, ParseOutcome}; -use sqlite_rs::record::Value; -use sqlite_rs::schema::read_schema; -use sqlite_rs::vdbe::execute_with_db_and_params; -use sqlite_rs::vfs::{PageSource, UnixVfs}; + +use sqlite_rs::api::{Connection, Value}; fn main() -> Result<(), Box> { - let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("examples/fixtures/sample.db"); - let (header, pager) = dump::open(&UnixVfs, &path)?; - - let mut schema_cursor = TableCursor::new(&pager, &header, 1); - let schemas = read_schema(&mut schema_cursor, header.text_encoding)?; - - // Prepare: parse and compile once. `?1` is a placeholder bound at - // execution time via `execute_with_db_and_params`. - let select = match parse_select("SELECT name, age FROM users WHERE id = ?1") { - ParseOutcome::Accepted(select) => *select, - _ => return Err("failed to parse the query".into()), - }; - let from = select.from.as_ref().ok_or("SELECT has no FROM clause")?; - let table = resolve_from_table_schema(&from.first, &schemas).map_err(|e| e.to_string())?; - let program = - compile_select_with_catalog(&select, &table, &schemas).map_err(|e| e.to_string())?; - - let source: Rc = Rc::new(pager); - - // Bind and run for a couple of different parameter values. - for id in [1_i64, 3] { - let rows = execute_with_db_and_params( - &program, - Rc::clone(&source), - header, - vec![Value::Integer(id)], - ) - .map_err(|e| e.to_string())?; - for row in rows { - let rendered: Vec = row - .iter() - .map(|v| String::from_utf8_lossy(&format_query_value(v)).into_owned()) - .collect(); - println!("id={id}: {}", rendered.join(" | ")); + // No file needed: an in-memory database exercises the same pager, + // b-tree and journal code a file does. + let conn = Connection::open_in_memory()?; + + conn.execute_batch( + "CREATE TABLE fruit(id INTEGER, name TEXT, grams INTEGER); + INSERT INTO fruit VALUES (1, 'apple', 150); + INSERT INTO fruit VALUES (2, 'banana', 120); + INSERT INTO fruit VALUES (3, 'cherry', 8);", + )?; + + // Compiled once. `param_count` is the largest `?NNN` index the + // statement uses, matching `sqlite3_bind_parameter_count`. + let by_id = conn.prepare("SELECT name, grams FROM fruit WHERE id = ?1")?; + println!( + "prepared a statement wanting {} parameter(s)", + by_id.param_count() + ); + println!("columns: {:?}\n", by_id.column_names()); + + for id in [1i64, 2, 3, 99] { + match by_id.query_row(vec![Value::from(id)])? { + Some(row) => { + // Typed reads, by index or by name. + let name: String = row.get(0)?; + let grams: i64 = row.get_by_name("grams")?; + println!("id {id}: {name} ({grams} g)"); + } + None => println!("id {id}: no such row"), } } + // Binding the wrong number of parameters is refused rather than + // silently bound to NULL — which is the point of a statement handle. + match by_id.execute(vec![]) { + Err(e) => println!("\nno parameters bound -> {e}"), + Ok(_) => println!("\nunexpectedly accepted an unbound parameter"), + } + + // A multi-row result streams: rows arrive in batches, so peak memory + // does not grow with the size of the result. + println!("\nheaviest first:"); + let mut rows = conn.query("SELECT name, grams FROM fruit ORDER BY grams DESC")?; + while let Some(row) = rows.next_row()? { + println!(" {:<8} {:>4} g", row.get::(0)?, row.get::(1)?); + } + Ok(()) } diff --git a/examples/read_database.rs b/examples/read_database.rs index ea55bc4a..c87d67fe 100644 --- a/examples/read_database.rs +++ b/examples/read_database.rs @@ -1,59 +1,62 @@ // Copyright 2026 Schuberg Philis // SPDX-License-Identifier: Apache-2.0 -//! Opens an existing SQLite file, lists its tables, and iterates every -//! row of one table. +//! Open an existing database file, list its tables, and read every row of +//! one of them. //! //! Run with: `cargo run --example read_database` use std::error::Error; use std::path::Path; -use std::rc::Rc; -use sqlite_rs::btree::TableCursor; -use sqlite_rs::codegen::{compile_select_with_catalog, resolve_from_table_schema}; -use sqlite_rs::dump; -use sqlite_rs::format::format_query_value; -use sqlite_rs::parser::{parse_select, ParseOutcome}; -use sqlite_rs::schema::read_schema; -use sqlite_rs::vdbe::execute_with_db; -use sqlite_rs::vfs::{PageSource, UnixVfs}; +use sqlite_rs::api::{Connection, OpenMode}; fn main() -> Result<(), Box> { - let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("examples/fixtures/sample.db"); - - // Opening a database parses its header and returns a `Pager` over it. - let (header, pager) = dump::open(&UnixVfs, &path)?; - - // Schema introspection: `sqlite_schema` always lives at root page 1. - let mut schema_cursor = TableCursor::new(&pager, &header, 1); - let schemas = read_schema(&mut schema_cursor, header.text_encoding)?; - - println!("Tables:"); - for schema in &schemas { - println!(" {} ({} columns)", schema.name, schema.columns.len()); + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("examples/fixtures/sample.db"); + + // `ReadWrite` rather than the default `ReadWriteCreate`, so a typo in + // the path is an error instead of a new empty database. `ReadOnly` + // would also refuse every write for the connection's whole life. + let conn = Connection::open_with(&fixture, OpenMode::ReadOnly)?; + println!("opened {} read-only\n", fixture.display()); + + let tables = conn.table_names()?; + println!("{} table(s): {}\n", tables.len(), tables.join(", ")); + + for table in &tables { + // Table names cannot be bound as parameters — a placeholder is a + // *value*, not an identifier — so this interpolates a name that + // came from the catalog itself, not from user input. + let count: i64 = conn + .query_row(&format!("SELECT count(*) FROM {table}"))? + .ok_or("count() returned no row")? + .get(0)?; + println!("{table}: {count} row(s)"); } - // Row iteration: resolve the table, compile `SELECT * FROM users`, - // then execute it against the same `Pager` (as a read-only `PageSource`). - let select = match parse_select("SELECT * FROM users") { - ParseOutcome::Accepted(select) => *select, - _ => return Err("failed to parse SELECT * FROM users".into()), + let Some(first) = tables.first() else { + println!("\nno tables to read"); + return Ok(()); }; - let from = select.from.as_ref().ok_or("SELECT has no FROM clause")?; - let table = resolve_from_table_schema(&from.first, &schemas).map_err(|e| e.to_string())?; - let program = - compile_select_with_catalog(&select, &table, &schemas).map_err(|e| e.to_string())?; - - let source: Rc = Rc::new(pager); - let rows = execute_with_db(&program, source, header).map_err(|e| e.to_string())?; - - println!("\nRows in users:"); - for row in rows { - let rendered: Vec = row - .iter() - .map(|v| String::from_utf8_lossy(&format_query_value(v)).into_owned()) + + println!("\nevery row of {first}:"); + let mut rows = conn.query(&format!("SELECT * FROM {first}"))?; + println!(" columns: {:?}", rows.column_names().join(", ")); + while let Some(row) = rows.next_row()? { + // `value()` hands back the raw storage class, for a reader that + // does not know the column types ahead of time. + let cells: Vec = (0..row.len()) + .map(|i| match row.value(i) { + Some(v) => format!("{v:?}"), + None => "".to_string(), + }) .collect(); - println!(" {}", rendered.join(" | ")); + println!(" {}", cells.join(" | ")); + } + + // Read-only means read-only: this is refused rather than attempted. + match conn.execute(&format!("DELETE FROM {first}")) { + Err(e) => println!("\nattempted write -> {e}"), + Ok(_) => println!("\nunexpectedly wrote to a read-only connection"), } Ok(()) diff --git a/src/api.rs b/src/api.rs index 2c1ffdb1..c2a98b72 100644 --- a/src/api.rs +++ b/src/api.rs @@ -46,7 +46,11 @@ use std::time::{Duration, Instant}; use crate::header::{DatabaseHeader, DEFAULT_PAGE_SIZE}; use crate::pager::Pager; -use crate::record::Value; +// Re-exported rather than merely imported, so the whole embedding API is +// reachable from this one module. A caller binding a parameter or reading a +// column needs `Value`, and having to name `sqlite_rs::record` for it would +// leave the facade incomplete in exactly the way Requirement 6 is about. +pub use crate::record::Value; use crate::schema::{TableSchema, ViewSchema}; use crate::vdbe::{Opcode, Program}; use crate::vfs::{MemoryVfs, UnixVfs, Vfs}; @@ -476,6 +480,11 @@ enum Request { /// Where to send the stream's head, or the failure to start it. reply: SyncSender>, }, + /// List the table names in the catalog. + TableNames { + /// Where to send them. + reply: SyncSender, Error>>, + }, /// Read the connection-scoped counters. Counters { /// Where to send them. @@ -777,6 +786,25 @@ impl Connection { Ok(first) } + /// The names of the tables in this database, in catalog order. + /// + /// Requirement 6 asks that the facade cover everything the engine + /// offers a consumer, and enumerating tables is one of those things — + /// `schema::read_schema` has always been able to, but only by reaching + /// past this module. + /// + /// This reads the decoded catalog rather than querying `sqlite_master`, + /// and that is not merely an optimisation: `sqlite_master` is currently + /// **not** queryable through `SELECT` at all + /// (`resolve_from_table_schema` does not resolve it, so + /// `SELECT name FROM sqlite_master` fails to compile). Introspection is + /// plan.md's V7; until then this is how a consumer lists tables. + pub fn table_names(&self) -> Result, Error> { + let (reply_tx, reply_rx) = sync_channel(0); + self.send(Request::TableNames { reply: reply_tx })?; + self.recv(reply_rx)? + } + /// Compiles one statement and keeps it, so it can be run repeatedly /// with different parameters. /// @@ -1454,6 +1482,12 @@ fn worker_main( Request::Query { sql, params, reply } => { engine.stream(&sql, params, &reply); } + Request::TableNames { reply } => { + let names = engine + .catalog() + .map(|(schemas, _)| schemas.iter().map(|s| s.name.clone()).collect()); + answer(&reply, names); + } Request::Counters { reply } => { answer(&reply, engine.counters); } diff --git a/src/lib.rs b/src/lib.rs index c9a438fe..f8dabbd7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,6 +26,27 @@ #![deny(unsafe_code)] #![warn(missing_docs)] +// ## The supported surface +// +// [`api`] is the API this crate supports for embedding: `Connection`, +// `Statement`, `Rows`, `Transaction`, `Error`, and `Value`. An application +// should need nothing else, and spec 013 Requirement 6 makes that a +// testable claim rather than an aspiration — +// `tests/unit/api_surface_test.rs` runs the whole workload through +// `sqlite_rs::api` alone. +// +// Every other module below is the **engine**: the parser, code generator, +// virtual machine, b-tree, pager and VFS that `api` is built on. They are +// public because the CLI in `src/bin/` is a separate binary that links this +// crate like any other consumer, and because they are genuinely useful for +// inspecting a database file. They are *not* a stability promise. Their +// signatures change whenever the implementation needs them to, without a +// major version bump, and a consumer wiring `dump::open` to +// `execute_transaction_step` is building on items that carry no such +// promise. +// +// If something a consumer needs is only reachable through the engine, that +// is a gap in `api` and worth reporting as one. pub mod api; pub mod btree; pub mod codegen; diff --git a/tests/unit/api_surface_test.rs b/tests/unit/api_surface_test.rs new file mode 100644 index 00000000..f65c866e --- /dev/null +++ b/tests/unit/api_surface_test.rs @@ -0,0 +1,214 @@ +// Copyright 2026 Schuberg Philis +// SPDX-License-Identifier: Apache-2.0 +//! The facade is sufficient on its own (spec 013 Requirement 6). +//! +//! Today `src/lib.rs` exports the whole engine — `btree`, `codegen`, +//! `dump`, `pager`, `parser`, `planner`, `vdbe`, `vfs` — while +//! `CHANGELOG.md` says pre-1.0 minor bumps may break the public API. A +//! consumer wiring `dump::open` to `execute_transaction_step` is therefore +//! building on items carrying no promise, and *SQE* confines every +//! `sqlite_rs::` reference to one module precisely because of that. The +//! stability policy is what fixes it; this test is what stops the policy +//! from being a lie. +//! +//! Two things are asserted, and the second is the one with teeth. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +// The *only* import in this file. Nothing from `pager`, `vdbe`, `codegen`, +// `dump`, `btree`, `parser`, `planner`, `schema`, `header` or `vfs`. +use sqlite_rs::api::{ + Connection, Error, FromValue, OpenMode, Row, Rows, Statement, TransactionBehavior, Value, +}; + +/// Requirement 6's first scenario: the whole workload, using only the items +/// this spec defines. +/// +/// That it compiles is the assertion. Every name it uses comes from +/// `sqlite_rs::api`, so if the facade were missing a capability this +/// function could not be written — which is what "no escape hatch" means +/// operationally. +#[test] +fn facade_is_sufficient_alone() { + // Create a database that did not exist. + let conn = Connection::open_in_memory().unwrap(); + + // Define a schema. + conn.execute_batch( + "CREATE TABLE iceberg_tables( + catalog_name TEXT, + table_namespace TEXT, + table_name TEXT, + metadata_location TEXT + ); + CREATE UNIQUE INDEX iceberg_tables_pk + ON iceberg_tables(catalog_name, table_namespace, table_name);", + ) + .unwrap(); + + // Prepare and bind. + let insert = conn + .prepare( + "INSERT INTO iceberg_tables + (catalog_name, table_namespace, table_name, metadata_location) + VALUES (?1, ?2, ?3, ?4)", + ) + .unwrap(); + assert_eq!(insert.param_count(), 4); + + // Run a transaction, and read the rows-affected count. + let tx = conn + .transaction_with(TransactionBehavior::Immediate) + .unwrap(); + for (ns, name) in [("prod", "orders"), ("prod", "customers"), ("dev", "orders")] { + let changed = insert + .execute(vec![ + Value::from("main"), + Value::from(ns), + Value::from(name), + Value::from(format!("s3://bucket/{ns}/{name}/v1.json")), + ]) + .unwrap(); + assert_eq!(changed, 1); + } + tx.commit().unwrap(); + assert_eq!(conn.changes().unwrap(), 1); + + // Read rows back as typed values, by index and by name. + let lookup = conn + .prepare( + "SELECT metadata_location FROM iceberg_tables + WHERE catalog_name = ?1 AND table_namespace = ?2 AND table_name = ?3", + ) + .unwrap(); + let row: Row = lookup + .query_row(vec![ + Value::from("main"), + Value::from("prod"), + Value::from("orders"), + ]) + .unwrap() + .expect("the row was just written"); + let location: String = row.get(0).unwrap(); + assert_eq!(location, "s3://bucket/prod/orders/v1.json"); + assert_eq!( + row.get_by_name::("metadata_location").unwrap(), + location + ); + + // Stream a multi-row result. + let mut rows: Rows = conn + .query("SELECT table_namespace, table_name FROM iceberg_tables") + .unwrap(); + let mut seen = 0; + while let Some(row) = rows.next_row().unwrap() { + let _ns: String = row.get(0).unwrap(); + seen += 1; + } + assert_eq!(seen, 3); + drop(rows); + + // The optimistic-concurrency swap, which is what the count is for. + let swap = conn + .prepare( + "UPDATE iceberg_tables SET metadata_location = ?1 + WHERE table_name = ?2 AND metadata_location = ?3", + ) + .unwrap(); + assert_eq!( + swap.execute(vec![ + Value::from("s3://bucket/prod/orders/v2.json"), + Value::from("orders"), + Value::from("s3://bucket/prod/orders/v1.json"), + ]) + .unwrap(), + 1, + "the first swap should win" + ); + assert_eq!( + swap.execute(vec![ + Value::from("s3://bucket/prod/orders/v3.json"), + Value::from("orders"), + Value::from("s3://bucket/prod/orders/v1.json"), + ]) + .unwrap(), + 0, + "the second swap should lose the race, not overwrite" + ); + + // Delete, and inspect an error's SQLite result code. + assert_eq!( + conn.execute("DELETE FROM iceberg_tables WHERE table_namespace = 'dev'") + .unwrap(), + 1 + ); + let dup = conn + .execute( + "INSERT INTO iceberg_tables + (catalog_name, table_namespace, table_name, metadata_location) + VALUES ('main', 'prod', 'orders', 'x')", + ) + .expect_err("violates the unique index"); + assert_eq!(dup.sqlite_code(), 19, "SQLITE_CONSTRAINT"); + assert!(!dup.is_retryable()); + + // And the read-only mode and the busy timeout are configurable from + // here too, completing the surface the spec lists. + conn.set_busy_timeout(std::time::Duration::from_millis(100)) + .unwrap(); + let _ = OpenMode::ReadOnly; + let _: fn(&Value) -> Result = ::from_value; + let _: fn(&Statement) -> usize = Statement::param_count; + let _: fn(&Error) -> bool = Error::is_retryable; +} + +/// The assertion with teeth: this file must not name the engine. +/// +/// `facade_is_sufficient_alone` above proves the facade is *enough* only so +/// long as nobody quietly widens it. If a future capability gap were +/// "fixed" by importing `sqlite_rs::pager`, that test would keep passing +/// and Requirement 6 would be silently false. Reading our own source is the +/// only way to catch it, and `tests/unit/layer_isolation.rs` already uses +/// this idiom for the engine's internal layering. +#[test] +fn this_file_names_no_engine_module() { + let source = include_str!("api_surface_test.rs"); + + // Comment lines are stripped before scanning. The prose here and above + // names engine modules deliberately — to say what must not appear — + // and matching on that would make the test fail on its own explanation. + // A real `use sqlite_rs::pager` is never inside a comment. + let code: String = source + .lines() + .filter(|line| !line.trim_start().starts_with("//")) + .collect::>() + .join("\n"); + + // Every module `src/lib.rs` declares apart from `api`. + const ENGINE: &[&str] = &[ + "btree", + "codegen", + "dump", + "format", + "header", + "integrity", + "pager", + "parser", + "planner", + "record", + "schema", + "sys", + "vdbe", + "vfs", + ]; + + for module in ENGINE { + let needle = format!("sqlite_rs::{module}"); + assert!( + !code.contains(&needle), + "{needle} appears in this file — either the facade has a gap that was \ + worked around by reaching into the engine, or this list is stale. \ + Requirement 6 says a consumer should never need to." + ); + } +} From 266d62a44b051ca5015bd231f96cda5b2a01e4c8 Mon Sep 17 00:00:00 2001 From: Diederik Siderius Date: Wed, 9 Sep 2026 21:16:06 +0200 Subject: [PATCH 11/14] docs: amend spec 013 to what was built, and add Requirement 8 (0.1.0 -> 0.2.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Amended in place rather than superseded: two specs describing `Connection` would double-count in `tools/assurance.py` and collide on domain. Status stays `draft`. The dashboard moves, which is the point --------------------------------------- main this branch Requirements 86 94 (planned excluded: 10 -> 3) Scenarios 276 299 Completeness 100% 100% Coverage 100% 100% Dead links 0 0 Spec 013 was 7/7 `(planned)` and therefore excluded from all scoring; it is now 0/8, and the three remaining planned requirements repo-wide belong to specs 002, 005 and 010. What changed beyond flipping the markers ---------------------------------------- **Req 1** restated around the retention rule, which is the half with the surprising semantics and the half that belongs to the connection. `last_insert_rowid` specified alongside it, with a scenario. Deleted "The only capability gap here", false once #694 landed. **Req 2** gained `open_in_memory`, a page-size scenario, and a read-only scenario. Read-only is now specified as *statement-level* enforcement, an ADR-0004 divergence: `Pager::open` calls `open_write` unconditionally (`src/pager.rs:419`), there is no read-only pager, and `VfsPageSource` bypasses `Pager` so it merges no WAL frames. Also records that creation was never verified against the oracle before now, and why (both `seed_db` helpers prefer the oracle and fall back to our path only when it is absent, so the two never ran together). **Req 3** gained `execute_batch`, `Error::ParamCount`, `Error::MultipleStatements`, and an explicit scoping of by-name column access to single-table `SELECT`s — `result_column_names` returns `column1, column2…` for joins and compounds (`src/codegen/prepare.rs:181`), and this spec's own acceptance list contains a `UNION`. **Req 4** replaced the unfalsifiable "all engine access happened on the connection's thread" with what is actually checkable, and replaced "the thread count is unchanged" with the structural argument (drop joins, so a worker that failed to terminate hangs the drop rather than leaking). Added the no-named-lifetimes constraint as the reason the worker thread is the only *expressible* design. It also had to be corrected on a claim it made: "Coordination between connections in one process is spec 007's file locks, as between processes." It is not. POSIX `fcntl` locks are `(process, inode)`-scoped, so two connections in one process do not exclude each other, and a write that reports success can be silently discarded. Measured, and now written down with `unixInodeInfo` named as the mechanism stock SQLite uses. **Req 5** `::ApiError` -> `::Error`; added the `sqlite_code()`/ `extended_sqlite_code()` MUST (both, under the names SQLite uses, rather than picking one); scoped the busy timeout to autocommit and said why; noted `Connection::pragma` cannot serve the nine introspection pragmas (ADR-0029, they live in the binary) and that `sqlite_master` is not queryable through `SELECT` at all. The hard-kill scenario now states what SIGKILL does *not* prove. **Req 6** split acceptance into Part A (on the tree) and Part B (*SQE*'s literal DDL, a ratchet on #687 and #697), and named the three documented header fields exhaustively instead of gesturing at them. **Req 7** `Statement::next_row` -> `Rows::next_row`; the memory scenario now names **both** constants; and it records that the plan must be non-blocking, because an `ORDER BY` with no usable index is a blocking operator and a first draft of the test measured one and failed at 33x, correctly. **Req 8 (new)** — a prepared statement must not run against a changed catalog. Not a caching concern: a program addresses tables by root page, and `DROP` returns that page to the freelist for a later `CREATE`. Every `Tests:` link points at a test that exists, with the symbol present — zero dead links, verified by `tools/assurance.py`. Gates: check-assurance PASS (100%/100%), no dead links. Spend: within estimate. Co-Authored-By: Claude Opus 5 --- .openspec/specs/013-embedding-api/spec.md | 442 ++++++++++++++++++---- 1 file changed, 378 insertions(+), 64 deletions(-) diff --git a/.openspec/specs/013-embedding-api/spec.md b/.openspec/specs/013-embedding-api/spec.md index 6698349e..598d83ff 100644 --- a/.openspec/specs/013-embedding-api/spec.md +++ b/.openspec/specs/013-embedding-api/spec.md @@ -1,6 +1,6 @@ --- domain: embedding-api -version: 0.1.0 +version: 0.2.0 status: draft date: 2026-08-28 --- @@ -156,17 +156,33 @@ changed, as `sqlite3_changes()` does, following SQLite's rules: a statement returning no rows does not reset it, and it counts rows changed rather than examined. -`execute_transaction_step` returns rows and the new autocommit flag, so a -caller cannot distinguish an `UPDATE` that matched from one that did not. Every -optimistic-concurrency scheme is built on that distinction. *SQE* swaps a -table's metadata pointer with a conditional `UPDATE` and treats zero rows -affected as a lost race; without the count that becomes SELECT-then-UPDATE in a -transaction, sound only while the consumer guarantees a single writer, and every -consumer reinvents it. - -**Implementation:** `src/api.rs::Connection::changes` (planned) - -**Tests:** `tests/unit/api_changes_test.rs` (planned) +Every optimistic-concurrency scheme is built on the distinction between an +`UPDATE` that matched and one that did not. *SQE* swaps a table's metadata +pointer with a conditional `UPDATE` and treats zero rows affected as a lost +race; without the count that becomes SELECT-then-UPDATE in a transaction, sound +only while the consumer guarantees a single writer, and every consumer +reinvents it. + +The rule with the surprising semantics is the *retention* one, and it belongs +to the connection rather than the engine: `sqlite3_changes()` reports what the +last *counting* statement did, so a `SELECT` or a DDL statement in between must +leave it standing. The engine half is `StepOutcome::changes` (#692), an +`Option` whose `None` means "not a counting statement, leave the stored +value alone" — `Some(0)` and `None` are different answers and the difference is +the whole point. + +`Connection::last_insert_rowid` follows the same retention rule and is +specified here with it: a row inserted into a table with a surrogate key is +unaddressable until the caller learns its rowid. It is driven by +`OPFLAG_LASTROWID` on `P5` rather than by the `Insert` opcode, because an +`UPDATE` also emits an `Insert` and must not move the value. + +**Implementation:** `src/api.rs::Connection::changes`, plus +`::Connection::last_insert_rowid` + +**Tests:** `tests/unit/api_changes_test.rs`, +`tests/unit/vdbe_last_insert_rowid_test.rs`, +`tests/corpus/last_insert_rowid_oracle_test.rs` #### Scenario: A conditional update reports whether it matched @@ -176,7 +192,7 @@ consumer reinvents it. - THEN the first reports one row changed, the second reports zero, and the pinned oracle agrees with both -**Tests:** `tests/unit/api_changes_test.rs::conditional_update_reports_match` (planned) +**Tests:** `tests/unit/api_changes_test.rs::conditional_update_reports_match`, `tests/corpus/api_oracle_test.rs::api_rows_affected_and_resulting_file_match_the_oracle` #### Scenario: A SELECT does not clobber the count @@ -184,7 +200,17 @@ consumer reinvents it. - WHEN a `SELECT` returning no rows runs next - THEN the count still reports two -**Tests:** `tests/unit/api_changes_test.rs::select_does_not_clobber_count` (planned) +**Tests:** `tests/unit/api_changes_test.rs::select_does_not_clobber_count` + +#### Scenario: An insert's rowid is retained until the next insert + +- GIVEN an `INSERT` into a table with an `INTEGER PRIMARY KEY` that supplied + its own key +- WHEN an `UPDATE`, a `DELETE` and a `SELECT` run next +- THEN the last-insert rowid still reports the supplied key, and the pinned + oracle agrees at every step + +**Tests:** `tests/unit/api_changes_test.rs::last_insert_rowid_is_retained_across_statements`, `tests/corpus/last_insert_rowid_oracle_test.rs::last_insert_rowid_matches_the_oracle` ### Requirement 2: Connection, Open or Create [MUST] @@ -196,9 +222,33 @@ handle MUST release them on drop, which `Pager` already does. *SQE* opens its catalog as `sqlite://?mode=rwc` and expects the file to appear on first use; a first-run laptop has no `empty.db` to copy. -**Implementation:** `src/api.rs::Connection::open` (planned), plus `::open_with` - -**Tests:** `tests/unit/api_connection_test.rs` (planned) +Creation writes `DatabaseHeader::new_empty_page1` (`src/header.rs:319`), the +same bootstrap the CLI's `exec` uses. That this produces a database stock +`sqlite3` accepts was, until this spec, **never verified**: both +`tests/tiers/tier2.rs::seed_db` and `tests/corpus/cli_write_test.rs::seed_db` +prefer the oracle to build their fixture and fall back to our own path only when +no oracle is installed, so the two never ran together. + +**Read-only is enforced per statement, not by the pager**, and that is a +deliberate divergence recorded under ADR-0004. `Pager::open` calls +`Vfs::open_write` unconditionally (`src/pager.rs:419`) and there is no +read-only pager; the read-only page source that does exist (`VfsPageSource`) +bypasses `Pager` entirely and so merges no WAL frames, which would silently +serve stale data for a WAL database. Refusing writes above the pager is the +honest option until spec 007 grows a read-only pager. The guard keys on +`OpenWrite` and the DDL opcodes rather than on `Insert`/`Delete`, because those +two also target ephemeral cursors: a materialized FROM-subquery emits an +`Insert` in a plain `SELECT`. + +An in-memory database (`open_in_memory`) is also provided, backed by +`MemoryVfs`, so it exercises the same pager, journal and b-tree code a file +does rather than a separate path. + +**Implementation:** `src/api.rs::Connection::open`, plus `::open_with`, +`::open_in_memory` and `::OpenMode` + +**Tests:** `tests/unit/api_connection_test.rs`, +`tests/corpus/api_oracle_test.rs`, `tests/corpus/bootstrap_oracle_test.rs` #### Scenario: Create produces a database the oracle reads @@ -206,7 +256,7 @@ appear on first use; a first-run laptop has no `empty.db` to copy. - WHEN opened `ReadWriteCreate` - THEN a valid database exists and the pinned oracle reports an empty schema -**Tests:** `tests/unit/api_connection_test.rs::create_then_oracle_reads_empty_schema` (planned) +**Tests:** `tests/corpus/api_oracle_test.rs::create_then_oracle_reads_empty_schema`, `tests/unit/api_connection_test.rs::open_creates_a_database_that_reopens_and_reads_back` #### Scenario: Without create, nothing is written @@ -214,7 +264,27 @@ appear on first use; a first-run laptop has no `empty.db` to copy. - WHEN opened `ReadWrite` - THEN it fails and no file exists afterwards -**Tests:** `tests/unit/api_connection_test.rs::readwrite_does_not_create` (planned) +**Tests:** `tests/unit/api_connection_test.rs::readwrite_does_not_create`, `tests/corpus/api_oracle_test.rs::readwrite_creates_nothing_the_oracle_can_find` + +#### Scenario: A created database is valid at every page size + +- GIVEN a page 1 built by `new_empty_page1` for each supported page size, + including the 65536 case that cannot be stored literally in the 16-bit field +- WHEN the pinned oracle opens it +- THEN `integrity_check` is `ok`, the page size round-trips, the schema is + empty, and the oracle can then grow the file + +**Tests:** `tests/corpus/bootstrap_oracle_test.rs::an_empty_database_we_build_is_valid_at_every_page_size` + +#### Scenario: Read-only refuses every write and permits every read + +- GIVEN a connection opened `ReadOnly` +- WHEN a `SELECT` whose plan materializes a FROM-subquery runs, and then an + `INSERT`, `UPDATE`, `DELETE`, `CREATE` and `DROP` are each attempted +- THEN the read succeeds, every write is refused with `SQLITE_READONLY`, and + the file is unchanged + +**Tests:** `tests/unit/api_connection_test.rs::readonly_reads_but_refuses_every_write`, `tests/unit/api_connection_test.rs::readonly_refusal_reports_sqlite_readonly` ### Requirement 3: Statement Handle [MUST] @@ -230,9 +300,28 @@ frequency, so compiling once saves nothing measurable; a handle owning its slots is what stops a transposed argument list writing a valid row that points at the wrong table. -**Implementation:** `src/api.rs::Statement` (planned), plus `::Row` - -**Tests:** `tests/unit/api_statement_test.rs` (planned) +Three things beyond the handle itself belong here, because they are what stop a +caller getting silently wrong answers rather than errors: + +- **Arity is checked exactly**, on every execution, and a mismatch is + `Error::ParamCount`. Stricter than stock SQLite, which leaves an unbound + parameter NULL — recorded as an ADR-0004 divergence, because refusing what + SQLite accepts is the safe direction and catching a transposed argument list + is this requirement's stated value. +- **`execute_batch`** runs a multi-statement script, and `execute`/`prepare` + refuse one with `Error::MultipleStatements`. A script is what schema setup + is; silently running only its first statement is not an option. +- **By-name column access is scoped to single-table `SELECT`s.** + `result_column_names` (`src/codegen/prepare.rs:181`) returns `column1`, + `column2`, … for a join or a compound, so by-name access will not find a + base-table name there. By-index access is unaffected. Stated rather than + hidden; the fix belongs in the name resolver and is not this spec's. + +**Implementation:** `src/api.rs::Statement`, plus `::Row`, `::FromValue` and +`::Connection::execute_batch` + +**Tests:** `tests/unit/api_statement_test.rs`, +`tests/unit/param_binding_test.rs`, `tests/unit/api_streaming_test.rs` #### Scenario: One compile, many bindings @@ -240,7 +329,7 @@ wrong table. - WHEN executed with 1, 2 and 3 bound - THEN each returns that row's name and compilation happened once -**Tests:** `tests/unit/api_statement_test.rs::compile_once_bind_many` (planned) +**Tests:** `tests/unit/api_statement_test.rs::compile_once_bind_many` #### Scenario: A named parameter is refused, not silently NULL @@ -248,7 +337,25 @@ wrong table. - WHEN prepared - THEN preparation fails naming the unsupported form -**Tests:** `tests/unit/api_statement_test.rs::named_param_is_refused_at_prepare` (planned) +**Tests:** `tests/unit/api_statement_test.rs::named_param_is_refused_at_prepare`, `tests/unit/param_binding_test.rs::named_parameters_are_refused_rather_than_bound_to_null` + +#### Scenario: A wrong argument count is refused, not padded with NULLs + +- GIVEN a prepared statement with two placeholders +- WHEN one value, and then three, are bound +- THEN each is refused with `SQLITE_RANGE`, nothing is written, and the handle + still works afterwards + +**Tests:** `tests/unit/api_statement_test.rs::a_wrong_argument_count_is_refused_every_time`, `tests/unit/api_changes_test.rs::the_wrong_number_of_parameters_is_refused` + +#### Scenario: Bound values of every storage class round-trip + +- GIVEN an `INSERT` binding an integer, a real, text, a blob and NULL +- WHEN the file is read back through the pinned oracle +- THEN the values and their `typeof()` match the oracle running the same + statements with the values inlined + +**Tests:** `tests/corpus/api_oracle_test.rs::parameterised_writes_match_the_oracle`, `tests/unit/api_streaming_test.rs::every_storage_class_reads_back_as_its_rust_type` ### Requirement 4: A `Send + Sync` Handle Over an Owned Worker Thread [MUST] @@ -273,31 +380,75 @@ driver does the same for a C `sqlite3*` channel, one per connection), so implementing it here gives every consumer once what each would otherwise write. -The thread MUST terminate on drop, and a request after it dies MUST error rather -than block. Coordination between connections in one process is spec 007's file -locks, as between processes; no shared cache, no global state. - -**Implementation:** `src/api.rs::Connection` (planned) +One further constraint makes the worker thread the only *expressible* design +rather than merely the preferred one, and it was not known when ADR-0041 was +written: `make check-mvl-limit` forbids named lifetime parameters in `src/`, so +no type in `src/api.rs` may hold an `Execution` — it borrows its `Program`. The +execution has to live inside a single worker stack frame, which is exactly what +this design gives it. (`src/codegen/stmt/insert.rs::ColumnSource` already +carries a comment making the same trade for the same reason.) -**Tests:** `tests/unit/api_threading_test.rs` (planned) +The thread MUST terminate on drop, and a request after it dies MUST error rather +than block. + +**Coordination between connections in one process is currently broken, and +that is a finding rather than a design choice.** This requirement previously +asserted it was "spec 007's file locks, as between processes". It is not: POSIX +`fcntl` locks are scoped to `(process, inode)`, which `src/vfs/lock.rs:96` +documents and `FileLockState::check_reserved_lock` states outright ("whether +some *other* process currently holds a write lock"). Two connections on one +file in one process therefore do not exclude each other. Measured: connection A +takes `BEGIN IMMEDIATE` and inserts a row; connection B's insert returns +`Ok(1)`; A commits; B's row is gone, and `integrity_check` reports `ok`. A +write that reported success is silently discarded. + +Stock SQLite closes this with `unixInodeInfo` in `os_unix.c` — a process-global +registry keyed by `(device, inode)` with its own mutex and lock counts, so two +connections in one process serialize like two processes. There is no equivalent +here. It is a pre-existing engine gap, but this requirement makes it reachable: +the whole point of a `Send + Sync` handle is that a *pool* can hold it. Tracked +as a ratchet +(`tests/unit/api_durability_test.rs::in_process_connections_lock_against_each_other`, +`#[ignore]`d) and needs its own ticket. No shared cache, no other global state. + +**Implementation:** `src/api.rs::Connection` + +**Tests:** `tests/unit/api_threading_test.rs` #### Scenario: The handle is shared across threads - GIVEN a connection opened on thread A - WHEN its handle is cloned into several threads that each run a query -- THEN every query succeeds, a static assertion proves the handle is - `Send + Sync`, and all engine access happened on the connection's thread +- THEN every query succeeds, and a compile-time assertion proves the handle is + `Send + Sync` + + The original third clause was "and all engine access happened on the + connection's thread", which is not falsifiable from outside — there is no + observable that distinguishes it. The testable statement of the same intent + is that **no engine type appears in the handle's public signature**, which + Requirement 6's surface test already asserts, plus the compile-time bound + above. -**Tests:** `tests/unit/api_threading_test.rs::handle_is_send_sync` (planned) +**Tests:** `tests/unit/api_threading_test.rs::handle_is_send_sync`, `tests/unit/api_threading_test.rs::a_shared_reference_works_across_threads` #### Scenario: The thread is released, and a dead engine errors - GIVEN a loop that opens and drops connections - WHEN it finishes -- THEN the thread count is unchanged, and a request on a dropped connection's - handle errors instead of blocking +- THEN every cycle's write committed and was visible to the next, which + requires each worker to have terminated and released its file locks before + the next connection opened + + "The thread count is unchanged" is asserted structurally instead of by + counting threads, for which there is no portable API here: `Drop` closes the + request channel and then *joins*, so a worker that failed to terminate would + hang the drop rather than leak. The test completing is the assertion. A + request on a dead worker returning `Error::ConnectionClosed` rather than + blocking is a property of the channel — and note it is unreachable through + the public API by construction, since the handle owns the sender that keeps + the worker alive. -**Tests:** `tests/unit/api_threading_test.rs::worker_thread_joins_on_drop` (planned) +**Tests:** `tests/unit/api_threading_test.rs::worker_thread_joins_on_drop`, `tests/unit/api_threading_test.rs::dropping_one_clone_leaves_the_rest_working` ### Requirement 5: Transactions and a Stated Durability Contract [MUST] @@ -321,13 +472,40 @@ Where a PRAGMA is accepted without being honored, record it as a divergence under ADR-0004. Retryable errors belong here too: spec 007's `VfsError::Locked` MUST surface as -a distinct, documented busy variant, and a busy timeout MUST be settable per -connection. - -**Implementation:** `src/api.rs::Transaction` (planned), plus `::Connection::pragma` -and `::ApiError` - -**Tests:** `tests/unit/api_transaction_test.rs`, `tests/unit/api_durability_test.rs` (planned) +a distinct, documented busy variant with `is_retryable()`, and a busy timeout +MUST be settable per connection. The error type MUST also expose SQLite's own +result codes — `sqlite_code()` for the primary and `extended_sqlite_code()` for +the extended, mirroring `sqlite3_errcode()`/`sqlite3_extended_errcode()` — so a +consumer can distinguish a UNIQUE violation (2067) from any other constraint +failure without matching on message text. + +Two constraints on how far the timeout goes: + +- **It retries only outside an explicit transaction.** In autocommit the + statement *is* the transaction, so rolling the pager back and re-running it + is a faithful retry. Inside one it is not: the statement's mutations share + the pending set with every earlier statement's, so re-running one would + double-apply it. A busy inside a transaction is the transaction's to retry, + which is what stock SQLite does with `SQLITE_BUSY` at `COMMIT`. +- **The rollback before each retry is load-bearing.** `Pager::flush` + (`src/pager.rs:524`) surfaces `VfsError::Locked` before any byte is + journaled and leaves `dirty` intact, so without clearing it a retried + `INSERT` would land once per attempt. + +`Connection::pragma` covers what a pool sets and what durability requires. It +**cannot** serve the introspection pragmas (`table_info` and the other eight): +those live in `src/bin/sqlite-rs/pragma_query.rs` per ADR-0029, inside the +binary. `Connection::table_names` covers the one introspection need a consumer +actually has, reading the decoded catalog — `sqlite_master` is not queryable +through `SELECT` at all today (`resolve_from_table_schema` does not resolve +it). The PRAGMA catalogue is plan.md's V7. + +**Implementation:** `src/api.rs::Transaction`, plus `::Connection::pragma`, +`::Connection::set_busy_timeout` and `::Error` + +**Tests:** `tests/unit/api_transaction_test.rs`, +`tests/unit/api_durability_test.rs`, +`tests/corpus/api_durability_oracle_test.rs` #### Scenario: Dropped transaction rolls back @@ -335,7 +513,7 @@ and `::ApiError` - WHEN the handle drops without `commit()` - THEN the row is absent and the pinned oracle agrees -**Tests:** `tests/unit/api_transaction_test.rs::drop_rolls_back` (planned) +**Tests:** `tests/unit/api_transaction_test.rs::drop_rolls_back`, `tests/unit/api_transaction_test.rs::an_early_return_rolls_back_and_leaves_the_connection_usable` #### Scenario: A committed transaction survives a hard kill @@ -343,16 +521,30 @@ and `::ApiError` - WHEN the process is killed without unwinding and the database reopened - THEN the rows are present and `integrity_check` passes under the oracle -**Tests:** `tests/unit/api_durability_test.rs::commit_survives_hard_kill` (planned) + What that does *not* establish, stated so the durability claim is not + overread: SIGKILL leaves the kernel page cache intact, so it cannot + distinguish `FULL` from `NORMAL` or `OFF`. It establishes that the commit was + complete in the file rather than buffered in the process, and that an abrupt + death leaves nothing malformed. Separating the `synchronous` levels needs a + power cut or a crash-injecting VFS, which is + `tests/corpus/crash_torture_test.rs`'s regime. + +**Tests:** `tests/corpus/api_durability_oracle_test.rs::commit_survives_hard_kill` #### Scenario: Busy is retryable and distinguishable -- GIVEN a second connection holding the WAL write lock +- GIVEN a second **process** holding the write lock - WHEN a write is attempted -- THEN the error is the busy variant, `is_retryable()` is true, and a retry - after release succeeds +- THEN the error is the busy variant, `is_retryable()` is true, `sqlite_code()` + is 5, and a retry after release succeeds -**Tests:** `tests/unit/api_durability_test.rs::busy_is_retryable` (planned) + A second *process*, not a second connection: two connections in one process + do not exclude each other at all — see Requirement 4. Using the pinned + `sqlite3` as the lock holder also makes the claim stronger, since the lock + protocol is then honoured against stock SQLite rather than only against + ourselves. + +**Tests:** `tests/corpus/api_durability_oracle_test.rs::busy_is_retryable`, `tests/corpus/api_durability_oracle_test.rs::a_retried_statement_succeeds_exactly_once` ### Requirement 6: Published Surface, Stability Policy, and Acceptance [MUST] @@ -375,10 +567,26 @@ parameters, `SELECT ... UNION` over two namespace sources, `LIMIT 1` existence probes, a conditional `UPDATE`, and `DELETE`. Every statement in it lands in V2 through V4. The gap was never SQL coverage. -**Implementation:** `src/lib.rs` (planned) — module docs, plus `CHANGELOG.md` -policy and `tests/corpus/fixtures/consumers/sqe/` +Acceptance lands in two parts, because one of them is blocked on bugs this +spec does not own: + +- **Part A** — the API's own oracle-diff family: rows-affected counts, the + resulting file read back through the pinned `sqlite3`, parameterised writes + across every storage class, and streamed reads compared row by row. On the + tree now. +- **Part B** — *SQE*'s literal statement list as a fixture family. The list + above includes `CREATE TABLE IF NOT EXISTS` with a three-column composite + `PRIMARY KEY`, and both halves of that are currently broken: #697 (the + `IF NOT EXISTS` guard is ignored, duplicating the `sqlite_master` row and + leaking a page) and #687 (no `sqlite_autoindex_*` is created for a declared + composite PK, so the file is malformed to stock `sqlite3` before any write). + A version using *SQE*'s own named-unique-index workaround passes today and + ships separately; the literal-DDL version is a ratchet on those two tickets. -**Tests:** `tests/unit/api_surface_test.rs`, `tests/corpus/consumer_sqe_test.rs` (planned) +**Implementation:** `src/lib.rs` — module docs, plus `CHANGELOG.md`'s API +stability policy and `src/api.rs`'s re-export of `Value` + +**Tests:** `tests/unit/api_surface_test.rs`, `tests/corpus/api_oracle_test.rs` #### Scenario: The facade needs no escape hatch @@ -387,16 +595,31 @@ policy and `tests/corpus/fixtures/consumers/sqe/` transaction and reads the rows-affected count - THEN it compiles without naming `pager`, `vdbe`, `codegen`, `dump` or `btree` -**Tests:** `tests/unit/api_surface_test.rs::facade_is_sufficient_alone` (planned) + Compiling is the assertion, and on its own it would rot: a future capability + gap "fixed" by importing an engine module would keep the test passing while + this requirement became silently false. So a second test reads the test + file's own source and fails if any of the fourteen engine modules is named + in it. + +**Tests:** `tests/unit/api_surface_test.rs::facade_is_sufficient_alone`, `tests/unit/api_surface_test.rs::this_file_names_no_engine_module` #### Scenario: The consumer corpus matches the oracle -- GIVEN the catalog statement set above, run through this API and through the - oracle +- GIVEN a consumer statement set run through this API and through the oracle - THEN both produce identical rows in order, identical rows-affected counts, and identical files modulo documented header fields -**Tests:** `tests/corpus/consumer_sqe_test.rs::catalog_statements_match_oracle` (planned) + The documented header fields are exactly three, asserted exhaustively rather + than waved at: the change counter (offset 24), version-valid-for (92) and the + SQLite version number (96). All three are absent from `DatabaseHeader`, so + they read as zero in a file we create. The change counter never advancing is + a real interop limitation — another SQLite connection holding a cached image + cannot learn our writes happened — but not a malformation, which is why + `integrity_check` misses it and a byte-level assertion is needed. + + This is Part A. Part B, *SQE*'s literal DDL, is a ratchet on #687 and #697. + +**Tests:** `tests/corpus/api_oracle_test.rs::api_rows_affected_and_resulting_file_match_the_oracle`, `tests/corpus/api_oracle_test.rs::queried_rows_match_the_oracle`, `tests/corpus/bootstrap_oracle_test.rs::our_fresh_header_differs_from_the_oracles_only_in_fields_we_do_not_model` ### Requirement 7: Incremental Row Access [MUST] @@ -417,13 +640,27 @@ Memory MUST be bounded by the rows the caller has actually pulled, not by the result set, and abandoning a partially-read statement MUST release its resources and its cursors without waiting for the rest. -**Implementation:** `src/api.rs::Statement::next_row` (planned), or an `Iterator` -impl, built on `src/vdbe/exec.rs::Execution::next_row` rather than on -`execute_with_db` -- #682 found the ordering matters, because a facade -retrofitted onto the materializing entry point cannot be made incremental -afterwards +Not an `Iterator` impl: collapsing `Result, Error>` into +`Option>` to fit the trait makes "the stream ended" and "the +stream failed" the same shape at the call site. ADR-0040 rejected an `Iterator` +on `Execution` for this reason and the reasoning carries. + +**Holding an undrained `Rows` blocks the connection**, and that follows from +Requirement 4's serialized access rather than being incidental: the engine +stays inside one execution until the result is drained or the handle dropped, +so no other statement on that connection runs meanwhile. A result that fits in +one channel batch completes and frees the worker whether it is read or not — a +deliberate mitigation for the easy accident of preparing a query and forgetting +it — but a larger one parks the worker until the handle goes. Dropping it +releases immediately, so this is a wait rather than a deadlock. -**Tests:** `tests/unit/api_streaming_test.rs` (planned) +**Implementation:** `src/api.rs::Rows::next_row`, built on +`src/vdbe/exec.rs::Execution::next_row` rather than on `execute_with_db` -- +#682 found the ordering matters, because a facade retrofitted onto the +materializing entry point cannot be made incremental afterwards + +**Tests:** `tests/unit/api_streaming_test.rs`, +`tests/corpus/api_oracle_test.rs` #### Scenario: A large result is read without materializing it @@ -445,7 +682,20 @@ afterwards Independence from result size is the property a consumer actually needs, and unlike proportionality it is testable. -**Tests:** `tests/unit/api_streaming_test.rs::partial_read_is_bounded` (planned) + There are **two** constants, not one. The page-cache floor above, and the + channel buffer: peak is roughly four batches of `CHUNK_ROWS` + (`src/api.rs`) — one being filled on the worker, two in the channel's + `CHUNK_SLOTS`, one held by the caller. Both are independent of the result + size, which is what makes the property hold. + + One caveat the test had to be corrected for: **the plan must be + non-blocking.** An `ORDER BY` with no usable index is a blocking operator — + the sorter consumes every row before emitting the first — so time-to-first-row + there is genuinely linear and streaming cannot change it. Stock SQLite sorts + the same way. A first draft of this test measured a sorted plan and failed at + 33x on a 50x larger table, correctly. + +**Tests:** `tests/unit/api_streaming_test.rs::partial_read_is_bounded`, `tests/unit/api_streaming_test.rs::blocking_plans_are_linear_by_nature` #### Scenario: Abandoning a statement releases it @@ -454,7 +704,71 @@ afterwards - THEN its cursors are released and a subsequent write on the same connection proceeds -**Tests:** `tests/unit/api_streaming_test.rs::abandoned_statement_releases_cursors` (planned) +**Tests:** `tests/unit/api_streaming_test.rs::abandoned_statement_releases_cursors`, `tests/unit/api_streaming_test.rs::dropping_a_large_unread_result_releases_the_connection`, `tests/unit/api_streaming_test.rs::an_unread_small_result_does_not_block_the_next_statement` + +### Requirement 8: A Prepared Statement Must Not Run Against a Changed Catalog [MUST] + +A statement compiled before a schema change MUST NOT execute against the old +plan: it MUST be recompiled, or the execution MUST fail. It MUST NOT read a +recycled root page. + +This is not a caching concern. A compiled program addresses tables by root +page, and `DROP` returns that page to the freelist for a later `CREATE` to +reuse — so running a stale program can read a page that now belongs to a +different table, with no error anywhere. The same applies to indexes in the +other direction: a write compiled before `CREATE INDEX` maintains no index +entries, leaving rows in the table that the index does not have, which is +exactly the malformation #685 was about. + +Recompiling is what `sqlite3_prepare_v2` does on `SQLITE_SCHEMA`, and is what +this specifies; failing would be safe too, but pushes a retry loop onto every +caller for something the connection can do itself. If recompilation fails +because the statement no longer compiles at all, the error MUST be reported and +MUST be repeatable rather than one-shot. + +The count of automatic recompilations MUST be observable, mirroring SQLite's +`SQLITE_STMTSTATUS_REPREPARE` ("the number of times that the prepared statement +has been automatically regenerated due to schema changes", `sqlite3.h:9274` at +the pinned 3.53.4). That is also what makes Requirement 3's "compilation +happened once" a checkable claim rather than an assertion about internals. + +**Implementation:** `src/api.rs::Statement::reprepare_count`, plus +`::Connection::prepare` + +**Tests:** `tests/unit/api_statement_test.rs`, +`tests/corpus/api_oracle_test.rs` + +#### Scenario: A statement recompiles after the schema moves + +- GIVEN a prepared `SELECT` that has run once +- WHEN an unrelated table is created and the statement runs again +- THEN it returns the right row, its reprepare count is 1, and it does not + recompile again while the schema holds still + +**Tests:** `tests/unit/api_statement_test.rs::a_statement_recompiles_after_a_schema_change` + +#### Scenario: A write prepared before an index maintains it afterwards + +- GIVEN an `INSERT` prepared before any index existed +- WHEN two indexes are created and the same handle inserts more rows +- THEN the pinned oracle reports `integrity_check = ok` and finds every row + through the index + + A unit test cannot make this claim: a stale program inserts the table row and + skips the index, but a freshly-compiled read may table-scan and find it + anyway. `integrity_check` is what detects a row present in the table with no + matching index entry. + +**Tests:** `tests/corpus/api_oracle_test.rs::a_prepared_write_after_create_index_keeps_the_file_valid` + +#### Scenario: A statement whose table is dropped reports the failure + +- GIVEN a prepared `SELECT` that has run once +- WHEN its table is dropped and the statement runs again +- THEN it fails rather than reusing the old program, and fails the same way on + every subsequent attempt + +**Tests:** `tests/unit/api_statement_test.rs::a_statement_whose_table_is_dropped_reports_the_failure` ## Not in this spec From e78824185452a2a49b14402100b07981b5206d1b Mon Sep 17 00:00:00 2001 From: Diederik Siderius Date: Wed, 9 Sep 2026 21:17:38 +0200 Subject: [PATCH 12/14] =?UTF-8?q?docs:=20ADR-0043=20=E2=80=94=20the=20embe?= =?UTF-8?q?dding=20API's=20failure=20surface?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four decisions in the facade close alternatives defensible enough to record, and CLAUDE.md's convention is that such a decision gets an ADR in the same PR. ADR-0041 settled where the API lives; it said nothing about what a failure looks like coming out of it. * The error type is **flat** — every payload a String, an i32 or a Copy enum. It has to be, because every failure crosses a channel from the worker, so it must be unconditionally Send + Sync + 'static; an error holding an Rc or borrowing engine state could not be returned at all. Rejected: the idiomatic wrapping error with `source()`, which cannot derive PartialEq and would need sixteen engine enums made Send + Sync to satisfy a facade. * **Both** result codes are exposed, under SQLite's own names — `sqlite_code()` primary, `extended_sqlite_code()` extended. Rejected: picking one, which was the open question in the plan and would have made the other unreachable. * **Busy is classified structurally**, never on message text, because a reworded Display would otherwise turn every retryable error permanent with no test failing. * **Busy retry is autocommit-only and rolls back first.** Rejected: retrying inside a transaction, which is unsound — it double-applies, since `Pager::flush` leaves the dirty set intact by design. * **A stale prepared statement recompiles** rather than failing, which is what `prepare_v2` does on SQLITE_SCHEMA. Rejected: failing, which pushes a retry loop onto every caller. Spec 013's "Decisions and rejected alternatives" line now cites both ADRs. Spend: within estimate. Co-Authored-By: Claude Opus 5 --- .../adr/0043-embedding-api-failure-surface.md | 127 ++++++++++++++++++ .openspec/adr/index.md | 1 + .openspec/specs/013-embedding-api/spec.md | 4 +- 3 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 .openspec/adr/0043-embedding-api-failure-surface.md diff --git a/.openspec/adr/0043-embedding-api-failure-surface.md b/.openspec/adr/0043-embedding-api-failure-surface.md new file mode 100644 index 00000000..60162e78 --- /dev/null +++ b/.openspec/adr/0043-embedding-api-failure-surface.md @@ -0,0 +1,127 @@ +# 0043 — The embedding API's failure surface: a flat error, both result codes, and autocommit-only busy retry + +**Status:** Accepted · **Date:** 2026-09-09 + +## Context + +Spec 013 Requirement 5 asks for three things that all land on the same type: +`VfsError::Locked` must surface as "a distinct, documented busy variant", a +busy timeout must be settable per connection, and a consumer must be able to +tell a UNIQUE violation from any other failure. ADR-0041 settled *where* the +API lives; it said nothing about what a failure looks like coming out of it. + +Four choices had to be made, and each closes an alternative that is defensible +enough to be worth writing down. + +**The error has to cross a thread.** Every failure travels back from the +connection's worker over a channel, so the error type must be `Send + Sync + +'static` unconditionally. An error that borrowed from engine state, or held an +`Rc`, could not be returned at all. That rules out the shape the sixteen engine +error enums have, several of which wrap layer errors by value. + +**The result code is two numbers, not one.** `sqlite3_errcode()` returns the +primary code (19 for any constraint violation) and +`sqlite3_extended_errcode()` the extended one (2067 for UNIQUE specifically). +A caller asking "is this a constraint problem?" wants the first; one +distinguishing UNIQUE from NOT NULL wants the second. Picking one would make +the other unreachable, and the question of which a future `sqlx-sqlite-rs` +driver needs was genuinely open. + +**A retried statement can be applied twice.** `Pager::flush` +(`src/pager.rs:524`) surfaces `VfsError::Locked` before any byte is journaled +and deliberately leaves `self.dirty` intact "so the caller can retry or roll +back". In autocommit, that dirty set is the statement's own work — already +applied in memory. Re-running the statement without discarding it first inserts +the row once per attempt. Measured before the rollback was added. + +**A prepared statement can outlive its plan.** A compiled program addresses +tables by root page, and `DROP` returns that page to the freelist for a later +`CREATE` to reuse. A statement prepared before a schema change and run after it +can read a page belonging to a different table, with no error anywhere. + +## Decision + +**The error type is flat.** `api::Error`'s every payload is a `String`, an +`i32` or a `Copy` enum; layer errors arrive as already-formatted `Display` +text. It therefore derives `PartialEq` and is unconditionally `Send + Sync + +'static`. `#[non_exhaustive]`, so variants can be added without a breaking +change. + +**Both result codes are exposed, under the names SQLite uses.** +`Error::sqlite_code()` returns the primary, `Error::extended_sqlite_code()` the +extended, and the primary is derived from the extended as the low byte — the +rule `sqlite3.h` encodes (`primary | (n<<8)`). `Error::is_retryable()` is true +for `Busy` and nothing else. + +**`Busy` is classified structurally, never by message text.** The match is on +`ExecError::FlushFailed(PagerError::Vfs(VfsError::Locked { .. }))` and the +`DumpError` equivalents, not on a substring. Requirement 5 makes busy a +distinct *retryable* variant, so a classification that a reworded `Display` +could silently break is the wrong trade: every busy error would become +permanent and no test would fail. + +**The busy timeout retries only in autocommit, and rolls back first.** In +autocommit the statement is the transaction, so `Pager::rollback` followed by +re-running it is a faithful retry of the whole unit. Inside an explicit +transaction it is not — the statement's mutations share the pending set with +every earlier statement's — so a busy there is reported immediately and is the +*transaction's* to retry. Stock SQLite behaves the same way with +`SQLITE_BUSY` at `COMMIT`. Backoff follows `sqliteDefaultBusyCallback`'s +ladder; the default timeout is zero, as SQLite's is. + +**A stale prepared statement is recompiled, not rejected.** The connection +carries a schema generation, bumped whenever the catalog is invalidated; a +statement compiled against an older one is recompiled on next use. That is what +`sqlite3_prepare_v2` does on `SQLITE_SCHEMA`. If it no longer compiles at all, +the failure is reported and the handle stays registered, so the error is +repeatable rather than one-shot. The count is observable through +`Statement::reprepare_count`, mirroring `SQLITE_STMTSTATUS_REPREPARE`. + +## Alternatives rejected + +**An error that wraps its layer error and implements `source()`.** The +idiomatic Rust shape, and it would give callers the full chain. Rejected +because it cannot derive `PartialEq` (so tests substring-match messages +instead of asserting errors), and because making sixteen engine enums +`Send + Sync` to satisfy the channel is a large change to satisfy a facade. +The cost is real and small: the engine's enums barely implement `source()` +themselves, and the message they format is the diagnostic. + +**One result code.** Simpler, and matches what most drivers expose. Rejected +because the two answer different questions and SQLite itself offers both; the +one-code version would have had to guess which, and the guess was open. + +**Retrying inside a transaction too.** More uniform, and superficially more +useful. Rejected as unsound: it double-applies. A variant that rolled the whole +transaction back and asked the caller to replay is a real design, but it +requires the caller's statements, which the connection does not keep. + +**Never retrying, and returning `Busy` for the caller to handle.** Honest, and +what the type already supports. Rejected because Requirement 5 makes a settable +timeout a MUST, and because every consumer would then write the same loop — +which is what the requirement exists to stop. + +**Failing a stale statement with a schema error.** Safe, and what SQLite's +older `sqlite3_prepare()` did. Rejected because it pushes a retry loop onto +every caller for something the connection can do itself, and `prepare_v2` +exists precisely because that was the wrong default. + +## Consequences + +The error type is comparable in tests, which is why the API suites assert +`Error::ParamCount { expected: 2, found: 1 }` rather than matching on text. No +`source()` chain is available to consumers; if one is ever needed, it is an +additive change to a `#[non_exhaustive]` enum. + +Busy handling is only exercisable against a *second process*, because two +connections in one process do not lock against each other at all (POSIX +`fcntl` is `(process, inode)`-scoped; stock SQLite closes this with +`unixInodeInfo`, and this crate has no equivalent). The corpus tests use the +pinned `sqlite3` as the lock holder, which also makes the claim stronger. The +in-process gap is tracked as a ratchet in +`tests/unit/api_durability_test.rs::in_process_connections_lock_against_each_other` +and is a pre-existing engine defect, not a consequence of this decision. + +`Statement::reprepare_count` is public API that exists partly to make a claim +testable ("compilation happened once"). That is acceptable because it is +SQLite's own counter rather than an invention. diff --git a/.openspec/adr/index.md b/.openspec/adr/index.md index f277b2ca..fa98d8fe 100644 --- a/.openspec/adr/index.md +++ b/.openspec/adr/index.md @@ -46,3 +46,4 @@ Specs record what the system must do; ADRs record **why it is shaped this way** | [0040](0040-streaming-execution-with-batch-as-wrapper.md) | One streaming execution primitive, with the batch path as its wrapper | 2026-09-01 | | [0041](0041-embedding-api-owns-the-connection-driver-out-of-tree.md) | The embedding API owns the connection; the `sqlx` driver stays out of tree | 2026-08-28 | | [0042](0042-rows-changed-counted-by-codegen-flag.md) | Codegen flags the one mutation that is a row change; `None` is not `Some(0)` | 2026-09-04 | +| [0043](0043-embedding-api-failure-surface.md) | The embedding API's failure surface: a flat error, both result codes, and autocommit-only busy retry | 2026-09-09 | diff --git a/.openspec/specs/013-embedding-api/spec.md b/.openspec/specs/013-embedding-api/spec.md index 598d83ff..c5996cad 100644 --- a/.openspec/specs/013-embedding-api/spec.md +++ b/.openspec/specs/013-embedding-api/spec.md @@ -20,7 +20,9 @@ query engine that stores catalog pointers in SQLite; cited as *SQE* where its measured need pins a decision). Requirement 1 is the one item on this list a consumer cannot work around. -Decisions and rejected alternatives: ADR-0041. +Decisions and rejected alternatives: ADR-0041 (where the API lives, and why +the `sqlx` driver stays out of tree), ADR-0043 (the failure surface: the error +type, the result codes, busy retry and schema refresh). ## Scope and inheritance From 9f2dbd6a9be938ff4831980402529848d886b417 Mon Sep 17 00:00:00 2001 From: Diederik Siderius Date: Thu, 10 Sep 2026 17:27:12 +0200 Subject: [PATCH 13/14] fix: close the gaps an audit of the consumer's spec proposal found (013) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven items, all found by checking what shipped against what the consumer actually asked for rather than against our own spec. The substantive one is Requirement 4's concurrency scenario. It asked for eight threads doing a hundred inserts each on a file, with the oracle confirming the result is well-formed. What existed was four threads doing one insert apiece, in memory, with no integrity check — which proves the handle compiles and does not crash, and barely queues two requests against each other. It is now 800 inserts through one handle, asserting the total, each thread's share individually, and survival across a reopen; the oracle half lives in `tests/corpus/api_oracle_test.rs` per the test-layout convention, against an indexed table so `integrity_check` has an index to validate. That matters because #685 was a class of write bug our own reads could not see. `parse_error` in the dispatcher was `format!("{other:?}")`, so an embedding consumer's error text contained Rust struct syntax — `Unsupported { message: "...", span: Span { line: 1, .. } }`. The parse outcome already carries a message and a position; it now reads as prose. A `PRAGMA` through `prepare` now names plan.md V7 and `Connection::pragma` instead of handing back a bare parser message about a pragma name. That was #695's one open acceptance criterion. The setting form still compiles, which is why the message has to be specific rather than "PRAGMA is unsupported". Two behaviours were already correct but untested, and the untested half of each is the one that would cost a consumer real data: refusing a file that is not a database must leave its bytes alone (`Pager::open` calls `Vfs::open_write` before the header is ever parsed, so nothing structural stops a future change from truncating it), and `ReadOnly` on a missing path must create nothing. Journal mode is recorded in the spec as a deliberate scoping rather than left as an omission: the consumer's proposal wanted it among the open options, and it is set with `pragma` instead because the mode is persistent state in the file's header, not a property of one handle's session. Five of the eight `tests/unit/api_*.rs` files imported `Value` from `sqlite_rs::record` while the commit next door declared `sqlite_rs::api` the supported surface. They now import it from `api`, which re-exports it. CHANGELOG gains the facade entry it should have had, under an `[Unreleased]` heading — the versioning policy above it governs which minor version a completed phase ships as, not where a change waits beforehand. 1670 tests (was 1666), corpus 405 (was 404), lint clean on both passes, assurance still 100%/100% with no dead links. Spend: small, on top of the facade's estimate. Co-Authored-By: Claude Opus 5 --- .openspec/specs/013-embedding-api/spec.md | 51 +++++++++++- CHANGELOG.md | 12 +++ src/api.rs | 27 ++++++- src/codegen/dispatch.rs | 23 +++++- tests/corpus/api_oracle_test.rs | 84 ++++++++++++++++++++ tests/unit/api_changes_test.rs | 3 +- tests/unit/api_connection_test.rs | 76 ++++++++++++++++++ tests/unit/api_statement_test.rs | 3 +- tests/unit/api_streaming_test.rs | 3 +- tests/unit/api_threading_test.rs | 94 ++++++++++++++++++++++- tests/unit/api_transaction_test.rs | 3 +- 11 files changed, 364 insertions(+), 15 deletions(-) diff --git a/.openspec/specs/013-embedding-api/spec.md b/.openspec/specs/013-embedding-api/spec.md index c5996cad..7b151295 100644 --- a/.openspec/specs/013-embedding-api/spec.md +++ b/.openspec/specs/013-embedding-api/spec.md @@ -246,6 +246,16 @@ An in-memory database (`open_in_memory`) is also provided, backed by `MemoryVfs`, so it exercises the same pager, journal and b-tree code a file does rather than a separate path. +**Journal mode is set after opening, not at open.** A consumer's proposal for +this requirement asked for `journal_mode: Delete | Wal` among the open +options, alongside read-only and create. It is not there: `Connection::pragma` +sets it, as `PRAGMA journal_mode = wal` does through any SQLite driver, and +the mode a database is in is persistent state in its header rather than a +property of one handle's session. Putting it in the open options would imply +a per-handle setting that does not exist, and would have to answer what +happens when two handles ask for different modes on one file. Deliberate, and +recorded here rather than left as an omission. + **Implementation:** `src/api.rs::Connection::open`, plus `::open_with`, `::open_in_memory` and `::OpenMode` @@ -278,6 +288,33 @@ does rather than a separate path. **Tests:** `tests/corpus/bootstrap_oracle_test.rs::an_empty_database_we_build_is_valid_at_every_page_size` +#### Scenario: A file that is not a database is refused, unchanged + +- GIVEN a file whose contents are not a SQLite database +- WHEN it is opened +- THEN the open fails naming what was wrong with the header, and the file's + bytes are byte-for-byte what they were + +**Tests:** `tests/unit/api_connection_test.rs::a_foreign_file_is_refused_without_being_touched` + +#### Scenario: Read-only on a missing file fails and creates nothing + +- GIVEN a path with no file at it +- WHEN opened `ReadOnly` +- THEN it fails and no file exists afterwards + +**Tests:** `tests/unit/api_connection_test.rs::readonly_on_a_missing_file_fails_and_creates_nothing` + +#### Scenario: Journal mode is reachable, and an unsupported pragma says where the rest live + +- GIVEN a connection +- WHEN `pragma("journal_mode", ...)` is set, and separately an introspection + pragma is prepared +- THEN the setting form is honoured, and the introspection form is refused + with a message naming plan.md V7 and `Connection::pragma` + +**Tests:** `tests/unit/api_connection_test.rs::an_unsupported_pragma_points_at_v7`, `tests/unit/api_transaction_test.rs::pragma_sets_a_value_the_engine_honours` + #### Scenario: Read-only refuses every write and permits every read - GIVEN a connection opened `ReadOnly` @@ -415,7 +452,7 @@ as a ratchet **Implementation:** `src/api.rs::Connection` -**Tests:** `tests/unit/api_threading_test.rs` +**Tests:** `tests/unit/api_threading_test.rs`, `tests/corpus/api_oracle_test.rs` #### Scenario: The handle is shared across threads @@ -433,6 +470,18 @@ as a ratchet **Tests:** `tests/unit/api_threading_test.rs::handle_is_send_sync`, `tests/unit/api_threading_test.rs::a_shared_reference_works_across_threads` +#### Scenario: Eight threads contending on one handle produce a valid file + +- GIVEN eight threads sharing one `Arc` on a file, each running a + hundred parameterised inserts against an indexed table +- WHEN every thread has joined +- THEN all eight hundred rows are present, each thread's hundred are + individually accounted for, the rows survive reopening the file, and the + pinned oracle reports `integrity_check` = `ok` with the same row count and + no duplicated key pair + +**Tests:** `tests/unit/api_threading_test.rs::eight_threads_sharing_one_handle_produce_a_valid_file`, `tests/corpus/api_oracle_test.rs::concurrent_writes_leave_a_file_the_oracle_accepts` + #### Scenario: The thread is released, and a dead engine errors - GIVEN a loop that opens and drops connections diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f24a3c4..b2d217c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,18 @@ If a consumer needs something only the engine offers, that is a gap in **Versioning policy:** one minor version per completed plan phase — the version number tells the plan's story, sub-steps stay inside a phase. V1 (READ CORE) = 0.1.0 through 0.4.0. *(History note: internal iterations briefly numbered 0.4.0–0.6.0 were renumbered into the phase scheme on 14 Aug 2026, before any tag or publication of those versions existed.)* +## [Unreleased] + +### Added + +- **Embedding API: the `Connection` facade (spec 013).** `sqlite_rs::api` + is now the supported surface for linking this crate into a host program: + `Connection` (`Send + Sync`, `Clone`, over an owned worker thread), + `Statement`, streaming `Rows`/`Row` with typed access, `Transaction`, + `OpenMode`, and a flat `Error` carrying both `sqlite_code()` and + `extended_sqlite_code()`. See the API stability policy above: `api` is the + promise, every other module is the engine. + ## [0.18.10] - 2026-08-31 ### Fixed diff --git a/src/api.rs b/src/api.rs index c2a98b72..440bfc71 100644 --- a/src/api.rs +++ b/src/api.rs @@ -2029,8 +2029,26 @@ impl Engine { return self.compile_select(sql); } let (schemas, views) = self.catalog()?; - crate::codegen::compile_statement(sql, schemas, views).map_err(|e| Error::Compile { - message: e.to_string(), + crate::codegen::compile_statement(sql, schemas, views).map_err(|e| { + // `PRAGMA = ` compiles; the introspection + // pragmas do not, and a consumer meeting one deserves to be + // told where they live rather than reading a parser message + // about an unsupported pragma name. The catalogue and its + // tiers are plan.md's V7 (ADR-0029 put the nine introspection + // pragmas in the CLI); spec 013's non-goals say this API + // covers only what a pool sets and what durability requires. + if is_pragma(sql) { + return Error::Compile { + message: format!( + "{e} — this API supports only `PRAGMA = ` \ + (see Connection::pragma); the introspection pragmas are \ + plan.md V7 and are available through the sqlite-rs CLI" + ), + }; + } + Error::Compile { + message: e.to_string(), + } }) } @@ -2092,6 +2110,11 @@ fn is_select(sql: &str) -> bool { .any(|kw| starts_with_keyword(head, kw)) } +/// Whether `sql` is a `PRAGMA` statement. +fn is_pragma(sql: &str) -> bool { + starts_with_keyword(sql.trim_start(), "PRAGMA") +} + /// Whether `sql` can change the `sqlite_master` catalog. /// /// Conservative by design: any statement starting with `CREATE`, `DROP` or diff --git a/src/codegen/dispatch.rs b/src/codegen/dispatch.rs index d9c7f3ed..578a145d 100644 --- a/src/codegen/dispatch.rs +++ b/src/codegen/dispatch.rs @@ -112,8 +112,27 @@ fn canonical(word: &str) -> &'static str { .unwrap_or("") } -fn parse_error(other: ParseOutcome) -> DispatchError { - DispatchError::ParseFailed(format!("{other:?}")) +/// Renders a rejected parse as prose. +/// +/// This used to be `format!("{other:?}")`, which put Rust struct syntax — +/// `Unsupported { message: "...", span: Span { line: 1, .. } }` — into a +/// message an embedding consumer sees verbatim. The parse outcome already +/// carries a human message and a position; use them. +fn parse_error(other: ParseOutcome) -> DispatchError { + match other { + ParseOutcome::Unsupported { message, span } | ParseOutcome::Invalid { message, span } => { + DispatchError::ParseFailed(format!( + "{message} (line {}, column {})", + span.line, span.column + )) + } + // Not reachable through the dispatcher, which only calls this on a + // rejection, but the match has to be total and a panic here would + // be a worse answer than a plain sentence. + ParseOutcome::Accepted(_) => { + DispatchError::ParseFailed("statement was accepted but not dispatched".to_string()) + } + } } /// Parses `sql`, picks the compiler for its leading keyword(s), and diff --git a/tests/corpus/api_oracle_test.rs b/tests/corpus/api_oracle_test.rs index d6ef92c3..afb61f7b 100644 --- a/tests/corpus/api_oracle_test.rs +++ b/tests/corpus/api_oracle_test.rs @@ -439,3 +439,87 @@ fn a_prepared_write_after_create_index_keeps_the_file_valid() { std::fs::remove_dir_all(&dir).ok(); } + +/// Requirement 4's contention scenario, judged by the oracle. +/// +/// Eight threads sharing one handle, a hundred inserts each. The unit +/// suite (`tests/unit/api_threading_test.rs::eight_threads_sharing_one_handle_produce_a_valid_file`) +/// checks that all 800 rows are there and that each thread's hundred +/// survived; what it cannot check is whether the *file* is well-formed. +/// +/// That distinction is not academic. #685 was a whole class of write bug +/// that our own reads could not see: a row present in the table with no +/// matching index entry reads back fine and fails `integrity_check`. A +/// concurrent write path that corrupted the free list or an interior page +/// could pass every assertion in the unit test and still produce a file +/// stock sqlite3 refuses. +#[test] +fn concurrent_writes_leave_a_file_the_oracle_accepts() { + let Some(bin) = pinned_oracle() else { + skip_no_oracle("concurrent_writes_leave_a_file_the_oracle_accepts"); + return; + }; + let db = scratch_dir("contention").join("threads.db"); + std::fs::remove_file(&db).ok(); + + const THREADS: i64 = 8; + const PER_THREAD: i64 = 100; + + { + let conn = std::sync::Arc::new(Connection::open(&db).unwrap()); + // An index, so index maintenance runs on every insert and the + // oracle's integrity_check has an index to validate against the + // table. Without one, integrity_check cannot see the failure mode + // this test exists for. + conn.execute_batch( + "CREATE TABLE t(thread INTEGER, seq INTEGER, payload TEXT); + CREATE INDEX t_thread_seq ON t(thread, seq);", + ) + .unwrap(); + + let mut handles = Vec::new(); + for thread in 0..THREADS { + let conn = std::sync::Arc::clone(&conn); + handles.push(std::thread::spawn(move || { + for seq in 0..PER_THREAD { + conn.execute_with( + "INSERT INTO t VALUES (?1, ?2, ?3)", + vec![ + Value::from(thread), + Value::from(seq), + Value::from(format!("t{thread}-s{seq}")), + ], + ) + .expect("every insert should succeed"); + } + })); + } + for handle in handles { + handle.join().expect("a worker thread panicked"); + } + } + + assert_eq!( + oracle_says(&bin, &db, "PRAGMA integrity_check;"), + "ok", + "800 concurrent inserts through one handle left a malformed file" + ); + assert_eq!( + oracle_says(&bin, &db, "SELECT count(*) FROM t;"), + (THREADS * PER_THREAD).to_string(), + "the oracle counts a different number of rows than we wrote" + ); + // Every (thread, seq) pair exactly once, read through the index the + // writes had to maintain. + assert_eq!( + oracle_says( + &bin, + &db, + "SELECT count(*) FROM (SELECT DISTINCT thread, seq FROM t);" + ), + (THREADS * PER_THREAD).to_string(), + "a (thread, seq) pair was written twice or lost" + ); + + std::fs::remove_file(&db).ok(); +} diff --git a/tests/unit/api_changes_test.rs b/tests/unit/api_changes_test.rs index 50298cf5..a109dec7 100644 --- a/tests/unit/api_changes_test.rs +++ b/tests/unit/api_changes_test.rs @@ -12,8 +12,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use sqlite_rs::api::{Connection, Error}; -use sqlite_rs::record::Value; +use sqlite_rs::api::{Connection, Error, Value}; fn seeded() -> Connection { let conn = Connection::open_in_memory().unwrap(); diff --git a/tests/unit/api_connection_test.rs b/tests/unit/api_connection_test.rs index 500c5eb7..7e01c250 100644 --- a/tests/unit/api_connection_test.rs +++ b/tests/unit/api_connection_test.rs @@ -210,3 +210,79 @@ fn a_batch_stops_at_the_first_failure() { conn.execute("DELETE FROM t WHERE a = 3").unwrap(); assert_eq!(conn.changes().unwrap(), 0); } + +/// Opening something that is not a database must fail, and — the half that +/// matters — must leave the file exactly as it was. +/// +/// `Pager::open` calls `Vfs::open_write` unconditionally +/// (`src/pager.rs:419`), so the file is opened for writing before its +/// header is ever parsed. Nothing in the type system stops a future change +/// from truncating or initialising it on the way to discovering it is not +/// a database, and a consumer pointed at the wrong path would lose the +/// file. So the byte comparison here is the point, not the error variant. +#[test] +fn a_foreign_file_is_refused_without_being_touched() { + let path = scratch("foreign"); + let original: &[u8] = b"hello world"; + std::fs::write(&path, original).unwrap(); + + let err = Connection::open(&path).expect_err("not a database"); + assert!( + matches!(err, Error::CannotOpen { .. }), + "expected CannotOpen, got {err:?}" + ); + assert!( + err.to_string().contains("magic"), + "the message should say what was wrong with it: {err}" + ); + + assert_eq!( + std::fs::read(&path).unwrap(), + original, + "refusing a foreign file must not modify it" + ); + clean(&path); +} + +/// `ReadOnly` on a path with no file is an error, not an empty database, +/// and it creates nothing. +#[test] +fn readonly_on_a_missing_file_fails_and_creates_nothing() { + let path = scratch("readonly-missing"); + clean(&path); + let dir = path.parent().unwrap().to_path_buf(); + std::fs::create_dir_all(&dir).unwrap(); + + let err = Connection::open_with(&path, OpenMode::ReadOnly) + .expect_err("read-only cannot open what does not exist"); + assert!( + matches!(err, Error::CannotOpen { .. }), + "expected CannotOpen, got {err:?}" + ); + assert!(!path.exists(), "read-only must not create the file"); + clean(&path); +} + +/// #695's open acceptance criterion: a `PRAGMA` that this API does not +/// support should say where the rest of them live, rather than handing back +/// a bare parser message about a pragma name. +#[test] +fn an_unsupported_pragma_points_at_v7() { + let conn = Connection::open_in_memory().unwrap(); + let err = conn + .prepare("PRAGMA table_info(t)") + .expect_err("introspection pragmas are not on this API"); + let text = err.to_string(); + assert!( + text.contains("V7"), + "the refusal should name the plan block that owns the catalogue: {text}" + ); + assert!( + text.contains("Connection::pragma"), + "and should point at what this API does support: {text}" + ); + + // The setting form still works, which is the whole reason the message + // has to be specific rather than "PRAGMA is unsupported". + conn.pragma("journal_mode", "delete").unwrap(); +} diff --git a/tests/unit/api_statement_test.rs b/tests/unit/api_statement_test.rs index a76c4df6..239ca212 100644 --- a/tests/unit/api_statement_test.rs +++ b/tests/unit/api_statement_test.rs @@ -16,8 +16,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use sqlite_rs::api::{Connection, Error}; -use sqlite_rs::record::Value; +use sqlite_rs::api::{Connection, Error, Value}; fn seeded() -> Connection { let conn = Connection::open_in_memory().unwrap(); diff --git a/tests/unit/api_streaming_test.rs b/tests/unit/api_streaming_test.rs index d965e994..a4b9aa53 100644 --- a/tests/unit/api_streaming_test.rs +++ b/tests/unit/api_streaming_test.rs @@ -18,8 +18,7 @@ use std::time::Instant; -use sqlite_rs::api::{Connection, Error}; -use sqlite_rs::record::Value; +use sqlite_rs::api::{Connection, Error, Value}; fn seeded(rows: i64) -> Connection { let conn = Connection::open_in_memory().unwrap(); diff --git a/tests/unit/api_threading_test.rs b/tests/unit/api_threading_test.rs index bebf3655..6945cc2b 100644 --- a/tests/unit/api_threading_test.rs +++ b/tests/unit/api_threading_test.rs @@ -18,8 +18,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; -use sqlite_rs::api::Connection; -use sqlite_rs::record::Value; +use sqlite_rs::api::{Connection, Value}; /// Compile-time proof, not a runtime check. If `Connection` ever stops /// being `Send + Sync` this fails to build, which is the point — @@ -100,6 +99,97 @@ fn a_shared_reference_works_across_threads() { assert_eq!(conn.execute("DELETE FROM t").unwrap(), 4); } +/// Requirement 4's contention scenario, on a real file: eight threads +/// sharing one handle, a hundred inserts each, and the oracle asked whether +/// the result is a well-formed database. +/// +/// The count matters. Four threads doing one insert apiece — which is what +/// this test used to be — proves the handle compiles and does not crash; it +/// barely queues two requests against each other. Eight hundred inserts +/// through one worker is what actually exercises the channel under +/// contention, and the row count is an exact figure that any lost or +/// double-applied request breaks. +/// +/// On a file rather than in memory, so the pager, journal and b-tree are +/// the real ones rather than the memory VFS's. +#[test] +fn eight_threads_sharing_one_handle_produce_a_valid_file() { + // No `clean` first: it removes the parent directory, and `scratch` + // has just created it. + let path = scratch("contention"); + + const THREADS: i64 = 8; + const PER_THREAD: i64 = 100; + + { + let conn = Arc::new(Connection::open(&path).unwrap()); + conn.execute("CREATE TABLE t(thread INTEGER, seq INTEGER)") + .unwrap(); + + let mut handles = Vec::new(); + for thread in 0..THREADS { + let conn = Arc::clone(&conn); + handles.push(std::thread::spawn(move || { + for seq in 0..PER_THREAD { + conn.execute_with( + "INSERT INTO t VALUES (?1, ?2)", + vec![Value::from(thread), Value::from(seq)], + ) + .expect("every insert should succeed"); + } + })); + } + for handle in handles { + handle.join().expect("a worker thread panicked"); + } + + let total: i64 = conn + .query_row("SELECT count(*) FROM t") + .unwrap() + .unwrap() + .get(0) + .unwrap(); + assert_eq!( + total, + THREADS * PER_THREAD, + "every insert must be applied exactly once" + ); + + // And no thread's rows were lost or duplicated individually, which a + // bare total would hide if one thread lost rows and another gained + // them. + for thread in 0..THREADS { + let n: i64 = conn + .query_row_with( + "SELECT count(*) FROM t WHERE thread = ?1", + vec![Value::from(thread)], + ) + .unwrap() + .unwrap() + .get(0) + .unwrap(); + assert_eq!(n, PER_THREAD, "thread {thread} lost or gained rows"); + } + } + + // Reopening proves the rows are on disk rather than only in a page + // cache the writing handle happened to hold. + let reopened = Connection::open(&path).unwrap(); + let total: i64 = reopened + .query_row("SELECT count(*) FROM t") + .unwrap() + .unwrap() + .get(0) + .unwrap(); + assert_eq!(total, THREADS * PER_THREAD, "rows did not survive reopen"); + drop(reopened); + + // Whether stock sqlite3 also calls the resulting file well-formed is + // asserted where the oracle lives: + // `tests/corpus/api_oracle_test.rs::concurrent_writes_leave_a_file_the_oracle_accepts`. + clean(&path); +} + /// The thread is released on drop, and this test would *hang* rather than /// fail if it were not. /// diff --git a/tests/unit/api_transaction_test.rs b/tests/unit/api_transaction_test.rs index 272b083f..c86e6759 100644 --- a/tests/unit/api_transaction_test.rs +++ b/tests/unit/api_transaction_test.rs @@ -12,8 +12,7 @@ use std::path::{Path, PathBuf}; -use sqlite_rs::api::{Connection, Error, TransactionBehavior}; -use sqlite_rs::record::Value; +use sqlite_rs::api::{Connection, Error, TransactionBehavior, Value}; fn scratch(label: &str) -> PathBuf { let dir = std::env::temp_dir().join(format!("sqlite-rs-api-tx-{}-{label}", std::process::id())); From 0ef830d063e9bc38c5a882c7651af89352c146ea Mon Sep 17 00:00:00 2001 From: Diederik Siderius Date: Fri, 11 Sep 2026 08:34:35 +0200 Subject: [PATCH 14/14] feat: a transaction holds the connection; other threads wait (013/Req 4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Requirement 4's serialization is per *statement* — the worker runs one request at a time — and a `Transaction` is several. `Connection` is `Clone` precisely so a pool or several tasks can hold it, so per-statement serialization left three interleavings reachable, reported by the first consumer to run two concurrent catalog commits: 1. another task's `BEGIN` inside an open transaction — refused, loud; 2. another task's statements committed with a transaction they know nothing about; 3. an autocommit write from a third task **rolled back** with someone else's transaction, having already returned `Ok(1)`. The third decides it. That is a write which reported success being silently discarded — the same failure class as two connections on one file not locking against each other, except reachable through a single `Connection`, which is the object this API tells consumers to share. A documented caveat cannot make a lost write visible, and the advice it would give ("hold your own mutex") is code every consumer would then write identically. `transaction_with` claims a slot on `Shared` before issuing `BEGIN` — before, not after, because a statement reaching the worker in between would land inside the transaction. Commit, rollback and `Drop` release it. The gate sits in `Connection::send`, which every request already funnels through. Three arms, and the middle one is load-bearing: the handle that *is* the transaction proceeds; the **thread that opened it** proceeds, because holding a `Transaction` and still using the original handle is what a single-threaded caller has always done and what SQLite does, and blocking there would be a deadlock against oneself rather than exclusion; everyone else waits. That arm is why `the_same_thread_may_still_use_the_connection_directly` hangs rather than fails if it is removed, which the test says out loud. Same-thread re-entry asking for a second transaction is `Error::TransactionActive`, not a wait: a nesting bug, with no `SAVEPOINT` to make nesting legitimate, and waiting would hide it as a hang. A `MutexGuard` in `Transaction` was the obvious shape and is not expressible — it carries a lifetime and `check-mvl-limit` forbids those in `src/`, the same constraint that made the worker thread the only expressible design. Hence a slot and a condvar. ADR-0045 records that, and why a raw `execute("BEGIN")` stays unguarded: nothing would release a slot it claimed. Five tests, including the silent case end to end — the other thread's write must wait, succeed, and survive a rollback that discards only its own row. Under a mutant that disables the gate, exactly the two cross-thread tests fail and the other thirteen pass. 1675 tests (was 1670), corpus 405, lint clean on both passes, assurance still 100%/100%. Spend: small-medium, on top of the facade's estimate. Co-Authored-By: Claude Opus 5 --- ...0045-a-transaction-holds-the-connection.md | 96 +++++++++ .openspec/adr/index.md | 1 + .openspec/specs/013-embedding-api/spec.md | 40 ++++ src/api.rs | 190 +++++++++++++++++- tests/unit/api_transaction_test.rs | 160 +++++++++++++++ 5 files changed, 481 insertions(+), 6 deletions(-) create mode 100644 .openspec/adr/0045-a-transaction-holds-the-connection.md diff --git a/.openspec/adr/0045-a-transaction-holds-the-connection.md b/.openspec/adr/0045-a-transaction-holds-the-connection.md new file mode 100644 index 00000000..3452fc09 --- /dev/null +++ b/.openspec/adr/0045-a-transaction-holds-the-connection.md @@ -0,0 +1,96 @@ +# ADR-0045: A transaction holds the connection; other threads wait + +**Date:** 2026-09-10 +**Status:** Accepted + +## Context + +Spec 013 Requirement 4 says statements on a `Connection` are serialized, and +they are: the worker thread runs one request at a time. But a `Transaction` +is several statements, and `Connection` is `Clone` precisely so a pool or +several async tasks can hold it. Per-statement serialization says nothing +about what happens between them. + +The first consumer to run two concurrent catalog commits found three +interleavings, reported against the facade: + +1. Task B's `BEGIN` lands inside task A's open transaction and is refused + with "cannot start a transaction within a transaction". Loud, and + survivable. +2. Task B's statements land inside A's transaction and are committed with + it — B's work becomes atomic with work it knows nothing about. +3. **An autocommit write from task C runs inside whichever transaction is + open and is rolled back with it.** `execute` returned `Ok(1)`; the row is + gone; nothing errored anywhere. + +The third is the one that decides this ADR. It is a write that reported +success being silently discarded, which is the same failure class as two +connections on one file not locking against each other — except this one is +reachable through a single `Connection`, which is the object the API tells +consumers to share. + +The consumer worked around it with a mutex around every call and said either +a fix or a documented caveat would do. + +## Decision + +`Connection::transaction` takes exclusion for the guard's lifetime. `Shared` +holds `Mutex>` plus a `Condvar`; `transaction_with` claims +the slot *before* issuing `BEGIN`, and `Transaction`'s commit, rollback and +`Drop` release it and wake the waiters. Every request passes through +`Connection::send`, which is the single choke point, so the gate lives there. + +Three arms, in order: + +- The handle **is** the transaction — `Transaction` holds a `Connection` + clone carrying the transaction's token. Proceeds. +- The handle is on the **thread that opened** the transaction. Proceeds. + Holding a `Transaction` and continuing to use the original handle is what + a single-threaded caller has always been able to do, and it matches + SQLite, where any statement on a connection with an open transaction runs + inside it. Blocking here would be a deadlock against oneself. +- Anyone else waits. + +Re-entry from the thread that already holds the transaction returns +`Error::TransactionActive` rather than waiting: that is a nesting bug, and +`SAVEPOINT` is out of scope, so there is nothing legitimate to nest. Waiting +would hide the bug as a hang. + +## Alternatives rejected + +**Document it and leave the behaviour.** The consumer explicitly offered +this and it costs one sentence. Rejected because of interleaving 3: a caveat +does not make a lost write visible, and the guidance it would give — "wrap +your own mutex around it" — is exactly the code every consumer would then +write identically. If the correct use of a type is to always hold a lock +around it, the type should hold the lock. + +**Hold a `MutexGuard` in `Transaction`.** The obvious shape, and not +expressible: `MutexGuard<'a, T>` carries a lifetime and `make check-mvl-limit` +forbids named lifetime parameters in `src/`. The same constraint that made +the worker thread the only expressible design (ADR-0041) applies here, which +is why this is a slot and a condvar rather than a guard. + +**Make every statement claim the slot, including a raw `BEGIN` through +`execute`.** Would close the gap for consumers who write `BEGIN` as SQL +rather than calling `transaction()`. Rejected because nothing would release +it: a caller who issues `BEGIN` and then returns early leaves the connection +wedged for every other thread, with no `Drop` to recover. Stock SQLite offers +no such protection either. `execute("BEGIN")` therefore stays unguarded, and +that is a documented limit rather than an oversight. + +## Consequences + +- A `Transaction` leaked rather than dropped blocks every other thread on + that connection, exactly as a leaked `MutexGuard` would. `Drop` is the + release, so this requires actively forgetting the value. +- Re-entry from a *different thread of the same async task* cannot be + distinguished from genuine contention and blocks. No API can see task + identity; the consumer wraps blocking calls in `spawn_blocking`, so one + task holds one thread for the duration, and the thread check covers it in + practice. +- Mixing `transaction()` with a raw `execute("BEGIN")` on another thread is + still unguarded, per the rejected alternative above. +- Throughput under contention drops to one transaction at a time per + connection. That is what the consumer already achieves with its own mutex, + and a connection is a single worker thread regardless. diff --git a/.openspec/adr/index.md b/.openspec/adr/index.md index fa98d8fe..6dfa2ef7 100644 --- a/.openspec/adr/index.md +++ b/.openspec/adr/index.md @@ -47,3 +47,4 @@ Specs record what the system must do; ADRs record **why it is shaped this way** | [0041](0041-embedding-api-owns-the-connection-driver-out-of-tree.md) | The embedding API owns the connection; the `sqlx` driver stays out of tree | 2026-08-28 | | [0042](0042-rows-changed-counted-by-codegen-flag.md) | Codegen flags the one mutation that is a row change; `None` is not `Some(0)` | 2026-09-04 | | [0043](0043-embedding-api-failure-surface.md) | The embedding API's failure surface: a flat error, both result codes, and autocommit-only busy retry | 2026-09-09 | +| [0045](0045-a-transaction-holds-the-connection.md) | A transaction holds the connection; other threads wait | 2026-09-10 | diff --git a/.openspec/specs/013-embedding-api/spec.md b/.openspec/specs/013-embedding-api/spec.md index 7b151295..66e8e09d 100644 --- a/.openspec/specs/013-embedding-api/spec.md +++ b/.openspec/specs/013-embedding-api/spec.md @@ -470,6 +470,46 @@ as a ratchet **Tests:** `tests/unit/api_threading_test.rs::handle_is_send_sync`, `tests/unit/api_threading_test.rs::a_shared_reference_works_across_threads` +A `Transaction` holds the connection for its lifetime, and other holders +wait. Requirement 4's serialization is per *statement* — the worker runs one +request at a time — and a transaction is several. Without exclusion, three +interleavings are reachable through the `Clone` this requirement exists to +allow: another task's `BEGIN` landing inside an open transaction (refused, +loud); another task's statements being committed with a transaction they know +nothing about; and an autocommit write from a third task being **rolled back** +with someone else's transaction, having already returned success. The last is +a lost write that reported success, so this is enforced rather than +documented (ADR-0045). + +The thread that opened the transaction may keep using the original handle — +that is what a single-threaded caller has always done, and it is what SQLite +does. Re-entry from that same thread asking for a *second* transaction is +`Error::TransactionActive` rather than a wait, because it is a nesting bug and +waiting would hide it as a hang. A raw `execute("BEGIN")` is not guarded: +nothing would release a slot it claimed. + +#### Scenario: A transaction excludes other threads for its lifetime + +- GIVEN one connection shared by two threads, one of which has opened a + transaction and written a row +- WHEN the other thread issues an autocommit write, and the transaction is + then rolled back +- THEN the other thread's write waits rather than joining the transaction, + succeeds once the transaction ends, and survives the rollback — which + discards only the transaction's own row + +**Tests:** `tests/unit/api_transaction_test.rs::another_threads_write_is_not_swallowed_by_a_rollback`, `tests/unit/api_transaction_test.rs::another_thread_waits_and_then_proceeds_after_a_commit` + +#### Scenario: Exclusion does not deadlock the thread that owns it + +- GIVEN a thread holding a `Transaction` +- WHEN it uses the original connection handle directly, and separately asks + for a second transaction +- THEN the statement runs inside the open transaction, and the second + transaction request is `Error::TransactionActive` rather than a wait + +**Tests:** `tests/unit/api_transaction_test.rs::the_same_thread_may_still_use_the_connection_directly`, `tests/unit/api_transaction_test.rs::a_second_transaction_on_the_same_thread_reports_rather_than_hangs`, `tests/unit/api_transaction_test.rs::the_slot_is_released_even_if_teardown_fails` + #### Scenario: Eight threads contending on one handle produce a valid file - GIVEN eight threads sharing one `Arc` on a file, each running a diff --git a/src/api.rs b/src/api.rs index 440bfc71..e53c0db7 100644 --- a/src/api.rs +++ b/src/api.rs @@ -39,9 +39,10 @@ //! phases; the protocol below is shaped to take them. use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc::{sync_channel, Receiver, SyncSender}; -use std::sync::{Arc, Mutex}; -use std::thread::JoinHandle; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread::{JoinHandle, ThreadId}; use std::time::{Duration, Instant}; use crate::header::{DatabaseHeader, DEFAULT_PAGE_SIZE}; @@ -247,6 +248,18 @@ pub enum Error { /// two ways: the connection was closed, or the worker panicked (which /// would be a bug in this crate). ConnectionClosed, + /// This thread already holds a transaction on this connection. + /// + /// Requirement 4's exclusion makes another holder of the same + /// connection *wait* for an open transaction rather than interleave + /// with it. Waiting is wrong for the thread that already owns it: that + /// is a nesting bug, and blocking would hide it as a hang. `SAVEPOINT` + /// is out of scope, so there is nothing legitimate to nest. + /// + /// Re-entry from a *different* thread of the same async task cannot be + /// distinguished from genuine contention and blocks, which is what a + /// mutex would do. + TransactionActive, /// A column was read as a type its value cannot convert to. TypeMismatch { /// The column, named if the statement has usable names, else its @@ -313,6 +326,7 @@ impl Error { Error::ColumnNotFound { .. } => code::ERROR, Error::MultipleStatements { .. } | Error::ConnectionClosed + | Error::TransactionActive | Error::StatementFinalized => code::MISUSE, Error::Sqlite { code, .. } => *code, Error::Busy { .. } => code::BUSY, @@ -375,6 +389,10 @@ impl std::fmt::Display for Error { Error::Corrupt { message } => write!(f, "database image is malformed: {message}"), Error::Io { message } => write!(f, "I/O error: {message}"), Error::ConnectionClosed => write!(f, "connection is closed"), + Error::TransactionActive => write!( + f, + "this thread already holds a transaction on this connection" + ), Error::StatementFinalized => write!(f, "statement has been finalized"), Error::TypeMismatch { column, @@ -404,6 +422,14 @@ impl std::error::Error for Error {} #[derive(Debug, Clone)] pub struct Connection { inner: Arc, + /// The transaction this handle speaks inside, if any. + /// + /// `None` for every handle a caller obtains from [`Connection::open`] + /// and friends. [`Connection::transaction`] builds one clone with the + /// token set and gives it to the [`Transaction`], which is how the + /// transaction's own statements pass the gate in [`Connection::send`] + /// while other holders of the same connection wait. + txn: Option, } /// The shared half of a [`Connection`], so clones address one worker. @@ -423,6 +449,38 @@ struct Shared { /// `JoinHandle` has to be owned to be joined, and because `Shared` is /// reachable from several threads until the last clone goes. worker: Mutex>>, + /// Which transaction, if any, currently owns this connection. + /// + /// Requirement 4's serialization is per *statement* — the worker runs + /// one request at a time. A transaction is several statements, and + /// without this slot two holders of one connection could interleave + /// them: one task's `BEGIN` landing inside another's open transaction, + /// and worse, an autocommit write from a third task running inside + /// whichever transaction happened to be open and being committed or + /// rolled back with it. That last case loses a write that reported + /// success, which is why this is a lock rather than a documented + /// caveat. + txn: Mutex>, + /// Signalled when [`Shared::txn`] goes back to `None`. + txn_free: Condvar, + /// Hands out transaction tokens. Monotonic so a token is never reused + /// and a stale handle cannot be mistaken for the current owner. + next_txn: AtomicU64, +} + +/// Who holds the transaction slot on a [`Shared`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct TxnOwner { + token: u64, + /// Recorded so re-entry from the same thread is an error rather than a + /// deadlock. A thread that already holds the transaction and asks for + /// another one has a bug, and blocking it forever would hide the bug + /// behind a hang; other threads genuinely should wait. + /// + /// This does not catch re-entry from a different thread of the same + /// async task, which no API can see. That case blocks, which is the + /// same thing a `Mutex` would do. + thread: ThreadId, } impl Drop for Shared { @@ -668,7 +726,11 @@ impl Connection { inner: Arc::new(Shared { requests: Some(request_tx), worker: Mutex::new(Some(handle)), + txn: Mutex::new(None), + txn_free: Condvar::new(), + next_txn: AtomicU64::new(1), }), + txn: None, }), Ok(Err(e)) => { // The worker returns straight after reporting a failure; @@ -838,9 +900,22 @@ impl Connection { /// Begins a transaction with the given locking behaviour. pub fn transaction_with(&self, behavior: TransactionBehavior) -> Result { - self.execute(behavior.statement())?; + // The slot is taken *before* `BEGIN`, not after: between the two + // there must be no window in which another holder's statement can + // reach the worker, or that statement lands inside this + // transaction. + let token = self.claim_transaction()?; + let conn = Connection { + inner: Arc::clone(&self.inner), + txn: Some(token), + }; + if let Err(e) = conn.execute(behavior.statement()) { + self.release_transaction(token); + return Err(e); + } Ok(Transaction { - conn: self.clone(), + conn, + token, done: false, }) } @@ -911,6 +986,7 @@ impl Connection { /// Hands `request` to the worker, or reports the worker is gone. fn send(&self, request: Request) -> Result<(), Error> { + self.await_turn(); self.inner .requests .as_ref() @@ -919,6 +995,98 @@ impl Connection { .map_err(|_| Error::ConnectionClosed) } + /// Blocks until this handle may speak to the worker. + /// + /// Free when no transaction is open, or when the open one is this + /// handle's own. Everyone else waits, which is what makes a + /// [`Transaction`] a unit rather than a sequence other holders of the + /// connection can interleave with. + /// + /// A poisoned lock is recovered rather than propagated: the state it + /// guards is a single `Option`, a panicking holder cannot leave it + /// torn, and turning every subsequent statement into an error because + /// one unrelated caller panicked would be a worse failure than + /// continuing. + fn await_turn(&self) { + let mut slot = self + .inner + .txn + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let me = std::thread::current().id(); + loop { + match *slot { + None => return, + // This handle *is* the transaction. + Some(owner) if Some(owner.token) == self.txn => return, + // The thread that opened the transaction, speaking through + // the original handle rather than the guard. Letting it + // through preserves what a single-threaded caller has + // always been able to do — hold a `Transaction` and keep + // using `conn` — and matches SQLite, where any statement on + // a connection with an open transaction runs inside it. + // + // Blocking here instead would not be exclusion, it would be + // a deadlock against oneself, and a worse bug than the one + // this slot exists to fix. + Some(owner) if owner.thread == me => return, + Some(_) => { + slot = self + .inner + .txn_free + .wait(slot) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + } + } + } + + /// Takes the transaction slot, waiting for another thread's transaction + /// to finish, and returns the token that identifies this one. + fn claim_transaction(&self) -> Result { + let me = std::thread::current().id(); + let mut slot = self + .inner + .txn + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + loop { + match *slot { + Some(owner) if owner.thread == me => return Err(Error::TransactionActive), + Some(_) => { + slot = self + .inner + .txn_free + .wait(slot) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + None => { + let token = self.inner.next_txn.fetch_add(1, Ordering::Relaxed); + *slot = Some(TxnOwner { token, thread: me }); + return Ok(token); + } + } + } + } + + /// Releases the transaction slot and wakes whoever is waiting. + fn release_transaction(&self, token: u64) { + let mut slot = self + .inner + .txn + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if slot.map(|owner| owner.token) == Some(token) { + *slot = None; + // `notify_all` rather than `notify_one`: the waiters are not + // interchangeable. Some want the slot (`claim_transaction`) and + // some only want it empty (`await_turn`), and waking a single + // arbitrary one can wake a claimer while autocommit callers + // keep waiting behind it for no reason. + self.inner.txn_free.notify_all(); + } + } + /// Waits for the worker's answer, or reports the worker is gone. /// /// The two halves are separate because either can be the one to notice: @@ -1230,6 +1398,7 @@ fn storage_class(value: &Value) -> &'static str { #[derive(Debug)] pub struct Transaction { conn: Connection, + token: u64, done: bool, } @@ -1237,7 +1406,13 @@ impl Transaction { /// Commits the transaction. pub fn commit(mut self) -> Result<(), Error> { self.done = true; - self.conn.execute("COMMIT").map(drop) + let result = self.conn.execute("COMMIT").map(drop); + // Released whether or not `COMMIT` succeeded. A failed commit + // leaves the engine's transaction state to the engine; what must + // not happen is the slot staying held by a `Transaction` that is + // being dropped, because nothing would ever free it. + self.conn.release_transaction(self.token); + result } /// Rolls the transaction back. @@ -1246,7 +1421,9 @@ impl Transaction { /// reported rather than discarded. pub fn rollback(mut self) -> Result<(), Error> { self.done = true; - self.conn.execute("ROLLBACK").map(drop) + let result = self.conn.execute("ROLLBACK").map(drop); + self.conn.release_transaction(self.token); + result } /// The connection this transaction runs on. @@ -1274,6 +1451,7 @@ impl Drop for Transaction { // on disk is unchanged — which is the outcome a rollback wanted. // A caller who needs to see the error calls `rollback()`. self.conn.execute("ROLLBACK").ok(); + self.conn.release_transaction(self.token); } } diff --git a/tests/unit/api_transaction_test.rs b/tests/unit/api_transaction_test.rs index c86e6759..499b6799 100644 --- a/tests/unit/api_transaction_test.rs +++ b/tests/unit/api_transaction_test.rs @@ -12,6 +12,8 @@ use std::path::{Path, PathBuf}; +use std::sync::Arc; + use sqlite_rs::api::{Connection, Error, TransactionBehavior, Value}; fn scratch(label: &str) -> PathBuf { @@ -225,3 +227,161 @@ fn a_rolled_back_statement_still_reported_its_count() { // ...but the count is not retroactively revised, matching SQLite. assert_eq!(conn.changes().unwrap(), 1); } + +/// A `Transaction` is a unit, not a sequence other holders of the same +/// connection can interleave with (spec 013 Requirement 4). +/// +/// Requirement 4's serialization is per *statement* — the worker runs one +/// request at a time. That is not enough for a transaction, which is +/// several. A consumer running two concurrent catalog commits hit the +/// visible half of this: the second task's `BEGIN` landed inside the +/// first's open transaction and was refused. +/// +/// This test pins the invisible half, which is worse. Without exclusion an +/// autocommit write from another task runs inside whichever transaction +/// happens to be open, and is committed or rolled back with it — so a write +/// that returned `Ok(1)` disappears when an unrelated task rolls back. That +/// is a lost write that reported success. +#[test] +fn another_threads_write_is_not_swallowed_by_a_rollback() { + let conn = Arc::new(Connection::open_in_memory().unwrap()); + conn.execute("CREATE TABLE t(who TEXT)").unwrap(); + + let tx = conn.transaction().unwrap(); + tx.execute_with("INSERT INTO t VALUES (?1)", vec![Value::from("in-txn")]) + .unwrap(); + + // Another thread's autocommit write. It must not join this + // transaction, so it has to wait for it — the thread parks inside + // `execute_with` until the rollback below releases the slot. + let other = { + let conn = Arc::clone(&conn); + std::thread::spawn(move || { + conn.execute_with("INSERT INTO t VALUES (?1)", vec![Value::from("other")]) + .expect("the other thread's insert should succeed") + }) + }; + + // Give the other thread a chance to be blocked rather than merely slow, + // so this test is about exclusion and not about scheduling luck. + std::thread::sleep(std::time::Duration::from_millis(50)); + assert!( + !other.is_finished(), + "the other thread's write should be waiting for the open transaction" + ); + + tx.rollback().unwrap(); + + let applied = other.join().expect("the other thread panicked"); + assert_eq!(applied, 1); + + // The rollback discarded its own row and nothing else. Without the + // slot, `other`'s insert would have been inside the transaction and + // would have gone with it, leaving zero rows. + let rows = conn.query_all("SELECT who FROM t").unwrap(); + let names: Vec = rows.iter().map(|r| r.get::(0).unwrap()).collect(); + assert_eq!( + names, + vec!["other".to_string()], + "the rollback should discard only its own write" + ); +} + +/// The other side of the same rule: a committed transaction and a waiting +/// thread both end up applied, in that order. +#[test] +fn another_thread_waits_and_then_proceeds_after_a_commit() { + let conn = Arc::new(Connection::open_in_memory().unwrap()); + conn.execute("CREATE TABLE t(who TEXT)").unwrap(); + + let tx = conn.transaction().unwrap(); + tx.execute_with("INSERT INTO t VALUES (?1)", vec![Value::from("in-txn")]) + .unwrap(); + + let other = { + let conn = Arc::clone(&conn); + std::thread::spawn(move || { + conn.execute_with("INSERT INTO t VALUES (?1)", vec![Value::from("other")]) + .unwrap() + }) + }; + std::thread::sleep(std::time::Duration::from_millis(50)); + assert!(!other.is_finished(), "should be waiting"); + + tx.commit().unwrap(); + other.join().expect("the other thread panicked"); + + let count: i64 = conn + .query_row("SELECT count(*) FROM t") + .unwrap() + .unwrap() + .get(0) + .unwrap(); + assert_eq!(count, 2, "both writes should be present after the commit"); +} + +/// Exclusion must not become a deadlock against oneself. Holding a +/// `Transaction` and continuing to use the original handle on the same +/// thread is what a single-threaded caller has always been able to do, and +/// what SQLite does: the statement runs inside the open transaction. +/// +/// Without the same-thread arm in the gate this test hangs rather than +/// fails, which is worth saying out loud. +#[test] +fn the_same_thread_may_still_use_the_connection_directly() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute("CREATE TABLE t(a INTEGER)").unwrap(); + + let tx = conn.transaction().unwrap(); + tx.execute("INSERT INTO t VALUES (1)").unwrap(); + // Same thread, original handle, transaction still open. + conn.execute("INSERT INTO t VALUES (2)").unwrap(); + tx.rollback().unwrap(); + + let count: i64 = conn + .query_row("SELECT count(*) FROM t") + .unwrap() + .unwrap() + .get(0) + .unwrap(); + assert_eq!( + count, 0, + "both inserts were inside the transaction, so the rollback took both" + ); +} + +/// Re-entry on the thread that already holds the transaction is a nesting +/// bug and is reported, not waited on. +#[test] +fn a_second_transaction_on_the_same_thread_reports_rather_than_hangs() { + let conn = Connection::open_in_memory().unwrap(); + let _outer = conn.transaction().unwrap(); + assert_eq!( + conn.transaction().expect_err("nesting is refused"), + Error::TransactionActive + ); +} + +/// Dropping the guard releases the slot even when the rollback itself +/// fails, so a failed teardown cannot wedge the connection for everyone +/// else. +#[test] +fn the_slot_is_released_even_if_teardown_fails() { + let conn = Arc::new(Connection::open_in_memory().unwrap()); + conn.execute("CREATE TABLE t(a INTEGER)").unwrap(); + + { + let tx = conn.transaction().unwrap(); + tx.execute("INSERT INTO t VALUES (1)").unwrap(); + // Roll back underneath the guard, so the guard's own ROLLBACK on + // drop has nothing to roll back and errors. + conn.execute("ROLLBACK").unwrap(); + } + + // If the slot leaked, this would hang rather than fail. + let other = { + let conn = Arc::clone(&conn); + std::thread::spawn(move || conn.execute("INSERT INTO t VALUES (2)").unwrap()) + }; + assert_eq!(other.join().expect("the other thread panicked"), 1); +}