fix(store): sync the current directory when saving to a bare file name

docs/books/src/reference/vector-search.md

@@ -1,30 +1,58 @@
 # Vector Search
 
-mq-db does not compute embeddings. Compute them with any model, store them in a custom table as JSON arrays, and rank with `cosine_similarity`, `dot_product` or `l2_distance`.
+mq-db does not compute embeddings. Compute them with any model, keep them in a custom table as JSON arrays, and rank with `cosine_similarity`, `dot_product` or `l2_distance`.
 
-Key each embedding on `blocks.content_hash`, not `blocks.id`. The hash depends only on the block's type and content, so an embedding keeps matching its block when the block moves or other documents change.
+## Key embeddings on `content_hash`
+
+`blocks.id` shifts when an earlier document changes, so do not key embeddings on it. [`blocks.content_hash`](sql-schema.md) depends only on a block's type and content: an unchanged block keeps its embedding across re-indexing and moves, and an edited block has a new hash, so it shows up as missing.
+
+## Workflow
+
+```bash
+# 1. Export the blocks that have no embedding yet (all of them the first time).
+mq-db sql --db store.mq-db --format json \
+  "SELECT DISTINCT content_hash, content FROM blocks WHERE block_type <> 'yaml'" > todo.json
+
+# 2. Embed `content` with your model and write one JSON object per line:
+#    {"content_hash": "1c8d281685b6b265", "embedding": [0.12, -0.03, ...]}
+#    -> embeddings.jsonl
+
+# 3. Load them.
+mq-db sql --db store.mq-db \
+  "CREATE TABLE emb AS SELECT * FROM read_json('embeddings.jsonl')"
+```
+
+After documents change and are re-indexed, list what needs embedding and append it:
+
+```sql
+SELECT DISTINCT content_hash, content FROM blocks
+WHERE content_hash NOT IN (SELECT content_hash FROM emb);
+
+INSERT INTO emb SELECT * FROM read_json('new-embeddings.jsonl');
+```
+
+Drop embeddings of blocks that no longer exist:
 
 ```sql
--- Hand these to your embedding model.
-SELECT DISTINCT content_hash, content FROM blocks;
+CREATE OR REPLACE TABLE emb AS
+SELECT * FROM emb WHERE content_hash IN (SELECT content_hash FROM blocks);
+```
 
--- One JSON object per line: {"content_hash": "9f86d0...", "embedding": [0.12, -0.03, ...]}
-CREATE TABLE emb AS SELECT * FROM read_json('embeddings.jsonl');
+## Searching
 
-SELECT b.content
-FROM emb e JOIN blocks b ON b.content_hash = e.content_hash
+```sql
+SELECT d.path, b.anchor, b.content
+FROM emb e
+JOIN blocks b ON b.content_hash = e.content_hash
+JOIN documents d ON d.id = b.document_id
 ORDER BY cosine_similarity(e.embedding, '[0.1, 0.2, 0.3]') DESC
 LIMIT 5;
 ```
 
-A block whose content changed gets a new hash, so it simply stops matching. Find the blocks that still need an embedding:
+Identical text in several places shares one embedding and appears once per location. Use `(documents.path, anchor)` instead when each location needs its own row. Integers compare equal to their text form (`1` and `'1'`), so a text column can join an integer key.
 
-```sql
-SELECT DISTINCT b.content_hash, b.content
-FROM blocks b LEFT JOIN emb e ON b.content_hash = e.content_hash
-WHERE e.content_hash IS NULL;
-```
+Pass the query vector as a string literal. A row whose vector is invalid or has a different length scores `NULL`; `NULL` sorts lowest, so those rows come last under `DESC`.
 
-Equal content shares one hash, so one embedding covers every copy. Use `(documents.path, anchor)` instead when each location needs its own row. `blocks.id` also works for a throwaway table, but it shifts whenever an earlier document gains or loses blocks. Integers compare equal to their text form (`1` and `'1'`), so a text column can join an integer key.
+## Limits
 
 Every query scans the whole table, which is fast enough for tens of thousands of vectors (about one second for 20,000 vectors of 384 dimensions) but has no index.

src/sql/aggregate.rs

@@ -521,7 +521,7 @@ impl<'a> SqlEngine<'a> {
                 for (expr, desc) in &sort_keys {
                     let va = eval_agg_expr_value(expr, group_by_exprs, &a.0, &a.1);
                     let vb = eval_agg_expr_value(expr, group_by_exprs, &b.0, &b.1);
-                    let mut ord = va.cmp_val(&vb).unwrap_or(std::cmp::Ordering::Equal);
+                    let mut ord = va.sort_cmp(&vb);
                     if *desc {
                         ord = ord.reverse();
                     }

src/sql/result_ops.rs

@@ -24,9 +24,7 @@ pub(super) fn apply_order_by_exprs(rows: &mut [(Row, Vec<String>)], exprs: &[Ord
     let mut order: Vec<usize> = (0..rows.len()).collect();
     order.sort_by(|&a, &b| {
         for (i, ob) in exprs.iter().enumerate() {
-            let ord = keys[a][i]
-                .cmp_val(&keys[b][i])
-                .unwrap_or(std::cmp::Ordering::Equal);
+            let ord = keys[a][i].sort_cmp(&keys[b][i]);
             // sort=None or Asc → ascending; Desc → descending
             let ord = if ob.options.sort == Some(OrderBySort::Desc) {
                 ord.reverse()

src/sql/tests.rs

@@ -3456,3 +3456,56 @@ fn test_sql_distinct_content_hash() {
         .unwrap();
     assert_eq!(out.rows.len(), 1);
 }
+
+#[test]
+fn test_sql_order_by_mixed_null_and_numbers_is_total() {
+    let store = DocumentStore::new();
+    let engine = SqlEngine::new(&store).unwrap();
+    engine
+        .execute("CREATE TABLE v (name TEXT, embedding TEXT)")
+        .unwrap();
+    let values: Vec<String> = (0..200)
+        .map(|i| {
+            if i % 3 == 0 {
+                format!("('n{i}', 'bad')")
+            } else {
+                format!("('n{i}', '[1,{i}]')")
+            }
+        })
+        .collect();
+    engine
+        .execute(&format!("INSERT INTO v VALUES {}", values.join(",")))
+        .unwrap();
+    let out = engine
+        .execute("SELECT name, cosine_similarity(embedding, '[1,0]') AS s FROM v ORDER BY s DESC")
+        .unwrap();
+    assert_eq!(out.rows.len(), 200);
+    let nulls = out.rows.iter().filter(|r| r[1] == "NULL").count();
+    assert_eq!(nulls, 67);
+    // Every number must come before every NULL.
+    let first_null = out.rows.iter().position(|r| r[1] == "NULL").unwrap();
+    assert!(out.rows[first_null..].iter().all(|r| r[1] == "NULL"));
+    // And the numbers must be in descending order.
+    let nums: Vec<f64> = out.rows[..first_null]
+        .iter()
+        .map(|r| r[1].parse().unwrap())
+        .collect();
+    assert!(nums.windows(2).all(|w| w[0] >= w[1]));
+}
+
+#[test]
+fn test_sql_order_by_mixed_integer_and_numeric_string_keys() {
+    let store = DocumentStore::new();
+    let engine = SqlEngine::new(&store).unwrap();
+    engine.execute("CREATE TABLE v (name TEXT)").unwrap();
+    engine
+        .execute("INSERT INTO v VALUES ('first'), ('second'), ('third')")
+        .unwrap();
+    let out = engine
+        .execute(
+            "SELECT name FROM v ORDER BY CASE WHEN name = 'first' THEN 10 ELSE '2' END ASC, name",
+        )
+        .unwrap();
+    let names: Vec<&str> = out.rows.iter().map(|r| r[0].as_str()).collect();
+    assert_eq!(names, vec!["second", "third", "first"]);
+}

src/sql/value.rs

@@ -61,6 +61,29 @@ impl Value {
         }
     }
 
+    /// A total order for `ORDER BY`: `NULL`, then booleans, then numbers
+    /// (integers, floats and canonical integer strings, by exact numeric
+    /// value; `NaN` last), then other strings. `NULL` sorts lowest, so it
+    /// comes last under `DESC`.
+    pub(super) fn sort_cmp(&self, other: &Value) -> std::cmp::Ordering {
+        fn rank(v: &Value) -> u8 {
+            match v {
+                Value::Null => 0,
+                Value::Bool(_) => 1,
+                _ if sort_number(v).is_some() => 2,
+                _ => 3,
+            }
+        }
+        match (self, other) {
+            (Value::Bool(a), Value::Bool(b)) => a.cmp(b),
+            (Value::Str(a), Value::Str(b)) if rank(self) == 3 && rank(other) == 3 => a.cmp(b),
+            _ => match (sort_number(self), sort_number(other)) {
+                (Some(a), Some(b)) => cmp_num(a, b),
+                _ => rank(self).cmp(&rank(other)),
+            },
+        }
+    }
+
     pub(super) fn cmp_val(&self, other: &Value) -> Option<std::cmp::Ordering> {
         match (self, other) {
             (Value::Int(a), Value::Str(s)) => canonical_int(s).map(|b| a.cmp(&b)),
@@ -76,6 +99,54 @@ impl Value {
     }
 }
 
+/// A numeric `ORDER BY` key.
+#[derive(Clone, Copy)]
+enum SortNum {
+    Int(i64),
+    Float(f64),
+}
+
+fn sort_number(v: &Value) -> Option<SortNum> {
+    match v {
+        Value::Int(i) => Some(SortNum::Int(*i)),
+        Value::Float(f) => Some(SortNum::Float(*f)),
+        Value::Str(s) => canonical_int(s).map(SortNum::Int),
+        _ => None,
+    }
+}
+
+/// Exact numeric comparison, with `NaN` above everything (including infinity).
+fn cmp_num(a: SortNum, b: SortNum) -> std::cmp::Ordering {
+    match (a, b) {
+        (SortNum::Int(a), SortNum::Int(b)) => a.cmp(&b),
+        (SortNum::Int(a), SortNum::Float(b)) => cmp_i64_f64(a, b),
+        (SortNum::Float(a), SortNum::Int(b)) => cmp_i64_f64(b, a).reverse(),
+        (SortNum::Float(a), SortNum::Float(b)) => match (a.is_nan(), b.is_nan()) {
+            (true, true) => std::cmp::Ordering::Equal,
+            (true, false) => std::cmp::Ordering::Greater,
+            (false, true) => std::cmp::Ordering::Less,
+            (false, false) => a.partial_cmp(&b).unwrap_or(std::cmp::Ordering::Equal),
+        },
+    }
+}
+
+fn cmp_i64_f64(i: i64, f: f64) -> std::cmp::Ordering {
+    use std::cmp::Ordering::*;
+    const TWO_POW_63: f64 = 9_223_372_036_854_775_808.0;
+    if f.is_nan() || f >= TWO_POW_63 {
+        return Less;
+    }
+    if f < -TWO_POW_63 {
+        return Greater;
+    }
+    // In range, so the truncation is exact.
+    let t = f.trunc();
+    match i.cmp(&(t as i64)) {
+        Equal => 0.0f64.partial_cmp(&(f - t)).unwrap_or(Equal),
+        o => o,
+    }
+}
+
 /// The integer `s` spells in canonical form (no sign prefix, padding or spaces).
 fn canonical_int(s: &str) -> Option<i64> {
     s.parse::<i64>().ok().filter(|n| n.to_string() == s)
@@ -138,3 +209,98 @@ impl Row {
             .and_then(|i| self.values.get(i))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::Value;
+    use std::cmp::Ordering::*;
+
+    fn s(x: &str) -> Value {
+        Value::Str(x.to_string())
+    }
+
+    #[test]
+    fn sort_cmp_orders_integers_and_canonical_strings_numerically() {
+        assert_eq!(Value::Int(10).sort_cmp(&s("2")), Greater);
+        assert_eq!(s("2").sort_cmp(&Value::Int(10)), Less);
+        assert_eq!(Value::Int(2).sort_cmp(&s("2")), Equal);
+        assert_eq!(Value::Float(2.5).sort_cmp(&s("10")), Less);
+        // Non-canonical and non-numeric strings stay above all numbers.
+        assert_eq!(Value::Int(10).sort_cmp(&s("02")), Less);
+        assert_eq!(s("02").sort_cmp(&s("abc")), Less);
+    }
+
+    #[test]
+    fn sort_cmp_is_exact_beyond_f64_precision() {
+        let a = Value::Int(9_007_199_254_740_992);
+        let b = Value::Int(9_007_199_254_740_993);
+        let f = Value::Float(9_007_199_254_740_992.0);
+        assert_eq!(a.sort_cmp(&f), Equal);
+        assert_eq!(b.sort_cmp(&f), Greater);
+        assert_eq!(f.sort_cmp(&b), Less);
+        assert_eq!(a.sort_cmp(&b), Less);
+        assert_eq!(
+            Value::Int(i64::MAX).sort_cmp(&Value::Float(2f64.powi(63))),
+            Less
+        );
+        assert_eq!(
+            Value::Int(i64::MIN).sort_cmp(&Value::Float(-(2f64.powi(63)))),
+            Equal
+        );
+        assert_eq!(Value::Int(0).sort_cmp(&Value::Float(0.5)), Less);
+        assert_eq!(Value::Int(-1).sort_cmp(&Value::Float(-0.5)), Less);
+    }
+
+    #[test]
+    fn sort_cmp_handles_nan_and_infinities() {
+        let nan = Value::Float(f64::NAN);
+        let inf = Value::Float(f64::INFINITY);
+        assert_eq!(nan.sort_cmp(&inf), Greater);
+        assert_eq!(nan.sort_cmp(&Value::Int(i64::MAX)), Greater);
+        assert_eq!(Value::Int(i64::MAX).sort_cmp(&inf), Less);
+        assert_eq!(
+            Value::Int(i64::MIN).sort_cmp(&Value::Float(f64::NEG_INFINITY)),
+            Greater
+        );
+        assert_eq!(nan.sort_cmp(&Value::Float(f64::NAN)), Equal);
+        assert_eq!(nan.sort_cmp(&s("x")), Less);
+    }
+
+    #[test]
+    fn sort_cmp_is_transitive_over_mixed_values() {
+        let vals = vec![
+            Value::Null,
+            Value::Bool(false),
+            Value::Bool(true),
+            Value::Int(-3),
+            Value::Int(2),
+            Value::Int(10),
+            Value::Int(9_007_199_254_740_992),
+            Value::Int(9_007_199_254_740_993),
+            Value::Int(i64::MAX),
+            Value::Int(i64::MIN),
+            Value::Float(-0.0),
+            Value::Float(2.0),
+            Value::Float(2.5),
+            Value::Float(9_007_199_254_740_992.0),
+            Value::Float(f64::INFINITY),
+            Value::Float(f64::NEG_INFINITY),
+            Value::Float(f64::NAN),
+            s("2"),
+            s("10"),
+            s("02"),
+            s("abc"),
+            s(""),
+        ];
+        for a in &vals {
+            for b in &vals {
+                assert_eq!(a.sort_cmp(b), b.sort_cmp(a).reverse(), "{a:?} {b:?}");
+                for c in &vals {
+                    if a.sort_cmp(b).is_le() && b.sort_cmp(c).is_le() {
+                        assert!(a.sort_cmp(c).is_le(), "{a:?} {b:?} {c:?}");
+                    }
+                }
+            }
+        }
+    }
+}

src/store.rs

@@ -270,6 +270,14 @@ impl Default for DocumentStore {
 /// Hash of file/document content used for change detection (not
 /// cryptographic: this only needs to detect "did this file change since
 /// last index", not resist adversarial collisions).
+/// The directory holding `path`; `.` for a bare file name, whose `parent()`
+/// is the empty path rather than `None`.
+fn containing_dir(path: &Path) -> &Path {
+    path.parent()
+        .filter(|p| !p.as_os_str().is_empty())
+        .unwrap_or_else(|| Path::new("."))
+}
+
 fn hash_bytes(bytes: &[u8]) -> u64 {
     let mut hasher = std::collections::hash_map::DefaultHasher::new();
     bytes.hash(&mut hasher);
@@ -1159,8 +1167,7 @@ impl DocumentStore {
         std::fs::rename(&tmp_path, path)?;
         // A rename is only power-loss durable after the containing directory
         // is synced as well.
-        let parent = path.parent().unwrap_or_else(|| Path::new("."));
-        std::fs::File::open(parent)?.sync_all()?;
+        std::fs::File::open(containing_dir(path))?.sync_all()?;
         drop(save_lock);
         Ok(temp_storage)
     }
@@ -1876,3 +1883,22 @@ mod tokenizer_tests {
         assert_eq!(loaded.tokenizer(), TokenizerKind::Word);
     }
 }
+
+#[cfg(test)]
+mod containing_dir_tests {
+    use super::containing_dir;
+    use rstest::rstest;
+    use std::path::Path;
+
+    #[rstest]
+    #[case("store.mq-db", ".")]
+    #[case("./store.mq-db", ".")]
+    #[case("dir/store.mq-db", "dir")]
+    #[case("/abs/store.mq-db", "/abs")]
+    fn bare_file_names_resolve_to_the_current_directory(
+        #[case] path: &str,
+        #[case] expected: &str,
+    ) {
+        assert_eq!(containing_dir(Path::new(path)), Path::new(expected));
+    }
+}