arena ranks and prefix

README.md

@@ -162,6 +162,18 @@ for (hash, path) in db.iter() {
     println!("{hash:016x} {path}");
 }
 
+// The same walk without reading a key.
+for path in db.values() {
+    println!("{path}");
+}
+
+// The arena is sorted by path, so a directory is one contiguous run and finding it
+// is a binary search - single-digit milliseconds on a 2.3M-entry table. Lazy, so
+// `take` stops the walk rather than filtering a list that was already built.
+for (hash, path) in db.prefix("assets/characters/ahri/").take(50) {
+    println!("{hash:016x} {path}");
+}
+
 // Opt-in resident mode: the whole table as an owned map. Costs the full decompressed
 // size in private memory and forfeits the shared page cache, so reach for it last.
 let map = db.load_all();
@@ -225,11 +237,13 @@ frame cache. `Send + Sync`.
 | `get_into` | copy into a reusable `String`; holds no frame afterwards |
 | `contains` | membership, never touches the arena |
 | `get_batch` · `for_each_batch` | bulk resolve, collected or streamed |
-| `iter` · `load_all` | enumerate in arena order, or decode into an owned map |
+| `iter` · `values` | enumerate in arena order, with keys or without |
+| `prefix` | every entry under a path prefix, by binary search |
+| `load_all` | decode the whole table into an owned map |
 | `hash_path` | hash a string with this table's algorithm and casing |
 | `verify_index` · `verify` | checksum + key order; the same plus a full arena walk |
 | `is_healthy` | sticky flag set by a failed read |
-| `len` · `key_width` · `hash_kind` · `casing` · `is_compressed` | shape |
+| `len` · `key_width` · `hash_kind` · `casing` · `is_compressed` · `arena_order_size` | shape |
 | `downgrade` | a `WeakHashDb` for registries that must not pin the table |
 
 **`LayeredHashDb`** - an overlay over N ordered bases.
@@ -240,7 +254,7 @@ frame cache. `Send + Sync`.
 | `insert` · `insert_path` · `extend` | write to the overlay, shadowing every base |
 | `get` · `contains` · `get_into` | overlay first, then each base in order |
 | `get_batch` · `for_each_batch` | staged bulk resolve; each base sees only the residual |
-| `iter` | every entry, each shadowed key yielded once by the layer that answers it |
+| `iter` · `prefix` | every entry, or those under a prefix; each shadowed key yielded once by the layer that answers it |
 | `bases` · `overlay_len` · `base_len` · `is_healthy` | shape |
 
 **`HashStore`** - the shared cache directory.
@@ -277,6 +291,7 @@ frame cache. `Send + Sync`.
 mimir <COMMAND>
 
 build    Build a .hashdb table from a txt hash list (lines of `<hex-hash> <path>`)
+ls       List the paths under a prefix, from a file or the shared cache
 get      Resolve one hash from a .hashdb file or the shared cache
 check    Say what an update would do, without downloading or locking anything
 update   Download the latest published tables into the shared cache
@@ -290,6 +305,11 @@ stats    Sizes, entry counts, compression ratio of a .hashdb file
 ```sh
 # Build a table from a CDragon txt list
 mimir build --input hashes.game.txt --table game --out game.hashdb
+mimir build --input hashes.game.txt --table game --out game.hashdb --arena-order
+
+# List the paths under a prefix (empty lists the table, in path order)
+mimir ls assets/characters/ahri/ --file game.lhdb
+mimir ls data/ --table game --limit 20
 
 # Resolve a hash, from a file or from the shared cache
 mimir get 0x1234abcd --file game.hashdb

crates/ltk_hashdb/examples/arena_order.rs

@@ -0,0 +1,128 @@
+//! What the arena-order section costs and what it buys, measured on a real table.
+//!
+//! ```sh
+//! cargo run --release --example arena_order -- game.lhdb            # as published
+//! cargo run --release --example arena_order -- game.lhdb game+ao.lhdb   # and rebuilt with the section
+//! ```
+//!
+//! With an output path it rebuilds the table with [`ArenaOrder::Stored`] and
+//! reports both, which is where `docs/BENCHMARKS.md` gets its figures. Timings
+//! are cold-per-process: each phase runs against a freshly opened table, so a
+//! rebuilt permutation is paid for exactly once, as a consumer pays for it.
+
+use std::io::BufWriter;
+use std::path::Path;
+use std::time::Instant;
+
+use ltk_hashdb::{ArenaOrder, Compression, HashDb, HashDbWriter};
+
+/// Prefixes to search for: one large directory, one small, one that matches nothing.
+const PROBES: [&str; 3] = [
+    "assets/characters/ahri/",
+    "data/menu/",
+    "zzz-nothing-starts-with-this/",
+];
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let mut args = std::env::args().skip(1);
+    let input = args
+        .next()
+        .ok_or("usage: arena_order <table.lhdb> [out.lhdb]")?;
+
+    report(Path::new(&input))?;
+
+    if let Some(output) = args.next() {
+        rebuild(Path::new(&input), Path::new(&output))?;
+        report(Path::new(&output))?;
+    }
+
+    Ok(())
+}
+
+/// Time the three arena-order reads, each against its own freshly opened table.
+fn report(path: &Path) -> Result<(), Box<dyn std::error::Error>> {
+    let db = HashDb::open(path)?;
+    let file_len = std::fs::metadata(path)?.len();
+    let section = db.arena_order_size();
+
+    println!("\n{}", path.display());
+    println!("  entries      {}", db.len());
+    println!("  file         {file_len} B");
+    match section {
+        Some(bytes) => println!(
+            "  arena order  {bytes} B stored ({:.1}% of the file)",
+            100.0 * bytes as f64 / file_len as f64
+        ),
+        None => println!("  arena order  not stored - the reader sorts for it"),
+    }
+
+    // The first arena-order read is what pays for a rebuild, so it gets a table
+    // of its own; nothing else in the process has warmed the permutation.
+    let db = HashDb::open(path)?;
+    let start = Instant::now();
+    let first = db.prefix(PROBES[0]).count();
+    println!(
+        "  prefix       {:>8.1} ms  first call, {first} hit(s) for {:?}",
+        start.elapsed().as_secs_f64() * 1e3,
+        PROBES[0]
+    );
+
+    for probe in PROBES {
+        let start = Instant::now();
+        let hits = db.prefix(probe).count();
+        println!(
+            "               {:>8.1} ms  warm, {hits} hit(s) for {probe:?}",
+            start.elapsed().as_secs_f64() * 1e3
+        );
+    }
+
+    let db = HashDb::open(path)?;
+    let start = Instant::now();
+    let count = db.values().count();
+    println!(
+        "  values       {:>8.1} ms  {count} paths",
+        start.elapsed().as_secs_f64() * 1e3
+    );
+
+    Ok(())
+}
+
+/// Rewrite `input` with the arena-order section, entry for entry.
+fn rebuild(input: &Path, output: &Path) -> Result<(), Box<dyn std::error::Error>> {
+    let db = HashDb::open(input)?;
+    let mut writer = HashDbWriter::with_key_config(
+        db.key_config(),
+        Compression::Zeekstd {
+            frame_size: 16 << 10,
+            level: 19,
+        },
+    )
+    .arena_order(ArenaOrder::Stored);
+
+    for (hash, path) in db.iter() {
+        writer.insert(hash, &path);
+    }
+
+    let start = Instant::now();
+    let stats = writer.build(BufWriter::new(std::fs::File::create(output)?))?;
+    println!(
+        "\nrebuilt {} in {:.1} s: {} B, of which {} B is the arena order",
+        output.display(),
+        start.elapsed().as_secs_f64(),
+        stats.file_len,
+        stats.arena_order_size,
+    );
+
+    // The two tables must agree entry for entry - a stored permutation and a
+    // sorted one are the same permutation or one of them is wrong.
+    let rebuilt = HashDb::open(output)?;
+    let mine: Vec<_> = rebuilt.values().map(|p| p.into_owned()).collect();
+    let theirs: Vec<_> = db.values().map(|p| p.into_owned()).collect();
+    assert_eq!(mine, theirs, "stored and sorted arena order disagree");
+    println!(
+        "stored and sorted arena order agree over all {} paths",
+        mine.len()
+    );
+
+    Ok(())
+}

crates/ltk_hashdb/examples/path_coding.rs

@@ -0,0 +1,184 @@
+//! Does factoring paths beat leaving zstd to it? Measured on a real table.
+//!
+//! ```sh
+//! cargo run --release --example path_coding -- game.lhdb
+//! ```
+//!
+//! Four encodings of the same path list, each compressed with the arena's own
+//! encoder, against the file they came from:
+//!
+//! - **raw** - what the arena stores today: sorted paths, concatenated.
+//! - **front-coded** - each path as `varint(shared prefix with the previous)` and
+//!   the suffix. Restart-free, so this is a *lower bound*: a real one needs a
+//!   restart per frame and a replay on every read.
+//! - **matched frames** - the same two at equal paths per frame. Front coding
+//!   packs several times more paths into a 16 KiB frame, so some of its lead is
+//!   a coarser lookup granularity rather than the coding; this separates them.
+//! - **interned** - directories stored once, each entry a `(dir_id, filename)`.
+//!   The dir_id array is index, not arena: fixed width, random access, and so
+//!   not compressed at all.
+
+use std::collections::HashMap;
+use std::io::Write;
+use std::path::Path;
+
+use ltk_hashdb::HashDb;
+
+/// The published arena settings, so every number here is comparable to the file.
+const FRAME_SIZE: u32 = 16 << 10;
+const LEVEL: i32 = 19;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let path = std::env::args()
+        .nth(1)
+        .ok_or("usage: path_coding <table.lhdb>")?;
+    let path = Path::new(&path);
+
+    let db = HashDb::open(path)?;
+    let file_len = std::fs::metadata(path)?.len();
+    let entries = db.len();
+
+    // The arena stores one copy of each distinct path, in path order - which is
+    // exactly `values` with adjacent duplicates dropped.
+    let mut paths: Vec<String> = Vec::with_capacity(entries);
+    for path in db.values() {
+        if paths.last().map(String::as_str) != Some(&*path) {
+            paths.push(path.into_owned());
+        }
+    }
+
+    println!();
+    println!("{}", path.display());
+    println!("  file            {file_len:>12} B");
+    println!("  entries         {entries:>12}");
+    println!("  distinct paths  {:>12}", paths.len());
+    println!(
+        "  arena on disk   {:>12} B  ({:.1}% of the file)",
+        db.arena_compressed_size(),
+        100.0 * db.arena_compressed_size() as f64 / file_len as f64
+    );
+    println!(
+        "  index on disk   {:>12} B  ({:.1}% of the file, and random access, so not compressible)",
+        file_len - db.arena_compressed_size(),
+        100.0 * (file_len - db.arena_compressed_size()) as f64 / file_len as f64
+    );
+
+    // --- raw: what we ship today ---------------------------------------------
+    let raw: Vec<u8> = paths.iter().flat_map(|p| p.as_bytes().to_vec()).collect();
+
+    // --- front coding ---------------------------------------------------------
+    let mut coded = Vec::with_capacity(raw.len() / 2);
+    let mut prev: &str = "";
+    for path in &paths {
+        let shared = path
+            .as_bytes()
+            .iter()
+            .zip(prev.as_bytes())
+            .take_while(|(a, b)| a == b)
+            .count();
+        push_varint(&mut coded, shared as u64);
+        coded.extend_from_slice(&path.as_bytes()[shared..]);
+        prev = path;
+    }
+
+    // Paths per frame at the published frame size, and the frame sizes that give
+    // each coding the other's granularity.
+    let raw_per_frame = FRAME_SIZE as usize * paths.len() / raw.len();
+    let coded_per_frame = FRAME_SIZE as usize * paths.len() / coded.len();
+    let narrow = (coded.len() * raw_per_frame / paths.len()) as u32;
+    let wide = (raw.len() * coded_per_frame / paths.len()) as u32;
+
+    println!();
+    println!("  encoding                        on disk  frame     paths/frame");
+    row("raw", &raw, FRAME_SIZE, raw_per_frame, file_len)?;
+    row("front-coded", &coded, FRAME_SIZE, coded_per_frame, file_len)?;
+    row(
+        "front-coded, raw's frames",
+        &coded,
+        narrow,
+        raw_per_frame,
+        file_len,
+    )?;
+    row(
+        "raw, front coding's frames",
+        &raw,
+        wide,
+        coded_per_frame,
+        file_len,
+    )?;
+
+    // --- interning ------------------------------------------------------------
+    let mut dirs: HashMap<&str, usize> = HashMap::new();
+    let mut dir_bytes = Vec::new();
+    let mut names = Vec::new();
+    for path in &paths {
+        let (dir, name) = match path.rfind('/') {
+            Some(cut) => path.split_at(cut + 1),
+            None => ("", path.as_str()),
+        };
+        let next = dirs.len();
+        if dirs.insert(dir, next).is_none() {
+            dir_bytes.extend_from_slice(dir.as_bytes());
+        }
+        names.extend_from_slice(name.as_bytes());
+    }
+
+    let id_width =
+        ((usize::BITS - dirs.len().saturating_sub(1).leading_zeros()).div_ceil(8)).max(1) as usize;
+    let ids = entries * id_width;
+    let arena = compressed(&dir_bytes, FRAME_SIZE)? + compressed(&names, FRAME_SIZE)?;
+
+    println!();
+    println!(
+        "  interned: {} directories stored once, {} B of file names",
+        dirs.len(),
+        names.len()
+    );
+    println!("    arena  {arena:>12} B  compressed");
+    println!("    ids    {ids:>12} B  {id_width} bytes per entry, in the index, not compressed");
+    println!(
+        "    total  {:>12} B  ({:+.1}% against the {} B arena it replaces)",
+        arena + ids,
+        100.0 * (arena + ids) as f64 / db.arena_compressed_size() as f64 - 100.0,
+        db.arena_compressed_size()
+    );
+
+    Ok(())
+}
+
+fn row(
+    name: &str,
+    bytes: &[u8],
+    frame_size: u32,
+    per_frame: usize,
+    file_len: u64,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let packed = compressed(bytes, frame_size)?;
+    println!(
+        "  {name:<28} {packed:>10} B  {frame_size:>6}  {per_frame:>6}   ({:.1}% of the file)",
+        100.0 * packed as f64 / file_len as f64
+    );
+
+    Ok(())
+}
+
+/// Compress with the arena's own encoder, so the numbers are comparable.
+fn compressed(bytes: &[u8], frame_size: u32) -> Result<usize, Box<dyn std::error::Error>> {
+    let mut out = Vec::new();
+    let mut encoder = zeekstd::EncodeOptions::new()
+        .compression_level(LEVEL)
+        .frame_size_policy(zeekstd::FrameSizePolicy::Uncompressed(frame_size))
+        .into_encoder(&mut out)?;
+    encoder.write_all(bytes)?;
+    encoder.finish()?;
+
+    Ok(out.len())
+}
+
+fn push_varint(out: &mut Vec<u8>, mut value: u64) {
+    while value >= 0x80 {
+        out.push((value as u8) | 0x80);
+        value >>= 7;
+    }
+    out.push(value as u8);
+}