diff --git a/README.md b/README.md index a088a00..e00d08b 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,18 @@ for (hash, path) in db.iter() { println!("{hash:016x} {path}"); } +// The same walk without reading a key. +for path in db.values() { + println!("{path}"); +} + +// The arena is sorted by path, so a directory is one contiguous run and finding it +// is a binary search - single-digit milliseconds on a 2.3M-entry table. Lazy, so +// `take` stops the walk rather than filtering a list that was already built. +for (hash, path) in db.prefix("assets/characters/ahri/").take(50) { + println!("{hash:016x} {path}"); +} + // Opt-in resident mode: the whole table as an owned map. Costs the full decompressed // size in private memory and forfeits the shared page cache, so reach for it last. let map = db.load_all(); @@ -225,11 +237,13 @@ frame cache. `Send + Sync`. | `get_into` | copy into a reusable `String`; holds no frame afterwards | | `contains` | membership, never touches the arena | | `get_batch` · `for_each_batch` | bulk resolve, collected or streamed | -| `iter` · `load_all` | enumerate in arena order, or decode into an owned map | +| `iter` · `values` | enumerate in arena order, with keys or without | +| `prefix` | every entry under a path prefix, by binary search | +| `load_all` | decode the whole table into an owned map | | `hash_path` | hash a string with this table's algorithm and casing | | `verify_index` · `verify` | checksum + key order; the same plus a full arena walk | | `is_healthy` | sticky flag set by a failed read | -| `len` · `key_width` · `hash_kind` · `casing` · `is_compressed` | shape | +| `len` · `key_width` · `hash_kind` · `casing` · `is_compressed` · `arena_order_size` | shape | | `downgrade` | a `WeakHashDb` for registries that must not pin the table | **`LayeredHashDb`** - an overlay over N ordered bases. @@ -240,7 +254,7 @@ frame cache. `Send + Sync`. | `insert` · `insert_path` · `extend` | write to the overlay, shadowing every base | | `get` · `contains` · `get_into` | overlay first, then each base in order | | `get_batch` · `for_each_batch` | staged bulk resolve; each base sees only the residual | -| `iter` | every entry, each shadowed key yielded once by the layer that answers it | +| `iter` · `prefix` | every entry, or those under a prefix; each shadowed key yielded once by the layer that answers it | | `bases` · `overlay_len` · `base_len` · `is_healthy` | shape | **`HashStore`** - the shared cache directory. @@ -277,6 +291,7 @@ frame cache. `Send + Sync`. mimir build Build a .hashdb table from a txt hash list (lines of ` `) +ls List the paths under a prefix, from a file or the shared cache get Resolve one hash from a .hashdb file or the shared cache check Say what an update would do, without downloading or locking anything update Download the latest published tables into the shared cache @@ -290,6 +305,11 @@ stats Sizes, entry counts, compression ratio of a .hashdb file ```sh # Build a table from a CDragon txt list mimir build --input hashes.game.txt --table game --out game.hashdb +mimir build --input hashes.game.txt --table game --out game.hashdb --arena-order + +# List the paths under a prefix (empty lists the table, in path order) +mimir ls assets/characters/ahri/ --file game.lhdb +mimir ls data/ --table game --limit 20 # Resolve a hash, from a file or from the shared cache mimir get 0x1234abcd --file game.hashdb diff --git a/crates/ltk_hashdb/examples/arena_order.rs b/crates/ltk_hashdb/examples/arena_order.rs new file mode 100644 index 0000000..bdf88a6 --- /dev/null +++ b/crates/ltk_hashdb/examples/arena_order.rs @@ -0,0 +1,128 @@ +//! What the arena-order section costs and what it buys, measured on a real table. +//! +//! ```sh +//! cargo run --release --example arena_order -- game.lhdb # as published +//! cargo run --release --example arena_order -- game.lhdb game+ao.lhdb # and rebuilt with the section +//! ``` +//! +//! With an output path it rebuilds the table with [`ArenaOrder::Stored`] and +//! reports both, which is where `docs/BENCHMARKS.md` gets its figures. Timings +//! are cold-per-process: each phase runs against a freshly opened table, so a +//! rebuilt permutation is paid for exactly once, as a consumer pays for it. + +use std::io::BufWriter; +use std::path::Path; +use std::time::Instant; + +use ltk_hashdb::{ArenaOrder, Compression, HashDb, HashDbWriter}; + +/// Prefixes to search for: one large directory, one small, one that matches nothing. +const PROBES: [&str; 3] = [ + "assets/characters/ahri/", + "data/menu/", + "zzz-nothing-starts-with-this/", +]; + +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let input = args + .next() + .ok_or("usage: arena_order [out.lhdb]")?; + + report(Path::new(&input))?; + + if let Some(output) = args.next() { + rebuild(Path::new(&input), Path::new(&output))?; + report(Path::new(&output))?; + } + + Ok(()) +} + +/// Time the three arena-order reads, each against its own freshly opened table. +fn report(path: &Path) -> Result<(), Box> { + let db = HashDb::open(path)?; + let file_len = std::fs::metadata(path)?.len(); + let section = db.arena_order_size(); + + println!("\n{}", path.display()); + println!(" entries {}", db.len()); + println!(" file {file_len} B"); + match section { + Some(bytes) => println!( + " arena order {bytes} B stored ({:.1}% of the file)", + 100.0 * bytes as f64 / file_len as f64 + ), + None => println!(" arena order not stored - the reader sorts for it"), + } + + // The first arena-order read is what pays for a rebuild, so it gets a table + // of its own; nothing else in the process has warmed the permutation. + let db = HashDb::open(path)?; + let start = Instant::now(); + let first = db.prefix(PROBES[0]).count(); + println!( + " prefix {:>8.1} ms first call, {first} hit(s) for {:?}", + start.elapsed().as_secs_f64() * 1e3, + PROBES[0] + ); + + for probe in PROBES { + let start = Instant::now(); + let hits = db.prefix(probe).count(); + println!( + " {:>8.1} ms warm, {hits} hit(s) for {probe:?}", + start.elapsed().as_secs_f64() * 1e3 + ); + } + + let db = HashDb::open(path)?; + let start = Instant::now(); + let count = db.values().count(); + println!( + " values {:>8.1} ms {count} paths", + start.elapsed().as_secs_f64() * 1e3 + ); + + Ok(()) +} + +/// Rewrite `input` with the arena-order section, entry for entry. +fn rebuild(input: &Path, output: &Path) -> Result<(), Box> { + let db = HashDb::open(input)?; + let mut writer = HashDbWriter::with_key_config( + db.key_config(), + Compression::Zeekstd { + frame_size: 16 << 10, + level: 19, + }, + ) + .arena_order(ArenaOrder::Stored); + + for (hash, path) in db.iter() { + writer.insert(hash, &path); + } + + let start = Instant::now(); + let stats = writer.build(BufWriter::new(std::fs::File::create(output)?))?; + println!( + "\nrebuilt {} in {:.1} s: {} B, of which {} B is the arena order", + output.display(), + start.elapsed().as_secs_f64(), + stats.file_len, + stats.arena_order_size, + ); + + // The two tables must agree entry for entry - a stored permutation and a + // sorted one are the same permutation or one of them is wrong. + let rebuilt = HashDb::open(output)?; + let mine: Vec<_> = rebuilt.values().map(|p| p.into_owned()).collect(); + let theirs: Vec<_> = db.values().map(|p| p.into_owned()).collect(); + assert_eq!(mine, theirs, "stored and sorted arena order disagree"); + println!( + "stored and sorted arena order agree over all {} paths", + mine.len() + ); + + Ok(()) +} diff --git a/crates/ltk_hashdb/examples/path_coding.rs b/crates/ltk_hashdb/examples/path_coding.rs new file mode 100644 index 0000000..b357a4e --- /dev/null +++ b/crates/ltk_hashdb/examples/path_coding.rs @@ -0,0 +1,184 @@ +//! Does factoring paths beat leaving zstd to it? Measured on a real table. +//! +//! ```sh +//! cargo run --release --example path_coding -- game.lhdb +//! ``` +//! +//! Four encodings of the same path list, each compressed with the arena's own +//! encoder, against the file they came from: +//! +//! - **raw** - what the arena stores today: sorted paths, concatenated. +//! - **front-coded** - each path as `varint(shared prefix with the previous)` and +//! the suffix. Restart-free, so this is a *lower bound*: a real one needs a +//! restart per frame and a replay on every read. +//! - **matched frames** - the same two at equal paths per frame. Front coding +//! packs several times more paths into a 16 KiB frame, so some of its lead is +//! a coarser lookup granularity rather than the coding; this separates them. +//! - **interned** - directories stored once, each entry a `(dir_id, filename)`. +//! The dir_id array is index, not arena: fixed width, random access, and so +//! not compressed at all. + +use std::collections::HashMap; +use std::io::Write; +use std::path::Path; + +use ltk_hashdb::HashDb; + +/// The published arena settings, so every number here is comparable to the file. +const FRAME_SIZE: u32 = 16 << 10; +const LEVEL: i32 = 19; + +fn main() -> Result<(), Box> { + let path = std::env::args() + .nth(1) + .ok_or("usage: path_coding ")?; + let path = Path::new(&path); + + let db = HashDb::open(path)?; + let file_len = std::fs::metadata(path)?.len(); + let entries = db.len(); + + // The arena stores one copy of each distinct path, in path order - which is + // exactly `values` with adjacent duplicates dropped. + let mut paths: Vec = Vec::with_capacity(entries); + for path in db.values() { + if paths.last().map(String::as_str) != Some(&*path) { + paths.push(path.into_owned()); + } + } + + println!(); + println!("{}", path.display()); + println!(" file {file_len:>12} B"); + println!(" entries {entries:>12}"); + println!(" distinct paths {:>12}", paths.len()); + println!( + " arena on disk {:>12} B ({:.1}% of the file)", + db.arena_compressed_size(), + 100.0 * db.arena_compressed_size() as f64 / file_len as f64 + ); + println!( + " index on disk {:>12} B ({:.1}% of the file, and random access, so not compressible)", + file_len - db.arena_compressed_size(), + 100.0 * (file_len - db.arena_compressed_size()) as f64 / file_len as f64 + ); + + // --- raw: what we ship today --------------------------------------------- + let raw: Vec = paths.iter().flat_map(|p| p.as_bytes().to_vec()).collect(); + + // --- front coding --------------------------------------------------------- + let mut coded = Vec::with_capacity(raw.len() / 2); + let mut prev: &str = ""; + for path in &paths { + let shared = path + .as_bytes() + .iter() + .zip(prev.as_bytes()) + .take_while(|(a, b)| a == b) + .count(); + push_varint(&mut coded, shared as u64); + coded.extend_from_slice(&path.as_bytes()[shared..]); + prev = path; + } + + // Paths per frame at the published frame size, and the frame sizes that give + // each coding the other's granularity. + let raw_per_frame = FRAME_SIZE as usize * paths.len() / raw.len(); + let coded_per_frame = FRAME_SIZE as usize * paths.len() / coded.len(); + let narrow = (coded.len() * raw_per_frame / paths.len()) as u32; + let wide = (raw.len() * coded_per_frame / paths.len()) as u32; + + println!(); + println!(" encoding on disk frame paths/frame"); + row("raw", &raw, FRAME_SIZE, raw_per_frame, file_len)?; + row("front-coded", &coded, FRAME_SIZE, coded_per_frame, file_len)?; + row( + "front-coded, raw's frames", + &coded, + narrow, + raw_per_frame, + file_len, + )?; + row( + "raw, front coding's frames", + &raw, + wide, + coded_per_frame, + file_len, + )?; + + // --- interning ------------------------------------------------------------ + let mut dirs: HashMap<&str, usize> = HashMap::new(); + let mut dir_bytes = Vec::new(); + let mut names = Vec::new(); + for path in &paths { + let (dir, name) = match path.rfind('/') { + Some(cut) => path.split_at(cut + 1), + None => ("", path.as_str()), + }; + let next = dirs.len(); + if dirs.insert(dir, next).is_none() { + dir_bytes.extend_from_slice(dir.as_bytes()); + } + names.extend_from_slice(name.as_bytes()); + } + + let id_width = + ((usize::BITS - dirs.len().saturating_sub(1).leading_zeros()).div_ceil(8)).max(1) as usize; + let ids = entries * id_width; + let arena = compressed(&dir_bytes, FRAME_SIZE)? + compressed(&names, FRAME_SIZE)?; + + println!(); + println!( + " interned: {} directories stored once, {} B of file names", + dirs.len(), + names.len() + ); + println!(" arena {arena:>12} B compressed"); + println!(" ids {ids:>12} B {id_width} bytes per entry, in the index, not compressed"); + println!( + " total {:>12} B ({:+.1}% against the {} B arena it replaces)", + arena + ids, + 100.0 * (arena + ids) as f64 / db.arena_compressed_size() as f64 - 100.0, + db.arena_compressed_size() + ); + + Ok(()) +} + +fn row( + name: &str, + bytes: &[u8], + frame_size: u32, + per_frame: usize, + file_len: u64, +) -> Result<(), Box> { + let packed = compressed(bytes, frame_size)?; + println!( + " {name:<28} {packed:>10} B {frame_size:>6} {per_frame:>6} ({:.1}% of the file)", + 100.0 * packed as f64 / file_len as f64 + ); + + Ok(()) +} + +/// Compress with the arena's own encoder, so the numbers are comparable. +fn compressed(bytes: &[u8], frame_size: u32) -> Result> { + let mut out = Vec::new(); + let mut encoder = zeekstd::EncodeOptions::new() + .compression_level(LEVEL) + .frame_size_policy(zeekstd::FrameSizePolicy::Uncompressed(frame_size)) + .into_encoder(&mut out)?; + encoder.write_all(bytes)?; + encoder.finish()?; + + Ok(out.len()) +} + +fn push_varint(out: &mut Vec, mut value: u64) { + while value >= 0x80 { + out.push((value as u8) | 0x80); + value >>= 7; + } + out.push(value as u8); +} diff --git a/crates/ltk_hashdb/src/header.rs b/crates/ltk_hashdb/src/header.rs index 2ba72fc..77d28a0 100644 --- a/crates/ltk_hashdb/src/header.rs +++ b/crates/ltk_hashdb/src/header.rs @@ -10,7 +10,7 @@ //! 12 key_width u8 4 = u32 table, 8 = u64 table //! 13 offset_width u8 4 or 8; width of arena offsets //! 14 opt_flags u8 optional; unknown bits ignored, none defined yet -//! 15 reserved u8 written as zero, ignored on read +//! 15 arena_order_width u8 1..=8 when an arena-order section is present, else 0 //! 16..24 entry_count u64 //! 24..32 keys_offset u64 file offset, 8-aligned //! 32..40 offsets_offset u64 file offset, offset_width-aligned @@ -18,12 +18,20 @@ //! 48..56 arena_decompressed_size u64 //! 56..64 arena_compressed_size u64 == decompressed if raw //! 64..72 checksum u64 xxh3-64 of keys‖offsets‖lengths‖arena (as stored) -//! 72..80 reserved [u8;8] written as zero, ignored on read +//! 72..80 arena_order_offset u64 file offset of the arena-order section; 0 = absent //! ``` //! //! The lengths section (`entry_count` × u16) has no header field: it sits //! immediately after the offsets, at `offsets_offset + entry_count × offset_width`. //! +//! Bytes 15 and 72..80 were reserved-and-zero through every build that shipped +//! before the arena-order section existed, which is what lets that section be +//! added without a version bump: an older reader sees a file whose reserved +//! fields it ignores, and reads it exactly as it always did. A capability that +//! needs no field of its own announces itself in `opt_flags` instead; this one +//! needs an offset, and a section offset of zero is already unambiguous, so it +//! is its own announcement rather than a flag that could disagree with it. +//! //! The two flag bytes differ in what an *unknown* bit means. A bit in `flags` //! changes how the file has to be read, so a build that does not know it must //! refuse the file; a bit in `opt_flags` only announces something a build may @@ -51,6 +59,45 @@ pub(crate) const FLAG_CASE_INSENSITIVE: u8 = 1 << 1; /// Every required flag bit this build understands; any other rejects the file. const KNOWN_FLAGS: u8 = FLAG_ARENA_COMPRESSED | FLAG_CASE_INSENSITIVE; +/// Where the optional arena-order section sits, and how wide its entries are. +/// +/// See `docs/FORMAT.md`; the section is `entry_count` packed entry indices in +/// arena order, followed by an 8-byte checksum of them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ArenaOrderRef { + pub offset: u64, + pub width: usize, +} + +impl ArenaOrderRef { + /// Bytes the whole section occupies, checksum included, or `None` on overflow. + pub fn len(&self, entry_count: u64) -> Option { + entry_count + .checked_mul(self.width as u64)? + .checked_add(ARENA_ORDER_CHECKSUM_SIZE as u64) + } +} + +/// Trailing xxh3-64 over the section's packed entries. +/// +/// The header's own `checksum` deliberately does not cover this section: it is +/// defined as keys‖offsets‖lengths‖arena, and a reader built before the section +/// existed still computes it that way. Giving the section its own digest keeps +/// both readers right about the same file. +pub(crate) const ARENA_ORDER_CHECKSUM_SIZE: usize = 8; + +/// Bytes needed to hold any entry index of a table this size, 1..=8. +/// +/// The narrowest packing that still addresses every entry - 3 bytes for the ~2.3M +/// entry `game` table, where a `u32` array would spend a quarter of its bytes on +/// zeroes. +pub(crate) fn arena_order_width(entry_count: u64) -> usize { + let max = entry_count.saturating_sub(1); + let bits = u64::BITS - max.leading_zeros(); + + (bits.div_ceil(8) as usize).max(1) +} + /// Width of the arena offsets: u32 unless the raw arena exceeds 4 GiB. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum OffsetWidth { @@ -80,6 +127,9 @@ pub(crate) struct Header { pub arena_decompressed_size: u64, pub arena_compressed_size: u64, pub checksum: u64, + + /// The arena-order section, when the writer emitted one. + pub arena_order: Option, } impl Header { @@ -104,7 +154,11 @@ impl Header { buf[12] = self.key_width.bytes() as u8; buf[13] = self.offset_width.bytes() as u8; // Byte 14 (`opt_flags`) stays zero: this build announces no optional - // capability. Bytes 15 and 72..80 are reserved and likewise zero. + // capability. + if let Some(order) = self.arena_order { + buf[15] = order.width as u8; + buf[72..80].copy_from_slice(&order.offset.to_le_bytes()); + } buf[16..24].copy_from_slice(&self.entry_count.to_le_bytes()); buf[24..32].copy_from_slice(&self.keys_offset.to_le_bytes()); buf[32..40].copy_from_slice(&self.offsets_offset.to_le_bytes()); @@ -152,6 +206,24 @@ impl Header { // a second flag byte at all. let u64_at = |i: usize| u64::from_le_bytes(buf[i..i + 8].try_into().unwrap()); + + // Absent is the norm and reads as zero, so the width is only meaningful + // - and only checked - once an offset claims a section is there. + let arena_order = match u64_at(72) { + 0 => None, + offset => match buf[15] { + width @ 1..=8 => Some(ArenaOrderRef { + offset, + width: width as usize, + }), + _ => { + return Err(OpenError::MalformedHeader( + "arena_order_width must be 1..=8", + )) + } + }, + }; + Ok(Self { hash_kind, flags, @@ -164,6 +236,7 @@ impl Header { arena_decompressed_size: u64_at(48), arena_compressed_size: u64_at(56), checksum: u64_at(64), + arena_order, }) } } diff --git a/crates/ltk_hashdb/src/layered.rs b/crates/ltk_hashdb/src/layered.rs index b59b6c4..c151f71 100644 --- a/crates/ltk_hashdb/src/layered.rs +++ b/crates/ltk_hashdb/src/layered.rs @@ -243,6 +243,33 @@ impl LayeredHashDb { overlay.chain(bases) } + /// Every entry whose path starts with `prefix`, overlay first and then each + /// base in priority order. + /// + /// [`iter`](Self::iter)'s shadowing rule applied to a search: an entry a + /// higher layer answers is yielded once, by that layer. Each base runs its + /// own binary search - see [`HashDb::prefix`] - so the cost is per base, not + /// per entry. The runs are chained rather than merged, so the result is in + /// path order *within* a layer and grouped by layer across them. + pub fn prefix<'a>(&'a self, prefix: &'a str) -> impl Iterator)> + 'a { + let overlay = self + .overlay + .iter() + .filter(move |(_, path)| path.starts_with(prefix)) + .map(|(&hash, path)| (hash, PathRef::borrowed(path))); + + let bases = self + .bases + .iter() + .enumerate() + .flat_map(move |(layer, base)| { + base.prefix(prefix) + .filter(move |(hash, _)| !self.shadows(*hash, layer)) + }); + + overlay.chain(bases) + } + /// Whether a layer above `layer` already answers `hash`. fn shadows(&self, hash: u64, layer: usize) -> bool { self.overlay.contains_key(&hash) diff --git a/crates/ltk_hashdb/src/lib.rs b/crates/ltk_hashdb/src/lib.rs index 675ea3f..583b3b9 100644 --- a/crates/ltk_hashdb/src/lib.rs +++ b/crates/ltk_hashdb/src/lib.rs @@ -67,6 +67,43 @@ pub enum Compression { Zeekstd { frame_size: u32, level: i32 }, } +/// Whether a table carries the arena-order index in the file. +/// +/// The arena is laid out in path order but the offsets are stored in key order, +/// so walking the arena forward means knowing the permutation between them. It +/// is what [`HashDb::values`], [`HashDb::prefix`], [`HashDb::iter`] and +/// [`HashDb::verify`] all walk, and a reader that does not find it in the file +/// reconstructs it on first use. +/// +/// So this is a space/time trade and nothing else: every operation works either +/// way, and both orders are identical. See `docs/BENCHMARKS.md` for the measured +/// figures behind the summary below. +/// +/// [`HashDb::values`]: crate::HashDb::values +/// [`HashDb::prefix`]: crate::HashDb::prefix +/// [`HashDb::iter`]: crate::HashDb::iter +/// [`HashDb::verify`]: crate::HashDb::verify +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ArenaOrder { + /// Leave it out; the reader rebuilds it the first time it is needed. + /// + /// The default, and what every table published so far does. The rebuild is a + /// sort over the offsets - about a third of a second on the 2.3M-entry game + /// table - after which it is shared by every clone of the table for as long + /// as one is open. + #[default] + Omitted, + + /// Store it: `entry_count` × 1..8 bytes, sized to the entry count. + /// + /// Turns the rebuild into a memory map: no sort, no per-process copy, and + /// the pages are shared across every process that opens the file. The cost + /// is file size - about 16% on the game table, less on the smaller ones - + /// paid by every consumer, including the ones that only ever call + /// [`HashDb::get`](crate::HashDb::get). + Stored, +} + impl Default for Compression { /// Publishing config: 16 KiB frames (the size/latency knee) at level 19. fn default() -> Self { diff --git a/crates/ltk_hashdb/src/reader.rs b/crates/ltk_hashdb/src/reader.rs index 31bc177..ef37fbf 100644 --- a/crates/ltk_hashdb/src/reader.rs +++ b/crates/ltk_hashdb/src/reader.rs @@ -8,13 +8,13 @@ use std::fs::File; use std::ops::Range; use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Weak}; +use std::sync::{Arc, OnceLock, Weak}; use xxhash_rust::xxh3::Xxh3; use zeekstd::SeekTable; use crate::cache::{Frame, FrameCache}; -use crate::header::Header; +use crate::header::{arena_order_width, Header, ARENA_ORDER_CHECKSUM_SIZE}; use crate::{Casing, HashKind, KeyConfig, KeyWidth, OpenError, PathRef, VerifyError}; /// Decompressed frame bytes a table caches by default: 4 MiB, i.e. 256 frames at the @@ -137,6 +137,13 @@ struct Inner { /// Present iff the arena is a zeekstd seekable stream. seek_table: Option, + /// The file's arena-order section, when it carries one. + stored_order: Option, + + /// Sorted on first use when the file carries none, then shared by every + /// clone of the table for as long as one of them is open. + rebuilt_order: OnceLock>, + cache: FrameCache, /// Frames decompressed so far; misses must never bump it (see unit tests). @@ -147,6 +154,39 @@ struct Inner { healthy: AtomicBool, } +/// Where the file keeps its arena-order section, from the header. +struct StoredOrder { + /// The packed entry indices, checksum excluded. + packed: Range, + + /// Bytes per entry index, 1..=8. + width: usize, + + /// xxh3-64 over `packed`, stored immediately after it. + checksum: u64, +} + +/// `arena_rank → entry_index`: which entry's path sits where in the arena. +/// +/// The same packing either way, so one accessor serves a table that stores the +/// permutation and one that had to sort for it. +#[derive(Clone, Copy)] +struct ArenaRanks<'a> { + packed: &'a [u8], + width: usize, +} + +impl ArenaRanks<'_> { + fn len(&self) -> usize { + self.packed.len() / self.width + } + + /// The entry at `rank`. Untrusted: callers check it against `entry_count`. + fn get(&self, rank: usize) -> usize { + read_uint(self.packed, rank * self.width, self.width) as usize + } +} + enum Backing { Mmap(memmap2::Mmap), Bytes(Cow<'static, [u8]>), @@ -322,6 +362,33 @@ impl HashDb { None }; + // The arena-order section, if the header claims one. Its contents are + // untrusted like everything else - each rank is range-checked where it is + // used, and `verify_index` proves the whole thing is a permutation. + let stored_order = match header.arena_order { + None => None, + Some(order) => { + if order.width < arena_order_width(header.entry_count) { + return Err(OpenError::MalformedHeader( + "arena_order_width too narrow for entry_count", + )); + } + + let len = order + .len(header.entry_count) + .ok_or(OpenError::MalformedHeader("section extent overflows"))?; + let whole = section(data.len(), order.offset, len)?; + let packed = whole.start..whole.end - ARENA_ORDER_CHECKSUM_SIZE; + let checksum = u64::from_le_bytes(data[packed.end..whole.end].try_into().unwrap()); + + Some(StoredOrder { + packed, + width: order.width, + checksum, + }) + } + }; + // Size the cache to this table: never more slots than it has frames, and // nothing at all for a raw arena, which is read straight out of the mmap. let cache = match &seek_table { @@ -344,6 +411,8 @@ impl HashDb { lengths, arena, seek_table, + stored_order, + rebuilt_order: OnceLock::new(), cache, decompressions: AtomicU64::new(0), healthy: AtomicBool::new(true), @@ -505,6 +574,17 @@ impl HashDb { self.inner.header.arena_compressed_size } + /// Bytes this file spends on the arena-order section, or `None` if it + /// carries none and the reader sorts for the order instead. + /// + /// See [`ArenaOrder`](crate::ArenaOrder) for what that costs either way. + pub fn arena_order_size(&self) -> Option { + self.inner + .stored_order + .as_ref() + .map(|stored| (stored.packed.len() + ARENA_ORDER_CHECKSUM_SIZE) as u64) + } + /// Number of zstd frames this table has decompressed over its lifetime. #[cfg(test)] pub(crate) fn decompressions(&self) -> u64 { @@ -520,9 +600,68 @@ impl HashDb { /// Iterate entries in arena order (path order, **not** key order) so each frame /// decompresses once. Entries that fail to decompress are skipped; `verify()` reports them. + /// + /// Reads the arena-order section when the table carries one and otherwise + /// sorts for it once, so the permutation is not rebuilt per call either way - + /// see [`ArenaOrder`](crate::ArenaOrder). pub fn iter(&self) -> impl Iterator)> { - self.inner.arena_order().into_iter().filter_map(move |i| { - let bytes = self.inner.lookup(i)?; + let ranks = self.inner.arena_ranks(); + + (0..ranks.len()).filter_map(move |rank| { + let i = ranks.get(rank); + let bytes = self.inner.lookup_rank(i)?; + Some((self.inner.key_at(i), PathRef::from(bytes))) + }) + } + + /// Every path in the table, in arena order, without reading a key. + /// + /// [`iter`](HashDb::iter) without the key array: the same walk, the same + /// one-decompression-per-frame, and nothing paged in from the keys. What a + /// name list, an autocomplete corpus, or a dump of the table wants. + /// + /// Paths that will not decompress are skipped, exactly as `iter` skips them. + pub fn values(&self) -> impl Iterator> { + let ranks = self.inner.arena_ranks(); + + (0..ranks.len()).filter_map(move |rank| { + let bytes = self.inner.lookup_rank(ranks.get(rank))?; + Some(PathRef::from(bytes)) + }) + } + + /// Every entry whose path starts with `prefix`, in path order. + /// + /// The arena is sorted by path, so the matches are one contiguous run and + /// finding it is a binary search - about `log2(entries)` frames decompressed, + /// however many entries the table holds and however many the prefix matches. + /// The iterator is lazy: `take(n)` stops the walk rather than filtering a + /// list that was already built. + /// + /// ```no_run + /// # use ltk_hashdb::HashDb; + /// # let db = HashDb::open("game.lhdb")?; + /// for (hash, path) in db.prefix("assets/characters/ahri/").take(20) { + /// println!("{hash:016x} {path}"); + /// } + /// # Ok::<(), ltk_hashdb::OpenError>(()) + /// ``` + /// + /// Only meaningful for a table whose arena is in path order. That is the + /// reference writer's layout and what every published table does, but the + /// format lets a writer lay the arena out however it likes, so this is a + /// convention the reader trusts rather than a rule it can check - `verify` + /// proves the arena order is a permutation, not that it is sorted. + /// + /// An empty prefix matches every entry, which makes this the keyed + /// counterpart to [`values`](HashDb::values). + pub fn prefix<'a>(&'a self, prefix: &str) -> impl Iterator)> + 'a { + let ranks = self.inner.arena_ranks(); + let range = self.inner.prefix_ranks(&ranks, prefix.as_bytes()); + + range.filter_map(move |rank| { + let i = ranks.get(rank); + let bytes = self.inner.lookup_rank(i)?; Some((self.inner.key_at(i), PathRef::from(bytes))) }) } @@ -550,6 +689,7 @@ impl HashDb { let inner = &*self.inner; inner.verify_checksum()?; inner.verify_key_order()?; + inner.verify_arena_order()?; // Compressed arenas are walked in arena order so each frame decompresses once // and only the current run is resident - never the whole arena at once. A raw @@ -559,8 +699,9 @@ impl HashDb { inner.verify_entry(i)?; } } else { - for i in inner.arena_order() { - inner.verify_entry(i)?; + let ranks = inner.arena_ranks(); + for rank in 0..ranks.len() { + inner.verify_entry(ranks.get(rank))?; } } @@ -587,12 +728,14 @@ impl HashDb { /// # Errors /// /// [`VerifyError::ChecksumMismatch`] if the stored bytes do not hash to the - /// header's digest, or [`VerifyError::Malformed`] if the keys are not - /// strictly ascending. + /// header's digest or the arena-order section does not hash to its own, or + /// [`VerifyError::Malformed`] if the keys are not strictly ascending or the + /// arena order is not a permutation running forward through the arena. pub fn verify_index(&self) -> Result<(), VerifyError> { let inner = &*self.inner; inner.verify_checksum()?; - inner.verify_key_order() + inner.verify_key_order()?; + inner.verify_arena_order() } } @@ -616,6 +759,7 @@ impl fmt::Debug for HashDb { &inner.seek_table.as_ref().map_or(0, SeekTable::num_frames), ) .field("cached_frames", &inner.cache.capacity()) + .field("stores_arena_order", &inner.stored_order.is_some()) .finish() } } @@ -625,12 +769,89 @@ impl Inner { self.header.entry_count as usize } - /// Entry indices sorted by arena offset (path order); walking them this way - /// decompresses each frame once, keeping only the current run resident. - fn arena_order(&self) -> Vec { - let mut order: Vec = (0..self.len()).collect(); - order.sort_unstable_by_key(|&i| self.offset_at(i)); - order + /// Entry indices in arena order (path order): read from the file's section, + /// or sorted for on first use and kept. + /// + /// Walking them decompresses each frame once and keeps only the current run + /// resident, which is why every full-table read goes through here. + fn arena_ranks(&self) -> ArenaRanks<'_> { + match &self.stored_order { + Some(stored) => ArenaRanks { + packed: &self.backing.bytes()[stored.packed.clone()], + width: stored.width, + }, + None => ArenaRanks { + packed: self.rebuilt_order.get_or_init(|| self.rebuild_order()), + width: arena_order_width(self.header.entry_count), + }, + } + } + + /// Sort the permutation the file did not carry, packed the way it would have + /// been so that one accessor serves both. + fn rebuild_order(&self) -> Box<[u8]> { + let n = self.len(); + let width = arena_order_width(self.header.entry_count); + + // Ties are entries sharing one arena extent: identical paths, and the + // empty path against whatever was written next. Length breaks them the + // way path order does - the empty string first - so this reproduces a + // stored section rather than merely resembling one. + let mut order: Vec = (0..n).collect(); + order.sort_unstable_by_key(|&i| (self.offset_at(i), self.len_at(i))); + + let mut packed = Vec::with_capacity(n * width); + for i in order { + packed.extend_from_slice(&(i as u64).to_le_bytes()[..width]); + } + + packed.into_boxed_slice() + } + + /// The half-open rank range whose paths start with `prefix`. + /// + /// Two binary searches over one sorted sequence: the first rank not below the + /// prefix, and the first past the run that carries it. + fn prefix_ranks(&self, ranks: &ArenaRanks<'_>, prefix: &[u8]) -> Range { + let start = self.partition_ranks(ranks, |path| path < prefix); + let end = self.partition_ranks(ranks, |path| path < prefix || path.starts_with(prefix)); + + start..end + } + + /// `partition_point` over the ranks, reading one path per probe. + fn partition_ranks(&self, ranks: &ArenaRanks<'_>, below: impl Fn(&[u8]) -> bool) -> usize { + let mut lo = 0; + let mut hi = ranks.len(); + while lo < hi { + let mid = lo + (hi - lo) / 2; + // A path that will not decompress sorts to the left, which is the + // same "skipped" the rest of the read path gives it. + let before = match self.lookup_rank(ranks.get(mid)) { + Some(bytes) => below(bytes.as_slice()), + None => true, + }; + + if before { + lo = mid + 1; + } else { + hi = mid; + } + } + + lo + } + + /// Entry `i`'s bytes for an arena-order walk, where `i` came from an + /// untrusted rank: out of range reads as a miss and marks the table + /// unhealthy, like any other corruption the read path swallows. + fn lookup_rank(&self, i: usize) -> Option> { + if i >= self.len() { + self.healthy.store(false, Ordering::Relaxed); + return None; + } + + self.lookup(i) } fn index_of(&self, hash: u64) -> Option { @@ -738,7 +959,6 @@ impl Inner { } } - /// Check one entry the way `verify` needs it checked: in bounds and valid UTF-8. /// The header's digest against the bytes as they sit on disk. Compressed /// arenas are hashed compressed, so nothing is decoded to run this. fn verify_checksum(&self) -> Result<(), VerifyError> { @@ -768,6 +988,56 @@ impl Inner { Ok(()) } + /// The arena-order section, when the file carries one: its own digest, that + /// it names every entry exactly once, and that it runs forward through the + /// arena. + /// + /// Index-only - nothing here decompresses a frame. What stays unproven is + /// that the arena is in *path* order, which is a writer convention rather + /// than something the format states; see [`HashDb::prefix`]. + fn verify_arena_order(&self) -> Result<(), VerifyError> { + let Some(stored) = &self.stored_order else { + return Ok(()); + }; + + let packed = &self.backing.bytes()[stored.packed.clone()]; + let mut hasher = Xxh3::new(); + hasher.update(packed); + if hasher.digest() != stored.checksum { + return Err(VerifyError::ChecksumMismatch); + } + + // n distinct in-range indices over n ranks is a permutation, so one + // seen-bit per entry settles it; the extent check then settles that the + // permutation is the arena's own order. + let ranks = ArenaRanks { + packed, + width: stored.width, + }; + let mut seen = vec![false; self.len()]; + let mut prev = (0u64, 0u16); + for rank in 0..ranks.len() { + let i = ranks.get(rank); + let slot = seen.get_mut(i).ok_or(VerifyError::Malformed( + "arena order names an entry that does not exist", + ))?; + if std::mem::replace(slot, true) { + return Err(VerifyError::Malformed("arena order names an entry twice")); + } + + let here = (self.offset_at(i), self.len_at(i)); + if here < prev { + return Err(VerifyError::Malformed( + "arena order does not run forward through the arena", + )); + } + prev = here; + } + + Ok(()) + } + + /// Check one entry the way `verify` needs it checked: in bounds and valid UTF-8. fn verify_entry(&self, i: usize) -> Result<(), VerifyError> { let bytes = self .bytes_at(i)? @@ -954,6 +1224,24 @@ mod tests { assert!(keys.iter().any(|&k| db.get(k).is_some())); } + /// `prefix` is a binary search, so finding a run in a many-frame table must + /// cost frames proportional to `log2(entries)` - not to the table. + #[test] + fn prefix_does_not_walk_the_whole_arena() { + let db = compressed_db(128); + let frames = db.inner.seek_table.as_ref().unwrap().num_frames() as u64; + assert!(frames > 8, "fixture should span many frames"); + + // `champ1` without the slash would also match champ10..champ19; with it, + // exactly one path. + assert_eq!(db.prefix("assets/characters/champ1/").count(), 1); + assert!( + db.decompressions() < frames, + "{} frames decompressed of {frames}", + db.decompressions() + ); + } + #[test] fn for_each_batch_matches_get_batch() { let db = compressed_db(128); diff --git a/crates/ltk_hashdb/src/writer.rs b/crates/ltk_hashdb/src/writer.rs index 8e33d33..786ea81 100644 --- a/crates/ltk_hashdb/src/writer.rs +++ b/crates/ltk_hashdb/src/writer.rs @@ -5,9 +5,10 @@ use std::io::{Seek, Write}; use xxhash_rust::xxh3::Xxh3; use crate::header::{ - Header, OffsetWidth, FLAG_ARENA_COMPRESSED, FLAG_CASE_INSENSITIVE, HEADER_SIZE, + arena_order_width, ArenaOrderRef, Header, OffsetWidth, FLAG_ARENA_COMPRESSED, + FLAG_CASE_INSENSITIVE, HEADER_SIZE, }; -use crate::{BuildError, Casing, Compression, HashKind, KeyConfig, KeyWidth}; +use crate::{ArenaOrder, BuildError, Casing, Compression, HashKind, KeyConfig, KeyWidth}; /// Collects `(key, path)` pairs, then [`HashDbWriter::build`] sorts by key, dedups, /// assigns arena offsets, and writes the file. @@ -16,6 +17,7 @@ pub struct HashDbWriter { compression: Compression, hash_kind: HashKind, casing: Casing, + arena_order: ArenaOrder, entries: Vec<(u64, Box)>, } @@ -23,8 +25,14 @@ pub struct HashDbWriter { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct BuildStats { pub entries: usize, + pub arena_decompressed_size: u64, + pub arena_compressed_size: u64, + + /// Bytes spent on the arena-order section, `0` when it was omitted. + pub arena_order_size: u64, + pub file_len: u64, } @@ -38,6 +46,7 @@ impl HashDbWriter { compression, hash_kind: HashKind::Unspecified, casing: Casing::Sensitive, + arena_order: ArenaOrder::Omitted, entries: Vec::new(), } } @@ -66,6 +75,17 @@ impl HashDbWriter { self } + /// Write the arena-order section, or leave it out (the default). + /// + /// The writer already knows the permutation - it is the sort it does to lay + /// the arena out - so storing it costs build time nothing and file size + /// [`ArenaOrder::Stored`]'s documented share. See that variant for what a + /// reader does with it and what it does without it. + pub fn arena_order(mut self, arena_order: ArenaOrder) -> Self { + self.arena_order = arena_order; + self + } + pub fn insert(&mut self, key: u64, path: &str) { self.entries.push((key, path.into())); } @@ -167,6 +187,26 @@ impl HashDbWriter { flags |= FLAG_CASE_INSENSITIVE; } + // `by_path` *is* the arena-order permutation - entry indices in the order + // their paths sit in the arena - so the section is a repacking of a sort + // that already happened, not a second one. + let order = match self.arena_order { + ArenaOrder::Omitted => None, + ArenaOrder::Stored => { + let width = arena_order_width(self.entries.len() as u64); + let mut packed = Vec::with_capacity(by_path.len() * width + 8); + for &i in &by_path { + push_uint(&mut packed, i as u64, width); + } + + let mut hasher = Xxh3::new(); + hasher.update(&packed); + packed.extend_from_slice(&hasher.digest().to_le_bytes()); + + Some((packed, width)) + } + }; + // Section offsets. The offsets section is padded to its own width; that only // bites when a u32-key table has an odd entry count and spills to u64 offsets. let keys_offset = HEADER_SIZE as u64; @@ -174,7 +214,11 @@ impl HashDbWriter { (keys_offset + keys.len() as u64).next_multiple_of(offset_width.bytes() as u64); let pad = (offsets_offset - keys_offset) as usize - keys.len(); let arena_offset = offsets_offset + offsets.len() as u64 + lengths.len() as u64; + let arena_end = arena_offset + stored_arena.len() as u64; + // The header's checksum stays keys‖offsets‖lengths‖arena, exactly as a + // reader built before the arena-order section computes it; the section + // carries its own digest instead. let mut hasher = Xxh3::new(); hasher.update(&keys); hasher.update(&offsets); @@ -193,6 +237,10 @@ impl HashDbWriter { arena_decompressed_size, arena_compressed_size: stored_arena.len() as u64, checksum: hasher.digest(), + arena_order: order.as_ref().map(|&(_, width)| ArenaOrderRef { + offset: arena_end, + width, + }), }; out.write_all(&header.encode())?; @@ -201,13 +249,21 @@ impl HashDbWriter { out.write_all(&offsets)?; out.write_all(&lengths)?; out.write_all(&stored_arena)?; + let arena_order_size = match &order { + Some((packed, _)) => { + out.write_all(packed)?; + packed.len() as u64 + } + None => 0, + }; out.flush()?; Ok(BuildStats { entries: self.entries.len(), arena_decompressed_size, arena_compressed_size: stored_arena.len() as u64, - file_len: arena_offset + stored_arena.len() as u64, + arena_order_size, + file_len: arena_end + arena_order_size, }) } } diff --git a/crates/ltk_hashdb/tests/roundtrip.rs b/crates/ltk_hashdb/tests/roundtrip.rs index 1b8ab71..f2e6f61 100644 --- a/crates/ltk_hashdb/tests/roundtrip.rs +++ b/crates/ltk_hashdb/tests/roundtrip.rs @@ -3,8 +3,8 @@ use std::io::Cursor; use ltk_hashdb::{ - BuildError, Casing, Compression, HashDb, HashDbWriter, HashKind, KeyWidth, LayeredHashDb, - OpenError, VerifyError, + ArenaOrder, BuildError, Casing, Compression, HashDb, HashDbWriter, HashKind, KeyWidth, + LayeredHashDb, OpenError, VerifyError, }; fn build_with( @@ -28,6 +28,45 @@ fn build(key_width: KeyWidth, hash_kind: HashKind, entries: &[(u64, &str)]) -> V build_with(key_width, hash_kind, Compression::None, entries) } +/// What `build_with` produces, plus the arena-order section. +fn build_ordered(compression: Compression, entries: &[(u64, &str)]) -> Vec { + let mut w = HashDbWriter::new(KeyWidth::U64, compression) + .hash_kind(HashKind::Xxh64) + .casing(Casing::AsciiInsensitive) + .arena_order(ArenaOrder::Stored); + w.extend(entries.iter().copied()); + + let mut out = Cursor::new(Vec::new()); + let stats = w.build(&mut out).expect("build"); + assert_eq!(stats.file_len, out.get_ref().len() as u64); + + out.into_inner() +} + +/// The two arena-order header fields and the entry count, straight off the wire. +fn arena_order_at(bytes: &[u8]) -> (usize, usize, usize) { + let offset = u64::from_le_bytes(bytes[72..80].try_into().unwrap()) as usize; + let count = u64::from_le_bytes(bytes[16..24].try_into().unwrap()) as usize; + + (offset, bytes[15] as usize, count) +} + +/// Recompute the section's trailing digest, the way a writer that meant to +/// produce these bytes would have. +fn restamp_arena_order(bytes: &mut [u8]) { + let (offset, width, count) = arena_order_at(bytes); + let end = offset + count * width; + + let mut hasher = xxhash_rust::xxh3::Xxh3::new(); + hasher.update(&bytes[offset..end]); + bytes[end..end + 8].copy_from_slice(&hasher.digest().to_le_bytes()); +} + +/// Every path, in the order the arena holds them. +fn values(db: &HashDb) -> Vec { + db.values().map(|p| p.into_owned()).collect() +} + const GAME_ENTRIES: &[(u64, &str)] = &[ (0x0000_0000_0000_0001, "assets/characters/aatrox/aatrox.bin"), ( @@ -518,3 +557,198 @@ fn iter_yields_paths_in_lexicographic_order() { let paths: Vec = db.iter().map(|(_, p)| p.into_owned()).collect(); assert_eq!(paths, ["a/1", "b/2", "c/3"]); } + +/// The whole compatibility argument for adding a section without a version bump: +/// a reader built before it exists must find the file it always found. So the +/// section may only *append*, and the two header fields it fills were reserved +/// and zero in every build that shipped - including byte 64..72, the checksum, +/// which stays defined over keys‖offsets‖lengths‖arena and is covered here by +/// the head comparison. +#[test] +fn the_arena_order_section_only_appends() { + for compression in [ + Compression::None, + Compression::Zeekstd { + frame_size: 128, + level: 3, + }, + ] { + let plain = build_with(KeyWidth::U64, HashKind::Xxh64, compression, GAME_ENTRIES); + let ordered = build_ordered(compression, GAME_ENTRIES); + + // Four entries fit in one byte each, plus the section's own digest. + assert_eq!(ordered.len() - plain.len(), GAME_ENTRIES.len() + 8); + + let mut head = ordered[..plain.len()].to_vec(); + head[15] = 0; + head[72..80].fill(0); + assert_eq!( + head, plain, + "only the two reserved header fields differ before the section" + ); + } +} + +/// A stored permutation and one a reader sorts for are the same permutation, or +/// one of them is wrong. The empty path is the tie that could break differently: +/// it occupies no arena bytes, so it shares its offset with whatever was written +/// next - and `binhashes` really does contain it. +#[test] +fn a_stored_arena_order_matches_the_one_a_reader_sorts_for() { + let entries: &[(u64, &str)] = &[ + (1, ""), + (2, "assets/z.bin"), + (3, "assets/a.bin"), + (4, "assets/a.bin"), + (5, "b"), + ]; + + for compression in [ + Compression::None, + Compression::Zeekstd { + frame_size: 16, + level: 3, + }, + ] { + let sorted = HashDb::open_bytes(build_with( + KeyWidth::U64, + HashKind::Xxh64, + compression, + entries, + )) + .expect("open"); + let stored = HashDb::open_bytes(build_ordered(compression, entries)).expect("open"); + + assert_eq!(sorted.arena_order_size(), None); + assert_eq!(stored.arena_order_size(), Some(entries.len() as u64 + 8)); + + let paths = values(&sorted); + assert_eq!(paths, values(&stored)); + assert!( + paths.windows(2).all(|w| w[0] <= w[1]), + "the arena is in path order: {paths:?}" + ); + + let by_key = |db: &HashDb| -> Vec<(u64, String)> { + db.iter().map(|(k, p)| (k, p.into_owned())).collect() + }; + assert_eq!(by_key(&sorted), by_key(&stored)); + } +} + +/// A prefix names a contiguous run of the arena, and `prefix` returns exactly it +/// - whether the run is read out of a stored permutation or a sorted one. +#[test] +fn prefix_returns_the_run_under_it() { + let entries: &[(u64, &str)] = &[ + (1, "assets/characters/ahri/ahri.bin"), + (2, "assets/characters/ahri/skins/skin01.bin"), + (3, "assets/characters/ahriX.bin"), + (4, "assets/characters/aatrox/aatrox.bin"), + (5, "data/menu/main.bin"), + ]; + + for bytes in [ + build_with(KeyWidth::U64, HashKind::Xxh64, Compression::None, entries), + build_ordered(Compression::None, entries), + ] { + let db = HashDb::open_bytes(bytes).expect("open"); + let hits = |prefix: &str| -> Vec { + db.prefix(prefix).map(|(_, p)| p.into_owned()).collect() + }; + + assert_eq!( + hits("assets/characters/ahri/"), + [ + "assets/characters/ahri/ahri.bin", + "assets/characters/ahri/skins/skin01.bin" + ] + ); + // The trailing slash is what separates a directory from a sibling whose + // name merely starts the same way. + assert_eq!(hits("assets/characters/ahri").len(), 3); + assert_eq!(hits("data/"), ["data/menu/main.bin"]); + assert_eq!(hits("assets/characters/ahri/ahri.bin").len(), 1); + assert!(hits("nothing/").is_empty()); + assert!(hits("zzz").is_empty()); + + // An empty prefix is the whole table, keyed - `values` with the keys on. + assert_eq!(hits(""), values(&db)); + + // And the keys come back with the paths they belong to. + for (key, path) in db.prefix("assets/characters/ahri/") { + assert_eq!(db.get(key).as_deref(), Some(&*path)); + } + } +} + +/// The section is untrusted like every other: a bad one must be reported by +/// `verify_index` without decompressing anything, and must degrade reads to +/// misses rather than panicking or reading out of bounds. +#[test] +fn a_damaged_arena_order_is_caught_and_survived() { + let good = build_ordered(Compression::None, GAME_ENTRIES); + HashDb::open_bytes(good.clone()) + .expect("open") + .verify_index() + .expect("a freshly built section passes"); + + let (at, ..) = arena_order_at(&good); + + // Damage, caught by the section's own digest - the header's does not cover it. + let mut rotted = good.clone(); + rotted[at] ^= 0xff; + assert!(matches!( + HashDb::open_bytes(rotted).expect("open").verify_index(), + Err(VerifyError::ChecksumMismatch) + )); + + // A forgery that hashes correctly still has to be a permutation... + let mut twice = good.clone(); + twice[at + 1] = twice[at]; + restamp_arena_order(&mut twice); + assert!(matches!( + HashDb::open_bytes(twice).expect("open").verify_index(), + Err(VerifyError::Malformed(_)) + )); + + // ...running forward through the arena... + let mut backwards = good.clone(); + backwards.swap(at, at + 1); + restamp_arena_order(&mut backwards); + assert!(matches!( + HashDb::open_bytes(backwards).expect("open").verify_index(), + Err(VerifyError::Malformed(_)) + )); + + // ...over entries that exist. This one also has to stay readable: a rank + // pointing past the table reads as a miss and says the table is unhealthy. + let mut phantom = good; + phantom[at] = 200; + restamp_arena_order(&mut phantom); + let db = HashDb::open_bytes(phantom).expect("open"); + assert!(matches!(db.verify_index(), Err(VerifyError::Malformed(_)))); + assert_eq!(db.values().count(), GAME_ENTRIES.len() - 1); + assert!(!db.is_healthy()); + assert!( + db.get(GAME_ENTRIES[0].0).is_some(), + "lookups are unaffected" + ); +} + +/// A width too narrow to address the table cannot describe a real permutation, +/// so it is a malformed header rather than a section to ignore. +#[test] +fn an_unusable_arena_order_width_is_rejected() { + let entries: Vec<(u64, String)> = (0..300u64).map(|i| (i, format!("p/{i:04}"))).collect(); + let borrowed: Vec<(u64, &str)> = entries.iter().map(|(k, p)| (*k, p.as_str())).collect(); + + let mut bytes = build_ordered(Compression::None, &borrowed); + assert_eq!(bytes[15], 2, "300 entries need two bytes per rank"); + + bytes[15] = 1; + assert!(matches!( + HashDb::open_bytes(bytes), + Err(OpenError::MalformedHeader(_)) + )); +} diff --git a/crates/ltk_mimir_cli/src/main.rs b/crates/ltk_mimir_cli/src/main.rs index 29fa1ce..aa009d6 100644 --- a/crates/ltk_mimir_cli/src/main.rs +++ b/crates/ltk_mimir_cli/src/main.rs @@ -1,5 +1,5 @@ -//! The `mimir` CLI. Verbs: build / get / check / update / gen / merge / bundle / -//! verify / stats. +//! The `mimir` CLI. Verbs: build / get / ls / check / update / gen / merge / +//! bundle / verify / stats. mod bundle; mod check; @@ -14,7 +14,7 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; use clap::builder::TypedValueParser as _; use clap::{Parser, Subcommand}; -use ltk_hashdb::{Compression, HashDb, HashDbWriter}; +use ltk_hashdb::{ArenaOrder, Compression, HashDb, HashDbWriter}; use ltk_mimir_cache::{HashStore, Table}; use ltk_mimir_gen::guessers::{ CharacterSkin, CrossReference, ExtensionSwap, NumericRange, PrefixVariants, RegionLocale, @@ -68,6 +68,36 @@ enum Command { /// tables use 19; decompression speed is level-independent. #[arg(long, default_value_t = 19, conflicts_with = "raw")] level: i32, + + /// Store the arena-order index, so `ls` and full-table walks need no + /// sort on first use - at ~16% more file. See docs/BENCHMARKS.md. + #[arg(long)] + arena_order: bool, + }, + + /// List the paths under a prefix, from a .hashdb file or the shared cache. + Ls { + /// Path prefix to list. Empty lists the whole table, in path order. + #[arg(default_value = "")] + prefix: String, + + /// Look in this .hashdb file directly. + #[arg(long, conflicts_with = "table", required_unless_present = "table")] + file: Option, + + /// List from the shared cache's active version of this table instead + /// (cache dir: MIMIR_DIR override, else the platform data dir). + #[arg( + long, + conflicts_with = "file", + required_unless_present = "file", + value_parser = table_parser() + )] + table: Option, + + /// Stop after this many paths; 0 lists every match. + #[arg(long, default_value_t = 50)] + limit: usize, }, /// Resolve one hash from a .hashdb file or the shared cache. @@ -248,15 +278,28 @@ fn main() -> Result<()> { raw, frame_size, level, + arena_order, } => { let compression = if raw { Compression::None } else { Compression::Zeekstd { frame_size, level } }; - build(input, table, out, compression) + let arena_order = if arena_order { + ArenaOrder::Stored + } else { + ArenaOrder::Omitted + }; + + build(input, table, out, compression, arena_order) } Command::Get { hash, file, table } => get(&hash, file, table), + Command::Ls { + prefix, + file, + table, + limit, + } => ls(&prefix, file, table, limit), Command::Gen { known, unknown, @@ -338,8 +381,15 @@ fn read_hash_lines(input: &Path, mut on_entry: impl FnMut(u64, &str, &str)) -> R Ok(()) } -fn build(input: PathBuf, table: Table, out: PathBuf, compression: Compression) -> Result<()> { - let mut writer = HashDbWriter::with_key_config(table.key_config(), compression); +fn build( + input: PathBuf, + table: Table, + out: PathBuf, + compression: Compression, + arena_order: ArenaOrder, +) -> Result<()> { + let mut writer = + HashDbWriter::with_key_config(table.key_config(), compression).arena_order(arena_order); read_hash_lines(&input, |hash, _, path| { writer.insert(hash, path); })?; @@ -355,6 +405,10 @@ fn build(input: PathBuf, table: Table, out: PathBuf, compression: Compression) - stats.arena_compressed_size, stats.file_len, ); + if stats.arena_order_size > 0 { + println!(" arena order: {} B", stats.arena_order_size); + } + Ok(()) } @@ -514,24 +568,53 @@ fn gen_hashes( Ok(()) } -fn get(hash: &str, file: Option, table: Option
) -> Result<()> { - let hash = parse_hex_hash(hash).with_context(|| format!("bad hex hash {hash:?}"))?; - +/// A table named either directly or by its version in the shared cache, plus +/// what to call it in a message. +fn open_table(file: Option, table: Option
) -> Result<(HashDb, String)> { // clap guarantees exactly one of `file` / `table` is set. - let (db, source) = match (file, table) { + match (file, table) { (Some(file), _) => { let db = HashDb::open(&file).with_context(|| format!("opening {}", file.display()))?; - (db, file.display().to_string()) + Ok((db, file.display().to_string())) } (None, Some(table)) => { let store = HashStore::discover()?; let db = store .open(table) .with_context(|| format!("opening {table} from the shared cache"))?; - (db, format!("the shared cache ({table})")) + Ok((db, format!("the shared cache ({table})"))) } (None, None) => unreachable!("clap requires --file or --table"), - }; + } +} + +fn ls(prefix: &str, file: Option, table: Option
, limit: usize) -> Result<()> { + let (db, source) = open_table(file, table)?; + let width = db.key_width().bytes() * 2; + + // `prefix` is lazy, so a limit stops the walk rather than filtering a list + // that was already built - which is what makes listing one directory of a + // 2.3M-entry table cost the same as listing one of a small one. + let mut listed = 0; + for (hash, path) in db.prefix(prefix) { + println!("{hash:0width$x} {path}"); + listed += 1; + if listed == limit { + println!("... (--limit {limit} reached)"); + return Ok(()); + } + } + + if listed == 0 { + bail!("nothing under {prefix:?} in {source}"); + } + + Ok(()) +} + +fn get(hash: &str, file: Option, table: Option
) -> Result<()> { + let hash = parse_hex_hash(hash).with_context(|| format!("bad hex hash {hash:?}"))?; + let (db, source) = open_table(file, table)?; match db.get(hash) { Some(path) => { @@ -581,5 +664,16 @@ fn stats(file: PathBuf) -> Result<()> { "raw".to_owned() } ); + println!( + "arena order: {}", + match db.arena_order_size() { + Some(bytes) => format!( + "{bytes} B stored ({:.1}% of the file)", + 100.0 * bytes as f64 / file_len as f64 + ), + None => "not stored - the reader sorts for it on first use".to_owned(), + } + ); + Ok(()) } diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 3d2c0bc..e47dae6 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -12,6 +12,8 @@ arena**, the **16 KiB default frame size**, and the **level 19** publishing defa ```text cargo run --release -p ltk_hashdb --example bench_real -- data/cdragon data/build cargo run --release -p ltk_hashdb --example compression_lab # ordering/level/dict study + cargo run --release -p ltk_hashdb --example arena_order -- game.lhdb out.lhdb + cargo run --release -p ltk_hashdb --example path_coding -- game.lhdb cargo bench -p ltk_hashdb # criterion, uses the files built above MIMIR_CDRAGON_DIR=data/cdragon cargo test --release --test golden ``` @@ -112,14 +114,88 @@ sample resolved through `get_batch` (`batch hit`), and 1 M random probes (`miss` The defaults are writer-side choices only - the format records the seek table, so any frame size and level stay readable by every reader. +## Arena order: storing the permutation vs sorting for it + +The arena is laid out by path and the offsets are stored by key, so anything that +walks the arena forward - `iter`, `values`, `prefix`, `verify` - needs the +permutation between the two. It can be sorted for, or read from the optional +arena-order section. Measured against the published `game` table (2,291,347 +entries, 44,114,614 B) with `--example arena_order`, each phase against a freshly +opened table: + +| | sorted for | stored in the file | +|---|---:|---:| +| file size | 44,114,614 B | 50,988,663 B (**+15.6 %**) | +| first arena-order call | 516 ms | 11 ms | +| `prefix`, 13,511 hits, warm | 1.4 ms | 1.6 ms | +| `prefix`, 499 hits, warm | 0.3 ms | 0.4 ms | +| `prefix`, no match, warm | 0.1 ms | 0.2 ms | +| `values()`, whole table | 776 ms | 331 ms | + +So the section buys one thing: it removes a **~0.5 s sort** the first time a +process walks the arena, plus the ~18 MB of scratch that sort needs and the +6.9 MB it leaves behind per process. Warm, the two are the same code on the same +bytes and measure the same. Across all eight published tables the section would +cost **+16.5 %** (61.1 MiB → 71.3 MiB), paid by every consumer including the ones +that only ever call `get`. + +That is why [`ArenaOrder::Omitted`] is the default and the published tables do not +carry it: a prefix search is already single-digit milliseconds without it, and a +half-second once per process is a smaller price than a sixth of the download for +everyone. `mimir build --arena-order` is there for consumers who would rather pay +the bytes - a long-running asset browser opening the table on every launch, say. + +[`ArenaOrder::Omitted`]: https://docs.rs/ltk_hashdb/latest/ltk_hashdb/enum.ArenaOrder.html + +## Would factoring the paths out be smaller? + +The 187 MB of paths in the `game` arena are enormously redundant - 2.29 M paths +share only 78,792 directories - so it is worth asking whether storing each +directory once beats leaving the redundancy to zstd. Measured with +`--example path_coding`, all at level 19 against the same 44,114,614 B file: + +| arena encoding | on disk | paths per frame | +|---|---:|---:| +| **raw** (what ships) | 12,035,676 B | 199 | +| front-coded | 7,274,750 B | 1,045 | +| front-coded, at raw's frame granularity | 9,380,187 B | 199 | +| raw, at front coding's frame granularity | 10,083,790 B | 1,045 | +| interned: directory table + `(dir_id, filename)` | **16,728,132 B** | 199 | + +Two results. + +**Interning - each directory stored once, an id per entry - makes the file +bigger**, by 39 % of the arena. The directory strings do compress away, but the +`dir_id` array is 3 bytes per entry of *index*: read at random, so never +compressed, and it costs more than the redundancy it removes. That is the general +shape of the trap. The arena is only **27 % of the file** and compresses 15.6x; +the other 73 % - keys, offsets, lengths - is random-access and stored raw. Any +scheme that moves bytes from the arena into the index trades compressible bytes +for incompressible ones. + +**Front coding is worth about 2.7 MB, not the 4.8 MB it first appears.** Comparing +the top two rows is unfair: the same 16 KiB frame holds 5x more front-coded paths, +so half the gain is a coarser lookup granularity that raw can also buy - and does, +in row four - by raising `--frame-size` at the cost of hit latency. At equal paths +per frame it is a real ~22 % off the arena, i.e. ~6 % off the file. + +That 6 % is not additive, though. A front-coded arena cannot be addressed by +`(offset, length)`, so it needs a restart per frame, a replay to read any entry, +and a `flags` bit - which is the one kind of change [FORMAT.md](FORMAT.md)'s +compatibility rule rules out, because every reader already shipped would reject +the file. + ## Memory profile - `open` maps the file: no allocation proportional to table size; the OS pages in what lookups touch (keys for misses, plus one decompressed frame per hit - the frame buffer is the only per-lookup allocation). -- `iter`/`load_all` materialize an index-ordering vector (8 bytes/entry) and - decompress one frame run at a time; `load_all` (opt-in) then owns everything - (~arena size + `HashMap` overhead) for the expand-once profile. +- `iter`/`values`/`prefix` need the arena order: a table carrying the section + reads it out of the mapping and allocates nothing, and one without it sorts + once and keeps the result (1-8 bytes/entry, 3 on `game`) for as long as a handle + to the table is open - shared by every clone, not rebuilt per call. +- `load_all` (opt-in) then owns everything (~arena size + `HashMap` overhead) for + the expand-once profile. ## Correctness vs the txt corpus (golden test) diff --git a/docs/CONSUMERS.md b/docs/CONSUMERS.md index 1a92eed..d3eef2e 100644 --- a/docs/CONSUMERS.md +++ b/docs/CONSUMERS.md @@ -121,6 +121,40 @@ for (hash, path) in db.iter() { /* streams in path order, one decompress per fra `iter` yields in **arena order** (lexicographic path order, *not* key order), which is also the natural order for building tree views or prefix scans. +`values()` is the same walk without reading a key - a name list, an autocomplete +corpus, a dump: + +```rust +for path in db.values() { /* every path, in path order */ } +``` + +### Searching by path prefix + +Because the arena is sorted by path, the entries under a directory are one +contiguous run, and finding it is a binary search rather than a scan: + +```rust +for (hash, path) in db.prefix("assets/characters/ahri/").take(50) { + println!("{hash:016x} {path}"); +} +``` + +The cost is about `log2(entries)` frames decompressed - single-digit milliseconds +on the 2.3 M-entry game table, whether the prefix matches 13,000 paths or none. +The iterator is lazy, so `take(n)` stops the walk instead of filtering a list that +was already built. An empty prefix is the whole table. `LayeredHashDb::prefix` +applies the same shadowing rule `iter` does, one binary search per base. + +This is the query consumers used to download the CommunityDragon txt list for. + +Two caveats. It is only meaningful for a table whose arena is in **path order** - +the reference writer's layout, and what every published table does, but the format +permits any layout and the reader cannot check it cheaply. And the first call on a +table that does not carry the arena-order section pays for a sort (~0.5 s on +`game`, once per process, shared by every clone); a table built with +`mimir build --arena-order` reads the order out of the file instead, at ~16 % more +file. See [BENCHMARKS.md](BENCHMARKS.md). + `load_all()` decodes the whole table into an owned `HashMap>`. This is the opt-in "resident mode" for tools that genuinely need map semantics or maximum lookup throughput - it forfeits the shared-page-cache benefit and costs the full decompressed @@ -403,7 +437,8 @@ let report = store.gc()?; `open` validates structure only. There are two checks above it: ```rust -db.verify_index()?; // xxh3 checksum over every stored byte, keys strictly ascending +db.verify_index()?; // xxh3 checksum over every stored byte, keys strictly ascending, + // and the arena-order section if there is one db.verify()?; // the above, plus every entry in bounds and valid UTF-8 ``` diff --git a/docs/FORMAT.md b/docs/FORMAT.md index 6516146..f7ea720 100644 --- a/docs/FORMAT.md +++ b/docs/FORMAT.md @@ -26,6 +26,8 @@ format under the `.lhdb` extension (identical bytes, League convention). │ Lengths (entry_count × 2) │ ├──────────────────────────────────────────────┤ │ Arena (arena_compressed_size bytes) │ +├─ optional ───────────────────────────────────┤ +│ Arena order (entry_count × width + 8 bytes) │ └──────────────────────────────────────────────┘ ``` @@ -46,7 +48,7 @@ stored once). | 12 | `key_width` | `u8` | 4 = u32 table, 8 = u64 table | | 13 | `offset_width` | `u8` | 4 or 8; the writer picks 4 while `arena_decompressed_size` fits in a u32, else 8; the reader honors whatever is declared | | 14 | `opt_flags` | `u8` | **optional** flags: none defined yet; an unrecognized bit **must** be ignored | -| 15 | reserved | `u8` | written as zero, ignored on read | +| 15 | `arena_order_width` | `u8` | bytes per entry in the arena-order section, `1..=8`; `0` when there is none | | 16 | `entry_count` | `u64` | | | 24 | `keys_offset` | `u64` | file offset of the keys section; writers must 8-align it (the reference writer emits 80), readers bounds-check and honor the declared value | | 32 | `offsets_offset` | `u64` | file offset of the offsets section; writers must `offset_width`-align it | @@ -54,7 +56,7 @@ stored once). | 48 | `arena_decompressed_size` | `u64` | raw (decompressed) arena length | | 56 | `arena_compressed_size` | `u64` | arena bytes on disk; == decompressed if raw | | 64 | `checksum` | `u64` | xxh3-64 of keys ‖ offsets ‖ lengths ‖ arena, each as stored on disk (inter-section padding excluded) | -| 72 | reserved | `[u8;8]` | written as zero, ignored on read | +| 72 | `arena_order_offset` | `u64` | file offset of the arena-order section; `0` = the file carries none | The lengths section has no header field: it sits immediately after the offsets, at `offsets_offset + entry_count × offset_width` (u16 entries are always 2-aligned there). @@ -75,6 +77,12 @@ So an index that speeds up a scan is an `opt_flags` bit; a different arena encoding is a `flags` bit. When in doubt, ask what a reader that ignores the bit would produce: a correct answer more slowly, or a wrong one. +A capability that needs a field of its own does not also need a bit. The +arena-order section is announced by its own offset being non-zero - bytes 72..80 +and byte 15 were reserved-and-zero in every build that shipped, so a reader +predating the section sees zeroes it already ignores, and there is no flag that +could disagree with the offset beside it. + ### `hash_kind` The algorithm alone - the casing rule is recorded separately in the @@ -127,6 +135,9 @@ refusing the file outright. Zstandard Seekable Format stream whose decompressed content is the raw arena (its seek table lives inside the stream; the whole arena is also a valid ordinary zstd stream). +- **Arena order** (optional) - `entry_count` entry indices, `arena_order_width` + bytes each, listing the entries in the order their paths sit in the arena; + followed by an xxh3-64 of those bytes. Present iff `arena_order_offset != 0`. Sections are contiguous except for zero padding between keys and offsets when `offsets_offset` needs realignment (only possible for a u32-key table with u64 offsets). @@ -141,6 +152,40 @@ table this compresses ~4× better than key order (see `docs/BENCHMARKS.md`) and directory-local batch lookups touch fewer frames. It also lets identical paths under different keys share one arena extent. +### Arena order + +The arena is laid out by path while offsets and lengths are stored by key, so +walking the arena forward - or binary-searching it for a path prefix - means +knowing the permutation between the two orders. A reader can always recover it by +sorting the offsets; this section is that permutation written down. + +``` +arena_order[rank] = entry index, for rank in 0 .. entry_count +``` + +- Entries are `arena_order_width` little-endian bytes each, wide enough to hold + `entry_count - 1`. The reference writer picks the narrowest such width (3 bytes + for the ~2.3M-entry League `game` table); a reader honors whatever is declared + and rejects a width too narrow to address the table. +- The `entry_count × width` bytes are followed by an **xxh3-64 of themselves**. + The header's `checksum` deliberately does not cover this section: it is defined + over keys‖offsets‖lengths‖arena, and a reader built before the section existed + still computes it that way, so the section carries its own digest instead. +- The permutation must name every entry exactly once, and must run forward: for + consecutive ranks, `(offset, length)` never decreases. That ordering is what + makes an arena walk decompress each frame once, and is checked by `verify`. +- It records the order, not the *rule*. That the arena is sorted by path is the + reference writer's layout (see *Arena ordering*) and what a prefix search + depends on, but the format does not require it and a reader cannot check it + cheaply. + +**Adding it does not change any other byte.** The section is appended after the +arena and the two header fields it fills were reserved and zero, so a reader that +predates it opens the file, reads it, and checksums it exactly as before. It is +therefore optional in both directions: a writer may leave it out, and a reader +may ignore it and sort for the order instead. See `docs/BENCHMARKS.md` for what +each choice costs. + ## Lookup ``` @@ -165,7 +210,11 @@ On open, before trusting any offset: - for compressed arenas: the trailing seek table parses, its total decompressed size equals `arena_decompressed_size`, and no frame's decompressed size exceeds the seekable-format maximum (1 GiB); -- all section extents in bounds (overflow-checked). +- all section extents in bounds (overflow-checked); +- if `arena_order_offset != 0`: `arena_order_width` is `1..=8` and wide enough for + `entry_count`, and the section is in bounds. Its *contents* are not read at open - + each rank is range-checked where it is used, and a rank naming an entry that does + not exist reads as a miss. Per-entry extents are **not** validated at open (that would touch every offsets/lengths page); instead every read bounds-checks its own extent against @@ -174,7 +223,8 @@ stay untrusted after open too: every frame extent and decompressed size is re-ch when the frame is read, a lookup whose frame fails to decompress reports a miss, and invalid UTF-8 is replaced lossily rather than erroring. -`verify()` additionally checks the xxh3 checksum, strict key ordering, and that every -entry's extent is in bounds and valid UTF-8 in the (decompressed) arena - it is opt-in +`verify()` additionally checks the xxh3 checksum, strict key ordering, the arena-order +section (its own digest, that it is a permutation, and that it runs forward), and that +every entry's extent is in bounds and valid UTF-8 in the (decompressed) arena - it is opt-in so `open` stays lazy (the shared-cache manifest carries a sha256 checked at download time).