From 060818c5a9a6d35c1cea2c16d031e872afaf4911 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 24 Oct 2026 18:46:05 +0300 Subject: [PATCH 01/15] perf(encode): write the sequence section under BMI2 where the tier has it --- zstd/src/encoding/blocks/compressed.rs | 505 ++++++++++++++----------- 1 file changed, 282 insertions(+), 223 deletions(-) diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index 26264899..c0d8c4d1 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -6,6 +6,7 @@ use crate::{ bit_io::BitWriter, blocks::block::BlockType, encoding::block_header::BlockHeader, + encoding::fastpath::FastpathKernel, encoding::frame_compressor::{CompressState, FseTables, PreviousFseTable, SharedFseTable}, encoding::workspace::{RegionVec, Workspace, region_bytes}, encoding::{Matcher, Sequence}, @@ -917,6 +918,7 @@ fn encode_block_parts( ) }); + let kernel = state.copy_kernel; // Destructured because the default-table accessors borrow the whole of // `FseTables`, which would collide with taking a `*_next` slot mutably. let FseTables { @@ -975,12 +977,11 @@ fn encode_block_parts( encode_table(&ml_mode, &mut writer); encode_sequences( + kernel, raw_sequences, codes, &mut writer, - &ll_mode, - &ml_mode, - &of_mode, + [&ll_mode, &ml_mode, &of_mode], [ll_default, ml_default, of_default], ); @@ -2776,245 +2777,303 @@ fn commit_last_used_table( } } -fn encode_sequences( - sequences: &[RawSequence], - // One packed [`SequenceCode`] per sequence, in the same order, from the - // pass that derived the offset codes. - codes: &[u32], - writer: &mut BitWriter<&mut Vec>, - ll_mode: &FseTableMode<'_>, - ml_mode: &FseTableMode<'_>, - of_mode: &FseTableMode<'_>, - // The three predefined tables, in LL / ML / OF order. Passed rather than - // read off `FseTables`, whose accessors borrow the whole struct while the - // caller is holding its slots. - defaults: [&FSETable; 3], -) { - fn mode_table<'a>(mode: &'a FseTableMode<'_>, default: &'a FSETable) -> Option<&'a FSETable> { - mode.as_table(default) - } - - debug_assert_eq!(codes.len(), sequences.len()); - let sequence = sequences[sequences.len() - 1]; - let code = SequenceCode(codes[codes.len() - 1]); - let (ll_code, ll_num_bits) = (code.ll_code(), code.ll_bits()); - let (ml_code, ml_num_bits) = (code.ml_code(), code.ml_bits()); - let (of_code, of_num_bits) = (code.of_code(), code.of_bits()); - let ll_add_bits = low_bits(sequence.ll, ll_num_bits); - let ml_add_bits = low_bits(sequence.ml - 3, ml_num_bits); - let of_add_bits = low_bits(sequence.off_base, of_num_bits); - let [ll_default, ml_default, of_default] = defaults; - let ll_table = mode_table(ll_mode, ll_default); - let ml_table = mode_table(ml_mode, ml_default); - let of_table = mode_table(of_mode, of_default); - // Carried as bare indices, not `Option`s. A component has a state exactly - // when it has a table, and the tables are resolved once above the loop, so - // wrapping the state too meant re-asking the same question three times per - // sequence and re-wrapping the answer. An absent component's index is never - // read: every site that touches one is already inside its table's `Some`. - let mut ll_state = ll_table.map_or(0, |table| table.start_state(ll_code).index); - let mut ml_state = ml_table.map_or(0, |table| table.start_state(ml_code).index); - let mut of_state = of_table.map_or(0, |table| table.start_state(of_code).index); - - writer.write_bits(ll_add_bits, ll_num_bits); - writer.write_bits(ml_add_bits, ml_num_bits); - writer.write_bits(of_add_bits, of_num_bits); - - // Upstream zstd-faithful sequence loop: write state diffs + extras via - // unchecked fast-path adds with explicit `flush_bulk` calls at - // safe burst boundaries. Per-sequence bit budget: - // state diffs: of (<=8) + ml (<=9) + ll (<=9) = 26 bits. - // extras: ll (<=16) + ml (<=16) + of (<=31). - // - // One flush per sequence is the common case: the two inside the body are - // conditional on the extras not fitting beside the diffs, which is - // upstream's own accounting in `ZSTD_encodeSequences_body` - // (`zstd_compress_sequences.c:303-367`), and the one at the end of the - // body is what leaves the accumulator with under a byte for the next - // round. - // - // Pre-reserve output capacity for the worst-case sequence section - // size (~10 bytes/sequence + 32 byte slack) so the per-flush - // `extend_from_slice` never triggers a Vec realloc. - if sequences.len() > 1 { - writer.reserve_output(sequences.len() * 12 + 64); - // Pre-loop flush: the safe `write_bits` calls above for the - // final sequence's add_bits leave `bits_in_partial` in - // 0..=63. Before the first unchecked-add burst we drain to - // < 8 leftover so the per-burst budget math (state diffs ≤ - // 30 + leftover ≤ 8 = 38 < 64) holds invariantly. - // SAFETY: `reserve_output` above guarantees capacity ≥ - // current_len + sequences.len() * 12 + 64 ≥ current_len + 8. - unsafe { - writer.flush_bulk(); - } - // Walked as a slice rather than by index: the same order, without the - // bounds check and the index arithmetic that `sequences[i]` pays on - // every sequence. The last one is coded through the FSE init states - // above, so it is not in this range. - for (&sequence, &code) in sequences[..sequences.len() - 1] - .iter() - .zip(codes[..codes.len() - 1].iter()) - .rev() - { - let code = SequenceCode(code); - let (ll_code, ll_num_bits) = (code.ll_code(), code.ll_bits()); - let (ml_code, ml_num_bits) = (code.ml_code(), code.ml_bits()); - let (of_code, of_num_bits) = (code.of_code(), code.of_bits()); - let ll_add_bits = low_bits(sequence.ll, ll_num_bits); - let ml_add_bits = low_bits(sequence.ml - 3, ml_num_bits); - let of_add_bits = low_bits(sequence.off_base, of_num_bits); - - // State diffs burst: max 30 bits (10+10+9 worst case for - // acc_log ≤ 9 ll/ml + acc_log ≤ 8 of) + ≤ 7 leftover from - // prior flush = ≤ 37 bits total — well under 64. - // - // SAFETY (for every `write_bits_64_no_check` below): - // - the prior `flush_bulk` left `bits_in_partial ≤ 7`; - // - each FSE state diff has `next.num_bits ≤ acc_log ≤ 10`; - // three diffs back-to-back add ≤ 30 bits → total ≤ 37, - // well below the 64-bit accumulator cap. - // - `diff = state.index - next.baseline` cannot exceed - // `(1 << num_bits) - 1`, so `diff >> num_bits == 0`. - // `reserve_output(sequences.len() * 12 + 64)` above - // pre-allocated enough spare capacity to cover every - // per-sequence flush in this loop (≤ 16 bytes per - // sequence, plus the 32-byte slack on top of the 64-byte - // header reserve). - // - // What the three diffs actually put in the accumulator, tallied so - // the budget below is checked against the widths the FSE tables - // produced rather than against the ceiling its derivation assumed. - #[cfg(debug_assertions)] - let mut state_diff_bits = 0usize; - if let Some(table) = of_table { - let next = table.next_state(of_code, of_state); - let diff = crate::fse::fse_encoder::transition_bits(of_state, next.num_bits); - unsafe { - writer.write_bits_64_no_check(diff as u64, next.num_bits as usize); - } +/// The sequence section's bitstream, shared by [`encode_sequences_scalar`] and +/// `encode_sequences_bmi2`. A macro so each tier's loop is compiled inside its +/// own function and the bit writer and the FSE lookups inline into it. +macro_rules! encode_sequences_body { + ($sequences:expr, $codes:expr, $writer:expr, $modes:expr, $defaults:expr) => {{ + let sequences: &[RawSequence] = $sequences; + let codes: &[u32] = $codes; + let writer: &mut BitWriter<&mut Vec> = $writer; + let [ll_mode, ml_mode, of_mode]: [&FseTableMode<'_>; 3] = $modes; + let [ll_default, ml_default, of_default]: [&FSETable; 3] = $defaults; + + debug_assert_eq!(codes.len(), sequences.len()); + let sequence = sequences[sequences.len() - 1]; + let code = SequenceCode(codes[codes.len() - 1]); + let (ll_code, ll_num_bits) = (code.ll_code(), code.ll_bits()); + let (ml_code, ml_num_bits) = (code.ml_code(), code.ml_bits()); + let (of_code, of_num_bits) = (code.of_code(), code.of_bits()); + let ll_add_bits = low_bits(sequence.ll, ll_num_bits); + let ml_add_bits = low_bits(sequence.ml - 3, ml_num_bits); + let of_add_bits = low_bits(sequence.off_base, of_num_bits); + let ll_table = ll_mode.as_table(ll_default); + let ml_table = ml_mode.as_table(ml_default); + let of_table = of_mode.as_table(of_default); + // Carried as bare indices, not `Option`s. A component has a state + // exactly when it has a table, and the tables are resolved once above + // the loop, so wrapping the state too meant re-asking the same question + // three times per sequence and re-wrapping the answer. An absent + // component's index is never read: every site that touches one is + // already inside its table's `Some`. + let mut ll_state = ll_table.map_or(0, |table| table.start_state(ll_code).index); + let mut ml_state = ml_table.map_or(0, |table| table.start_state(ml_code).index); + let mut of_state = of_table.map_or(0, |table| table.start_state(of_code).index); + + writer.write_bits(ll_add_bits, ll_num_bits); + writer.write_bits(ml_add_bits, ml_num_bits); + writer.write_bits(of_add_bits, of_num_bits); + + // Upstream zstd-faithful sequence loop: write state diffs + extras via + // unchecked fast-path adds with explicit `flush_bulk` calls at + // safe burst boundaries. Per-sequence bit budget: + // state diffs: of (<=8) + ml (<=9) + ll (<=9) = 26 bits. + // extras: ll (<=16) + ml (<=16) + of (<=31). + // + // One flush per sequence is the common case: the two inside the body + // are conditional on the extras not fitting beside the diffs, which is + // upstream's own accounting in `ZSTD_encodeSequences_body` + // (`zstd_compress_sequences.c:303-367`), and the one at the end of the + // body is what leaves the accumulator with under a byte for the next + // round. + // + // Pre-reserve output capacity for the worst-case sequence section + // size (~10 bytes/sequence + 32 byte slack) so the per-flush + // `extend_from_slice` never triggers a Vec realloc. + if sequences.len() > 1 { + writer.reserve_output(sequences.len() * 12 + 64); + // Pre-loop flush: the safe `write_bits` calls above for the + // final sequence's add_bits leave `bits_in_partial` in + // 0..=63. Before the first unchecked-add burst we drain to + // < 8 leftover so the per-burst budget math (state diffs ≤ + // 30 + leftover ≤ 8 = 38 < 64) holds invariantly. + // SAFETY: `reserve_output` above guarantees capacity ≥ + // current_len + sequences.len() * 12 + 64 ≥ current_len + 8. + unsafe { + writer.flush_bulk(); + } + // Walked as a slice rather than by index: the same order, without + // the bounds check and the index arithmetic that `sequences[i]` + // pays on every sequence. The last one is coded through the FSE + // init states above, so it is not in this range. + for (&sequence, &code) in sequences[..sequences.len() - 1] + .iter() + .zip(codes[..codes.len() - 1].iter()) + .rev() + { + let code = SequenceCode(code); + let (ll_code, ll_num_bits) = (code.ll_code(), code.ll_bits()); + let (ml_code, ml_num_bits) = (code.ml_code(), code.ml_bits()); + let (of_code, of_num_bits) = (code.of_code(), code.of_bits()); + let ll_add_bits = low_bits(sequence.ll, ll_num_bits); + let ml_add_bits = low_bits(sequence.ml - 3, ml_num_bits); + let of_add_bits = low_bits(sequence.off_base, of_num_bits); + + // State diffs burst: max 30 bits (10+10+9 worst case for + // acc_log ≤ 9 ll/ml + acc_log ≤ 8 of) + ≤ 7 leftover from + // prior flush = ≤ 37 bits total — well under 64. + // + // SAFETY (for every `write_bits_64_no_check` below): + // - the prior `flush_bulk` left `bits_in_partial ≤ 7`; + // - each FSE state diff has `next.num_bits ≤ acc_log ≤ 10`; + // three diffs back-to-back add ≤ 30 bits → total ≤ 37, + // well below the 64-bit accumulator cap. + // - `diff = state.index - next.baseline` cannot exceed + // `(1 << num_bits) - 1`, so `diff >> num_bits == 0`. + // `reserve_output(sequences.len() * 12 + 64)` above + // pre-allocated enough spare capacity to cover every + // per-sequence flush in this loop (≤ 16 bytes per + // sequence, plus the 32-byte slack on top of the 64-byte + // header reserve). + // + // What the three diffs actually put in the accumulator, tallied + // so the budget below is checked against the widths the FSE + // tables produced rather than against the ceiling its + // derivation assumed. #[cfg(debug_assertions)] - { - state_diff_bits += next.num_bits as usize; + let mut state_diff_bits = 0usize; + if let Some(table) = of_table { + let next = table.next_state(of_code, of_state); + let diff = crate::fse::fse_encoder::transition_bits(of_state, next.num_bits); + unsafe { + writer.write_bits_64_no_check(diff as u64, next.num_bits as usize); + } + #[cfg(debug_assertions)] + { + state_diff_bits += next.num_bits as usize; + } + of_state = next.index; } - of_state = next.index; - } - if let Some(table) = ml_table { - let next = table.next_state(ml_code, ml_state); - let diff = crate::fse::fse_encoder::transition_bits(ml_state, next.num_bits); - unsafe { - writer.write_bits_64_no_check(diff as u64, next.num_bits as usize); + if let Some(table) = ml_table { + let next = table.next_state(ml_code, ml_state); + let diff = crate::fse::fse_encoder::transition_bits(ml_state, next.num_bits); + unsafe { + writer.write_bits_64_no_check(diff as u64, next.num_bits as usize); + } + #[cfg(debug_assertions)] + { + state_diff_bits += next.num_bits as usize; + } + ml_state = next.index; } + if let Some(table) = ll_table { + let next = table.next_state(ll_code, ll_state); + let diff = crate::fse::fse_encoder::transition_bits(ll_state, next.num_bits); + unsafe { + writer.write_bits_64_no_check(diff as u64, next.num_bits as usize); + } + #[cfg(debug_assertions)] + { + state_diff_bits += next.num_bits as usize; + } + ll_state = next.index; + } + // The three state diffs and the three extra-bit fields share + // one 64-bit accumulator, and a flush here is only needed when + // the extras that follow would not fit beside them. Upstream + // asks exactly that question (`zstd_compress_sequences.c:350`): + // `ofBits + mlBits + llBits >= 64 - 7 - (LLFSELog + MLFSELog + + // OffFSELog)`, which with our accumulator logs (9 + 9 + 8 = 26) + // is 31. Below it the accumulator holds at most 7 leftover + 26 + // diff bits + 30 extra bits = 63, so nothing has to leave yet. + // Flushing unconditionally instead cost a second store, length + // commit and shift on every sequence, and a level-1 block has + // hundreds of thousands of them. + let extra_bits_total = of_num_bits + ml_num_bits + ll_num_bits; + // The thresholds are arithmetic on widths, so they are only + // right while the widths are what they were derived from. + // Pinned here, where the arithmetic happens, so a widened + // encoder fails at its cause rather than as an accumulator + // overflow further down or, in a release build, as a corrupted + // stream with nothing to point at. #[cfg(debug_assertions)] { - state_diff_bits += next.num_bits as usize; + debug_assert!( + state_diff_bits <= 26, + "state diffs took {state_diff_bits} bits; the thresholds below are \ + derived from a 26-bit ceiling (LLFSELog 9 + MLFSELog 9 + OffFSELog 8)", + ); + debug_assert!( + ll_num_bits <= 16 && ml_num_bits <= 16 && of_num_bits <= 31, + "extra-bit widths ll={ll_num_bits} ml={ml_num_bits} of={of_num_bits} \ + exceed the 16 / 16 / 31 the thresholds are derived from", + ); + // The bound the branch itself rests on: without a flush, + // the leftover, the diffs and all three extra fields have + // to fit. + debug_assert!( + extra_bits_total >= 31 || 7 + state_diff_bits + extra_bits_total <= 64, + "no flush at {extra_bits_total} extra bits, but 7 leftover + \ + {state_diff_bits} diff bits + {extra_bits_total} would overflow the \ + 64-bit accumulator", + ); } - ml_state = next.index; - } - if let Some(table) = ll_table { - let next = table.next_state(ll_code, ll_state); - let diff = crate::fse::fse_encoder::transition_bits(ll_state, next.num_bits); + if extra_bits_total >= 31 { + unsafe { + writer.flush_bulk(); + } + } + + // Extras burst: ll (≤16) + ml (≤16) + of (≤ 31). Whether the + // accumulator was drained above or not, ll and ml fit: after a + // flush it holds ≤ 7 + 32 = 39, and without one the three + // extras together were under 31. + // + // SAFETY: `encode_literal_length` / `encode_match_len` + // bound `*_num_bits ≤ 16` and return a clean `*_add_bits` + // (low `num_bits` bits only). `encode_offset` bounds + // `of_num_bits ≤ ilog2(of) ≤ 31`, and the two conditionals + // (`>= 31` above, `> 56` below) are upstream's own accounting + // for when the offset field still fits beside what is already + // there. unsafe { - writer.write_bits_64_no_check(diff as u64, next.num_bits as usize); + writer.write_bits_64_no_check(ll_add_bits as u64, ll_num_bits); + writer.write_bits_64_no_check(ml_add_bits as u64, ml_num_bits); } - #[cfg(debug_assertions)] - { - state_diff_bits += next.num_bits as usize; + // Upstream `zstd_compress_sequences.c:355`: with the + // accumulator drained before the diffs, 7 leftover plus all + // three extra fields must stay under 64, so the offset needs + // its own container only past 56 bits of extras. + if extra_bits_total > 56 { + unsafe { + writer.flush_bulk(); + } } - ll_state = next.index; - } - // The three state diffs and the three extra-bit fields share one - // 64-bit accumulator, and a flush here is only needed when the - // extras that follow would not fit beside them. Upstream asks - // exactly that question (`zstd_compress_sequences.c:350`): - // `ofBits + mlBits + llBits >= 64 - 7 - (LLFSELog + MLFSELog + - // OffFSELog)`, which with our accumulator logs (9 + 9 + 8 = 26) is - // 31. Below it the accumulator holds at most 7 leftover + 26 diff - // bits + 30 extra bits = 63, so nothing has to leave yet. Flushing - // unconditionally instead cost a second store, length commit and - // shift on every sequence, and a level-1 block has hundreds of - // thousands of them. - let extra_bits_total = of_num_bits + ml_num_bits + ll_num_bits; - // The thresholds are arithmetic on widths, so they are only right - // while the widths are what they were derived from. Pinned here, - // where the arithmetic happens, so a widened encoder fails at its - // cause rather than as an accumulator overflow further down or, in - // a release build, as a corrupted stream with nothing to point at. - #[cfg(debug_assertions)] - { - debug_assert!( - state_diff_bits <= 26, - "state diffs took {state_diff_bits} bits; the thresholds below are \ - derived from a 26-bit ceiling (LLFSELog 9 + MLFSELog 9 + OffFSELog 8)", - ); - debug_assert!( - ll_num_bits <= 16 && ml_num_bits <= 16 && of_num_bits <= 31, - "extra-bit widths ll={ll_num_bits} ml={ml_num_bits} of={of_num_bits} \ - exceed the 16 / 16 / 31 the thresholds are derived from", - ); - // The bound the branch itself rests on: without a flush, the - // leftover, the diffs and all three extra fields have to fit. - debug_assert!( - extra_bits_total >= 31 || 7 + state_diff_bits + extra_bits_total <= 64, - "no flush at {extra_bits_total} extra bits, but 7 leftover + \ - {state_diff_bits} diff bits + {extra_bits_total} would overflow the \ - 64-bit accumulator", - ); - } - if extra_bits_total >= 31 { unsafe { + writer.write_bits_64_no_check(of_add_bits as u64, of_num_bits); writer.flush_bulk(); } } + } + if let Some(table) = ml_table { + writer.write_bits(ml_state as u64, table.table_size.ilog2() as usize); + } + if let Some(table) = of_table { + writer.write_bits(of_state as u64, table.table_size.ilog2() as usize); + } + if let Some(table) = ll_table { + writer.write_bits(ll_state as u64, table.table_size.ilog2() as usize); + } - // Extras burst: ll (≤16) + ml (≤16) + of (≤ 31). Whether the - // accumulator was drained above or not, ll and ml fit: after a - // flush it holds ≤ 7 + 32 = 39, and without one the three extras - // together were under 31. - // - // SAFETY: `encode_literal_length` / `encode_match_len` - // bound `*_num_bits ≤ 16` and return a clean `*_add_bits` - // (low `num_bits` bits only). `encode_offset` bounds - // `of_num_bits ≤ ilog2(of) ≤ 31`, and the two conditionals - // (`>= 31` above, `> 56` below) are upstream's own accounting for - // when the offset field still fits beside what is already there. - unsafe { - writer.write_bits_64_no_check(ll_add_bits as u64, ll_num_bits); - writer.write_bits_64_no_check(ml_add_bits as u64, ml_num_bits); - } - // Upstream `zstd_compress_sequences.c:355`: with the accumulator - // drained before the diffs, 7 leftover plus all three extra fields - // must stay under 64, so the offset needs its own container only - // past 56 bits of extras. - if extra_bits_total > 56 { - unsafe { - writer.flush_bulk(); - } - } - unsafe { - writer.write_bits_64_no_check(of_add_bits as u64, of_num_bits); - writer.flush_bulk(); - } + let bits_to_fill = writer.misaligned(); + if bits_to_fill == 0 { + writer.write_bits(1u32, 8); + } else { + writer.write_bits(1u32, bits_to_fill); } - } - if let Some(table) = ml_table { - writer.write_bits(ml_state as u64, table.table_size.ilog2() as usize); - } - if let Some(table) = of_table { - writer.write_bits(of_state as u64, table.table_size.ilog2() as usize); - } - if let Some(table) = ll_table { - writer.write_bits(ll_state as u64, table.table_size.ilog2() as usize); - } + }}; +} - let bits_to_fill = writer.misaligned(); - if bits_to_fill == 0 { - writer.write_bits(1u32, 8); - } else { - writer.write_bits(1u32, bits_to_fill); +/// Write the sequence section's bitstream for `sequences` under the three +/// table `modes`, in LL / ML / OF order. +/// +/// `kernel` is the tier resolved when the compressor was built. Where it has +/// BMI2 the loop runs under it, as upstream's does (`ZSTD_encodeSequences_bmi2`): +/// every field is written at a width known only at run time, and BMI2's +/// `shlx` / `shrx` / `bzhi` take that width in any register, where the base +/// x86 shifts take it only in `CL` and stall on the flags they leave. The bytes +/// written are the same on every tier. +fn encode_sequences( + kernel: FastpathKernel, + sequences: &[RawSequence], + // One packed [`SequenceCode`] per sequence, in the same order, from the + // pass that derived the offset codes. + codes: &[u32], + writer: &mut BitWriter<&mut Vec>, + modes: [&FseTableMode<'_>; 3], + // The three predefined tables, in the same order as `modes`. Passed rather + // than read off `FseTables`, whose accessors borrow the whole struct while + // the caller is holding its slots. + defaults: [&FSETable; 3], +) { + match kernel { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" + ))] + // SAFETY: the tier is only selected after confirming BMI2 (and AVX2). + FastpathKernel::Avx2Bmi2 => unsafe { + encode_sequences_bmi2(sequences, codes, writer, modes, defaults) + }, + _ => encode_sequences_scalar(sequences, codes, writer, modes, defaults), } } +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" +))] +#[target_feature(enable = "bmi2")] +unsafe fn encode_sequences_bmi2( + sequences: &[RawSequence], + codes: &[u32], + writer: &mut BitWriter<&mut Vec>, + modes: [&FseTableMode<'_>; 3], + defaults: [&FSETable; 3], +) { + encode_sequences_body!(sequences, codes, writer, modes, defaults) +} + +fn encode_sequences_scalar( + sequences: &[RawSequence], + codes: &[u32], + writer: &mut BitWriter<&mut Vec>, + modes: [&FseTableMode<'_>; 3], + defaults: [&FSETable; 3], +) { + encode_sequences_body!(sequences, codes, writer, modes, defaults) +} + fn encode_seqnum(seqnum: usize, writer: &mut BitWriter>>) { const UPPER_LIMIT: usize = 0xFFFF + 0x7F00; match seqnum { From 66d94d90c4c957f7cee1a34c625a357c7e6f91dd Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 24 Oct 2026 19:07:31 +0300 Subject: [PATCH 02/15] perf(encode): hold the sequence bitstream in locals across its loop --- zstd/src/bit_io/bit_writer.rs | 265 ++++++++++++------------- zstd/src/encoding/blocks/compressed.rs | 56 +++--- 2 files changed, 153 insertions(+), 168 deletions(-) diff --git a/zstd/src/bit_io/bit_writer.rs b/zstd/src/bit_io/bit_writer.rs index 5e4d3244..a8d03489 100644 --- a/zstd/src/bit_io/bit_writer.rs +++ b/zstd/src/bit_io/bit_writer.rs @@ -115,143 +115,36 @@ impl>> BitWriter { self.bit_idx += data.len() * 8; } - /// Pre-reserve additional capacity in the output buffer so the - /// upstream zstd-faithful FSE fast path can do `flush_bulk` writes without - /// triggering `Vec::extend_from_slice`'s grow-on-realloc branch. - /// `additional` is the number of bytes the upcoming bursts will - /// emit (caller's bit budget / 8, rounded up). - #[inline] - pub fn reserve_output(&mut self, additional: usize) { - self.output.as_mut().reserve(additional); - } - - /// Upstream zstd `BIT_addBitsFast` (`bitstream.h:193-200`): always accumulate - /// `bits` into the bottom of `partial`, no overflow check. - /// - /// # Safety - /// - /// Caller MUST guarantee BOTH preconditions BEFORE calling: + /// Start a [`BitBurst`] that commits at most `max_bytes` bytes, counting the + /// up to 7 whole bytes it drains from the accumulator here so that it starts + /// with under 8 bits held. /// - /// 1. `num_bits + bits_in_partial <= 64` AND **not both 64**, i.e. - /// if `bits_in_partial == 64` then `num_bits` MUST be 0 — and - /// the explicit `num_bits == 0` early-return below handles that - /// case without ever evaluating `bits << bits_in_partial`. The - /// point of the precondition: any other combination would either - /// shift-by-64 (undefined for `u64 << 64` in Rust) or overflow - /// `bits_in_partial` past 64. - /// 2. `num_bits == 64 || bits >> num_bits == 0` — value must be - /// clean of high junk past `num_bits`. Dirty high bits leak - /// into the packed stream at the next OR. - /// - /// The `debug_assert!`s below catch both in test/debug builds but - /// do NOT survive `cargo build --release`, hence the `unsafe` - /// signature. The function does not perform any memory-unsafe - /// operation, but a violation produces a silently-corrupted - /// output stream that decoders cannot recover — equivalent in - /// blast radius to undefined behaviour for the consumer. - /// - /// Used by the upstream zstd-faithful FSE sequence encoder which knows its - /// per-sequence bit budget at compile time and inserts explicit - /// [`Self::flush_bulk`] calls between bursts — saving the - /// per-call branch + spill that `write_bits_64`'s overflow check - /// pays. - #[inline(always)] - pub unsafe fn write_bits_64_no_check(&mut self, bits: u64, num_bits: usize) { - debug_assert!( - num_bits + self.bits_in_partial <= 64, - "write_bits_64_no_check would overflow partial: would push to {} bits", - num_bits + self.bits_in_partial, - ); - debug_assert!( - self.bits_in_partial < 64 || num_bits == 0, - "write_bits_64_no_check called with full accumulator and num_bits>0; \ - caller must flush_bulk before adding more bits", - ); - debug_assert!( - num_bits == 64 || bits >> num_bits == 0, - "value has dirty high bits beyond num_bits={num_bits}", - ); - // Masked so a full accumulator cannot make this a shift by 64, which is - // undefined for `u64`. It is not a guard costing anything: x86 and - // AArch64 shift instructions mask the count themselves, so the mask - // disappears. It replaces a `num_bits == 0` early return, which was a - // branch on every call, and there are six a sequence. Upstream needs - // neither, because its `BIT_addBitsFast` keeps `bitPos` strictly under - // 64 and can shift unconditionally. - // - // Correct in the one case the mask changes: the accumulator is full - // only where the caller passes `num_bits == 0`, and the second - // precondition then forces `bits == 0`, so the OR contributes nothing - // whatever the shift count. - self.partial |= bits << (self.bits_in_partial & 63); - self.bits_in_partial += num_bits; - } - - /// Upstream zstd `BIT_flushBitsFast` (`bitstream.h:202-214`): write the - /// full 8 bytes of `partial` directly to the output buffer via a - /// single `MEM_writeLEST` (unaligned 8-byte LE store), then - /// advance the Vec's `len` by only `nb_bytes` so the scratch - /// bytes past the commit point get overwritten by the next - /// flush. No overflow check — caller must have pre-reserved at - /// least 8 bytes of spare capacity via [`Self::reserve_output`]. - /// - /// `extend_from_slice` for tiny (0..=7-byte) tails was previously - /// hot — its per-call capacity check + memcpy dispatch cost - /// dwarfs the actual byte write at FSE flush cadence (~54K - /// flushes per compress on the L1 fast path). The direct - /// unaligned store path matches upstream zstd's hot loop cycle-for-cycle. - /// - /// # Safety - /// - /// Caller MUST guarantee that `output.capacity() >= output.len() + - /// 8` BEFORE calling this method (typically via a prior - /// [`Self::reserve_output`]). Violating the precondition produces a - /// 8-byte out-of-bounds write to whatever memory follows the - /// `Vec`'s allocation — undefined behaviour. The accompanying - /// `debug_assert!` catches misuse in test/debug builds but does - /// NOT survive `cargo build --release`, hence the `unsafe` - /// signature. - #[inline(always)] - pub unsafe fn flush_bulk(&mut self) { - let nb_bytes = self.bits_in_partial >> 3; - let bytes = self.partial.to_le_bytes(); + /// For a loop that knows its bit budget per step and flushes between + /// steps itself, which is how upstream writes the sequence section. + pub(crate) fn burst(&mut self, max_bytes: usize) -> BitBurst<'_, V> { + // Room for every flush of the burst: each writes 8 bytes and keeps at + // most 7 of them, so `max_bytes` committed plus 8 past the last. + let room = max_bytes + .checked_add(8) + .expect("a burst larger than the address space"); let output = self.output.as_mut(); - // `bit_idx` counts the bits already committed to `output`, and every - // path that appends bytes advances both, so it is the buffer's length - // in bits at all times. Taking the length from it keeps this off the - // `Vec` header, which is otherwise loaded on every flush, and this runs - // two to three times per encoded sequence. - let len = self.bit_idx >> 3; - debug_assert_eq!( - len, - output.len(), - "bit_idx must track the committed output length", - ); - debug_assert!( - output.capacity() >= len + 8, - "flush_bulk requires 8 bytes of spare capacity; caller forgot reserve_output", - ); - // SAFETY: the function-level Safety contract requires - // `output.capacity() >= len + 8`. We write 8 bytes starting at - // `len`, then commit only `nb_bytes` — the remaining `8 - - // nb_bytes` bytes stay within capacity but past `len`, and - // the next flush_bulk overwrites them. - unsafe { - let dst = output.as_mut_ptr().add(len); - core::ptr::copy_nonoverlapping(bytes.as_ptr(), dst, 8); - output.set_len(len + nb_bytes); - } - // `nb_bytes == 8` means the accumulator was full (64 bits). A - // raw `partial >>= 64` is UB on `u64`, so we zero explicitly. - // Upstream zstd's `BIT_flushBitsFast` writes a fresh 8 bytes the next - // round so the post-flush state must be clean either way. - if nb_bytes == 8 { - self.partial = 0; - } else { - self.partial >>= nb_bytes * 8; - } - self.bits_in_partial &= 7; - self.bit_idx += nb_bytes * 8; + output.reserve(room); + let len = output.len(); + debug_assert_eq!(len * 8, self.bit_idx, "bit_idx tracks the output length"); + let base = output.as_mut_ptr(); + let mut burst = BitBurst { + partial: self.partial, + bits_in_partial: self.bits_in_partial, + // SAFETY: `len <= capacity`. + cursor: unsafe { base.add(len) }, + // SAFETY: `reserve(room)` above made `len + room <= capacity`. + end: unsafe { base.add(len + room) }, + writer: self, + }; + // SAFETY: the accumulator holds under 64 bits outside a burst + // (`write_bits_64` drains it at 64), and the reserve covers 8 bytes. + unsafe { burst.flush() }; + burst } /// Bridge for the upstream zstd-faithful Huffman encoder (`HufCStream`) so @@ -452,5 +345,107 @@ impl>> BitWriter { } } +/// A run of unchecked writes into a [`BitWriter`] with the accumulator and the +/// output cursor held in locals, upstream's `BIT_CStream_t` (`bitstream.h`). +/// +/// Through the writer, every add stored the accumulator back and every flush +/// reloaded the buffer pointer from the `Vec` and stored its length: the byte +/// store goes through a `u8` pointer, which may alias the writer and the `Vec` +/// header, so neither could stay in a register across it. The burst holds no +/// such state behind a reference and commits the length once, in +/// [`Self::finish`]. +#[must_use = "a burst's bits reach the writer only through `finish`"] +pub(crate) struct BitBurst<'a, V: AsMut>> { + writer: &'a mut BitWriter, + partial: u64, + bits_in_partial: usize, + /// Next byte to write. + cursor: *mut u8, + /// End of the room [`BitWriter::burst`] reserved; every flush stays + /// 8 bytes short of it. + end: *mut u8, +} + +impl>> BitBurst<'_, V> { + /// Upstream zstd `BIT_addBitsFast` (`bitstream.h:193-200`): accumulate the + /// low `num_bits` of `bits` above what is held, with no overflow check. + /// + /// # Safety + /// + /// The accumulator must have room: `bits_in_partial + num_bits <= 64`, and + /// `bits` must be clean above `num_bits` (`num_bits == 64 || bits >> + /// num_bits == 0`). Neither violation touches memory, but either corrupts + /// the stream with nothing to detect it, so the caller proves both from + /// its bit budget. Debug builds assert them. + #[inline(always)] + pub(crate) unsafe fn add(&mut self, bits: u64, num_bits: usize) { + debug_assert!( + self.bits_in_partial + num_bits <= 64, + "{num_bits} bits do not fit beside the {} held", + self.bits_in_partial, + ); + debug_assert!( + num_bits == 64 || bits >> num_bits == 0, + "value has dirty high bits beyond num_bits={num_bits}", + ); + // Masked so a full accumulator cannot make this a shift by 64, which is + // undefined for `u64`. x86 and AArch64 shifts mask the count + // themselves, so the mask costs nothing, where a `num_bits == 0` early + // return was a branch on every add. The accumulator is full only when + // `num_bits == 0`, and the second precondition then makes `bits == 0`, + // so the OR adds nothing whatever the count. + self.partial |= bits << (self.bits_in_partial & 63); + self.bits_in_partial += num_bits; + } + + /// Upstream zstd `BIT_flushBitsFast` (`bitstream.h:202-214`): store all + /// 8 bytes of the accumulator at the cursor (one unaligned store) and + /// advance it by the whole bytes only; the rest is rewritten by the next + /// flush. + /// + /// # Safety + /// + /// The burst must still be within the `max_bytes` it was started with, so + /// that the 8-byte store lands inside the reserved room. Debug builds + /// assert it. + #[inline(always)] + pub(crate) unsafe fn flush(&mut self) { + let nb_bytes = self.bits_in_partial >> 3; + debug_assert!( + self.cursor.wrapping_add(8) <= self.end, + "a burst flushed past the room it reserved", + ); + // SAFETY: the cursor is 8 bytes short of the reserved end (above). + unsafe { + core::ptr::copy_nonoverlapping(self.partial.to_le_bytes().as_ptr(), self.cursor, 8); + self.cursor = self.cursor.add(nb_bytes); + } + // A full accumulator (64 bits) leaves nothing: shifting a `u64` by 64 + // is undefined, so it is zeroed instead. + self.partial = if nb_bytes == 8 { + 0 + } else { + self.partial >> (nb_bytes * 8) + }; + self.bits_in_partial &= 7; + } + + /// Commit the bytes written and hand the bits still held back to the + /// writer. + pub(crate) fn finish(self) { + let writer = self.writer; + let output = writer.output.as_mut(); + // SAFETY: the cursor started inside this allocation and every flush + // kept it within the reserved room, and the writer is borrowed for the + // whole burst, so the buffer was never reallocated. + let len = unsafe { self.cursor.offset_from(output.as_mut_ptr()) } as usize; + // SAFETY: `[old len, len)` was written by the flushes, within capacity. + unsafe { output.set_len(len) }; + writer.bit_idx = len * 8; + writer.partial = self.partial; + writer.bits_in_partial = self.bits_in_partial; + } +} + #[cfg(test)] mod tests; diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index c0d8c4d1..c8745226 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -2815,8 +2815,9 @@ macro_rules! encode_sequences_body { writer.write_bits(of_add_bits, of_num_bits); // Upstream zstd-faithful sequence loop: write state diffs + extras via - // unchecked fast-path adds with explicit `flush_bulk` calls at - // safe burst boundaries. Per-sequence bit budget: + // unchecked adds with explicit flushes at safe boundaries, through a + // burst that keeps the accumulator and the cursor in registers. + // Per-sequence bit budget: // state diffs: of (<=8) + ml (<=9) + ll (<=9) = 26 bits. // extras: ll (<=16) + ml (<=16) + of (<=31). // @@ -2826,22 +2827,12 @@ macro_rules! encode_sequences_body { // (`zstd_compress_sequences.c:303-367`), and the one at the end of the // body is what leaves the accumulator with under a byte for the next // round. - // - // Pre-reserve output capacity for the worst-case sequence section - // size (~10 bytes/sequence + 32 byte slack) so the per-flush - // `extend_from_slice` never triggers a Vec realloc. if sequences.len() > 1 { - writer.reserve_output(sequences.len() * 12 + 64); - // Pre-loop flush: the safe `write_bits` calls above for the - // final sequence's add_bits leave `bits_in_partial` in - // 0..=63. Before the first unchecked-add burst we drain to - // < 8 leftover so the per-burst budget math (state diffs ≤ - // 30 + leftover ≤ 8 = 38 < 64) holds invariantly. - // SAFETY: `reserve_output` above guarantees capacity ≥ - // current_len + sequences.len() * 12 + 64 ≥ current_len + 8. - unsafe { - writer.flush_bulk(); - } + // At most 63 bits a sequence leave as whole bytes, under 8 bytes; + // 12 a sequence and 64 over covers that and the up to 7 bytes the + // burst drains from the init-state writes above as it starts, + // which also leaves it holding under 8 bits. + let mut burst = writer.burst(sequences.len() * 12 + 64); // Walked as a slice rather than by index: the same order, without // the bounds check and the index arithmetic that `sequences[i]` // pays on every sequence. The last one is coded through the FSE @@ -2863,18 +2854,16 @@ macro_rules! encode_sequences_body { // acc_log ≤ 9 ll/ml + acc_log ≤ 8 of) + ≤ 7 leftover from // prior flush = ≤ 37 bits total — well under 64. // - // SAFETY (for every `write_bits_64_no_check` below): - // - the prior `flush_bulk` left `bits_in_partial ≤ 7`; + // SAFETY (for every `burst.add` below): + // - the prior flush left `bits_in_partial ≤ 7`; // - each FSE state diff has `next.num_bits ≤ acc_log ≤ 10`; // three diffs back-to-back add ≤ 30 bits → total ≤ 37, // well below the 64-bit accumulator cap. // - `diff = state.index - next.baseline` cannot exceed // `(1 << num_bits) - 1`, so `diff >> num_bits == 0`. - // `reserve_output(sequences.len() * 12 + 64)` above - // pre-allocated enough spare capacity to cover every - // per-sequence flush in this loop (≤ 16 bytes per - // sequence, plus the 32-byte slack on top of the 64-byte - // header reserve). + // And for every `burst.flush`: the burst was started with room + // for 12 bytes a sequence, more than one sequence's flushes + // commit. // // What the three diffs actually put in the accumulator, tallied // so the budget below is checked against the widths the FSE @@ -2886,7 +2875,7 @@ macro_rules! encode_sequences_body { let next = table.next_state(of_code, of_state); let diff = crate::fse::fse_encoder::transition_bits(of_state, next.num_bits); unsafe { - writer.write_bits_64_no_check(diff as u64, next.num_bits as usize); + burst.add(diff as u64, next.num_bits as usize); } #[cfg(debug_assertions)] { @@ -2898,7 +2887,7 @@ macro_rules! encode_sequences_body { let next = table.next_state(ml_code, ml_state); let diff = crate::fse::fse_encoder::transition_bits(ml_state, next.num_bits); unsafe { - writer.write_bits_64_no_check(diff as u64, next.num_bits as usize); + burst.add(diff as u64, next.num_bits as usize); } #[cfg(debug_assertions)] { @@ -2910,7 +2899,7 @@ macro_rules! encode_sequences_body { let next = table.next_state(ll_code, ll_state); let diff = crate::fse::fse_encoder::transition_bits(ll_state, next.num_bits); unsafe { - writer.write_bits_64_no_check(diff as u64, next.num_bits as usize); + burst.add(diff as u64, next.num_bits as usize); } #[cfg(debug_assertions)] { @@ -2960,7 +2949,7 @@ macro_rules! encode_sequences_body { } if extra_bits_total >= 31 { unsafe { - writer.flush_bulk(); + burst.flush(); } } @@ -2977,8 +2966,8 @@ macro_rules! encode_sequences_body { // for when the offset field still fits beside what is already // there. unsafe { - writer.write_bits_64_no_check(ll_add_bits as u64, ll_num_bits); - writer.write_bits_64_no_check(ml_add_bits as u64, ml_num_bits); + burst.add(ll_add_bits as u64, ll_num_bits); + burst.add(ml_add_bits as u64, ml_num_bits); } // Upstream `zstd_compress_sequences.c:355`: with the // accumulator drained before the diffs, 7 leftover plus all @@ -2986,14 +2975,15 @@ macro_rules! encode_sequences_body { // its own container only past 56 bits of extras. if extra_bits_total > 56 { unsafe { - writer.flush_bulk(); + burst.flush(); } } unsafe { - writer.write_bits_64_no_check(of_add_bits as u64, of_num_bits); - writer.flush_bulk(); + burst.add(of_add_bits as u64, of_num_bits); + burst.flush(); } } + burst.finish(); } if let Some(table) = ml_table { writer.write_bits(ml_state as u64, table.table_size.ilog2() as usize); From 8715657897bf7fc634ce0ee5953f7e51396471e1 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 24 Oct 2026 19:29:16 +0300 Subject: [PATCH 03/15] perf(encode): warm the pre-split window only for the sampling splitter --- zstd/src/encoding/frame_compressor.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index bb4c756c..f561c2dc 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -2235,16 +2235,19 @@ impl FrameCompressor { let savings = start as i64 - (out.len() - blocks_start) as i64; // Borrowed path only: warm the pre-split window before the // cache-cold strided fingerprint read. Gated to exactly the - // conditions under which `optimal_block_size` reads `block` - // (a pre-split level, a full 128 KiB block remaining, the - // block-size cap admits a full block, and `savings >= 3` so the - // splitter actually runs) — so non-pre-split levels, the first - // block, and the trailing partial block pay nothing. See - // `warm_presplit_window`. + // conditions under which `optimal_block_size` reads the whole + // window (a sampling pre-split level, a full 128 KiB block + // remaining, the block-size cap admits a full block, and + // `savings >= 3` so the splitter actually runs) — so other + // levels, the first block, and the trailing partial block pay + // nothing. Level 0 reads three 512-byte segments + // (`ZSTD_splitBlock_fromBorders`, `zstd_preSplit.c:198-226`), not + // the window, so warming 128 KiB for it was 2048 loads a block + // for 1.5 KiB read. See `warm_presplit_window`. if savings >= 3 && input.len() - start >= MAX_BLOCK_SIZE as usize && block_capacity >= MAX_BLOCK_SIZE as usize - && pre_split.is_some() + && pre_split.is_some_and(|split_level| split_level != 0) { warm_presplit_window(&input[start..start + MAX_BLOCK_SIZE as usize]); } From 788433c4b209b2456594692f929bd1f89a36332b Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 24 Oct 2026 20:11:29 +0300 Subject: [PATCH 04/15] refactor(encode): choose the block's kernel tier once - with_encode_tier! is the one place a tier tag becomes a type; the block entry calls it and everything below is generic over the tier - the literal gather, the Huffman burst and the sequence-section loop take their kernel from K::TIER, which folds at monomorphisation; the Huffman encoder's own per-stream BMI2 detect is gone - every tier the CPU runs is checked to write the scalar tier's bytes --- .../burst_gate_tests.rs | 5 +- zstd/src/encoding/blocks/compressed.rs | 91 +++++++---- .../blocks/compressed/literal_runs.rs | 12 +- zstd/src/encoding/blocks/compressed/tests.rs | 141 +++++++++++++---- zstd/src/encoding/fastpath/mod.rs | 147 ++++++++++++++++++ zstd/src/huff0/huf_cstream.rs | 58 +++---- zstd/src/huff0/huf_cstream/tests.rs | 2 +- zstd/src/huff0/huff0_encoder.rs | 29 ++-- 8 files changed, 369 insertions(+), 116 deletions(-) diff --git a/zstd/src/decoding/literals_section_decoder/burst_gate_tests.rs b/zstd/src/decoding/literals_section_decoder/burst_gate_tests.rs index dc4d0d63..9bd7e6e2 100644 --- a/zstd/src/decoding/literals_section_decoder/burst_gate_tests.rs +++ b/zstd/src/decoding/literals_section_decoder/burst_gate_tests.rs @@ -49,7 +49,10 @@ fn build_huf4x_block(data: &[u8]) -> (LiteralsSection, Vec) { { let mut writer = BitWriter::from(&mut source); let mut encoder = HuffmanEncoder::new(&table, &mut writer); - encoder.encode4x(data, true); + crate::encoding::fastpath::with_encode_tier!( + crate::encoding::fastpath::select_kernel(), + K => encoder.encode4x::(data, true) + ); writer.flush(); } let section = LiteralsSection { diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index c8745226..141f96eb 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -6,7 +6,7 @@ use crate::{ bit_io::BitWriter, blocks::block::BlockType, encoding::block_header::BlockHeader, - encoding::fastpath::FastpathKernel, + encoding::fastpath::{EncodeTier, with_encode_tier}, encoding::frame_compressor::{CompressState, FseTables, PreviousFseTable, SharedFseTable}, encoding::workspace::{RegionVec, Workspace, region_bytes}, encoding::{Matcher, Sequence}, @@ -434,10 +434,18 @@ impl Matcher for EntropyOnlyMatcher { pub(crate) fn compress_block( state: &mut CompressState, output: &mut Vec, +) -> BlockTables { + with_encode_tier!(state.copy_kernel, K => compress_block_with::(state, output)) +} + +/// [`compress_block`] compiled for the tier `K`. +fn compress_block_with( + state: &mut CompressState, + output: &mut Vec, ) -> BlockTables { let mut scratch = core::mem::take(&mut state.block_scratch); - collect_block_parts(state, &mut scratch.parts); - let tables = encode_block_parts( + collect_block_parts::(state, &mut scratch.parts); + let tables = encode_block_parts::( state, &scratch.parts.literals, &mut scratch.parts.sequences, @@ -449,6 +457,28 @@ pub(crate) fn compress_block( } pub(crate) fn compress_block_with_post_split( + state: &mut CompressState, + last_block: bool, + output: &mut Vec, + #[cfg(feature = "lsm")] block_decompressed_sizes: Option<&mut Vec>, + #[cfg(all(feature = "lsm", feature = "hash"))] block_checksums: Option<&mut Vec>, +) { + with_encode_tier!( + state.copy_kernel, + K => compress_block_with_post_split_with::( + state, + last_block, + output, + #[cfg(feature = "lsm")] + block_decompressed_sizes, + #[cfg(all(feature = "lsm", feature = "hash"))] + block_checksums, + ) + ) +} + +/// [`compress_block_with_post_split`] compiled for the tier `K`. +fn compress_block_with_post_split_with( state: &mut CompressState, last_block: bool, output: &mut Vec, @@ -456,7 +486,7 @@ pub(crate) fn compress_block_with_post_split( #[cfg(all(feature = "lsm", feature = "hash"))] mut block_checksums: Option<&mut Vec>, ) { let mut scratch = core::mem::take(&mut state.block_scratch); - collect_block_parts(state, &mut scratch.parts); + collect_block_parts::(state, &mut scratch.parts); if scratch.parts.sequences.len() <= 4 { let source_len = state.matcher.get_last_space().len(); #[cfg(feature = "lsm")] @@ -481,7 +511,7 @@ pub(crate) fn compress_block_with_post_split( compressed: &mut scratch.compressed, codes: &mut scratch.sequence_codes, }; - let emitted_raw = emit_single_sequence_block( + let emitted_raw = emit_single_sequence_block::( state, last_block, source_len, @@ -626,7 +656,7 @@ pub(crate) fn compress_block_with_post_split( compressed: &mut scratch.compressed, codes: &mut scratch.sequence_codes, }; - let emitted_raw = emit_single_sequence_block( + let emitted_raw = emit_single_sequence_block::( state, last_block && last_partition, src_size, @@ -654,7 +684,10 @@ pub(crate) fn compress_block_with_post_split( /// sequence (`ZSTD_storeSeq`); here the match loop calls the emit from a dozen /// sites, and a copy inlined at each of them costs the loop more in decode /// pressure than the calls it saves (+2.16% cycles at level 3, measured). -fn collect_block_parts(state: &mut CompressState, parts: &mut EncodedBlockParts) { +fn collect_block_parts( + state: &mut CompressState, + parts: &mut EncodedBlockParts, +) { parts.literals.clear(); parts.sequences.clear(); // Trailing literals, reported after the last sequence; the runs are read @@ -699,9 +732,7 @@ fn collect_block_parts(state: &mut CompressState, parts: &mut Enc block_start_hist, ); } - let kernel = state.copy_kernel; - literal_runs::gather_literals( - kernel, + literal_runs::gather_literals::( state.matcher.get_last_space(), &parts.sequences, tail, @@ -787,7 +818,7 @@ fn record_sequence( } } -fn encode_block_parts( +fn encode_block_parts( state: &mut CompressState, literals_vec: &[u8], raw_sequences: &mut [RawSequence], @@ -855,7 +886,7 @@ fn encode_block_parts( rle_literals(literals_vec, &mut writer); HuffOutcome::Clear } else if literals_vec.len() >= min_lits { - match compress_literals( + match compress_literals::( literals_vec, state.last_huff_table.as_ref(), &mut writer, @@ -918,7 +949,6 @@ fn encode_block_parts( ) }); - let kernel = state.copy_kernel; // Destructured because the default-table accessors borrow the whole of // `FseTables`, which would collide with taking a `*_next` slot mutably. let FseTables { @@ -976,8 +1006,7 @@ fn encode_block_parts( encode_table(&of_mode, &mut writer); encode_table(&ml_mode, &mut writer); - encode_sequences( - kernel, + encode_sequences::( raw_sequences, codes, &mut writer, @@ -1746,7 +1775,7 @@ struct SingleSequenceEmitBuffers<'a> { codes: &'a mut RegionVec, } -fn emit_single_sequence_block( +fn emit_single_sequence_block( state: &mut CompressState, last_block: bool, source_len: usize, @@ -1758,7 +1787,7 @@ fn emit_single_sequence_block( // The tables are only read here and the block's decisions come back to be // committed, so writing the partition raw leaves nothing to restore. buffers.compressed.clear(); - let tables = encode_block_parts( + let tables = encode_block_parts::( state, literals, sequences, @@ -3007,14 +3036,12 @@ macro_rules! encode_sequences_body { /// Write the sequence section's bitstream for `sequences` under the three /// table `modes`, in LL / ML / OF order. /// -/// `kernel` is the tier resolved when the compressor was built. Where it has -/// BMI2 the loop runs under it, as upstream's does (`ZSTD_encodeSequences_bmi2`): -/// every field is written at a width known only at run time, and BMI2's -/// `shlx` / `shrx` / `bzhi` take that width in any register, where the base -/// x86 shifts take it only in `CL` and stall on the flags they leave. The bytes -/// written are the same on every tier. -fn encode_sequences( - kernel: FastpathKernel, +/// Where the tier `K` has BMI2 the loop runs under it, as upstream's does +/// (`ZSTD_encodeSequences_bmi2`): every field is written at a width known only +/// at run time, and BMI2's `shlx` / `shrx` / `bzhi` take that width in any +/// register, where the base x86 shifts take it only in `CL` and stall on the +/// flags they leave. The bytes written are the same on every tier. +fn encode_sequences( sequences: &[RawSequence], // One packed [`SequenceCode`] per sequence, in the same order, from the // pass that derived the offset codes. @@ -3026,13 +3053,13 @@ fn encode_sequences( // the caller is holding its slots. defaults: [&FSETable; 3], ) { - match kernel { + match K::TIER { #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-avx2" ))] // SAFETY: the tier is only selected after confirming BMI2 (and AVX2). - FastpathKernel::Avx2Bmi2 => unsafe { + crate::encoding::fastpath::FastpathKernel::Avx2Bmi2 => unsafe { encode_sequences_bmi2(sequences, codes, writer, modes, defaults) }, _ => encode_sequences_scalar(sequences, codes, writer, modes, defaults), @@ -3337,7 +3364,7 @@ fn rle_literals(literals: &[u8], writer: &mut BitWriter<&mut Vec>) { /// `compress_literals` byte-for-byte (same size_format ladder, same /// min_gain raw-fallback gate) so the wire output is identical to /// the size-comparison reuse path when both would pick reuse. -fn emit_reuse_literals( +fn emit_reuse_literals( literals: &[u8], last_table: &huff0_encoder::HuffmanTable, writer: &mut BitWriter<&mut Vec>, @@ -3364,7 +3391,7 @@ fn emit_reuse_literals( if size_format == 0 { encoder.encode(literals, false); } else { - encoder.encode4x(literals, false); + encoder.encode4x::(literals, false); } let encoded_len = (writer.index() - index_before) / 8; writer.change_bits(size_index, encoded_len as u64, size_bits); @@ -3434,7 +3461,7 @@ fn end_samples_look_flat(literals: &[u8], counts: &mut [usize; 256]) -> bool { head_largest + tail_largest <= ((2 * SUSPECT_SAMPLE_SIZE) >> 7) + 4 } -fn compress_literals( +fn compress_literals( literals: &[u8], last_table: Option<&huff0_encoder::HuffmanTable>, writer: &mut BitWriter<&mut Vec>, @@ -3459,7 +3486,7 @@ fn compress_literals( && let Some(prev) = last_table && prev.estimate_compressed_size(literals).is_some() { - return emit_reuse_literals(literals, prev, writer, reset_idx, strategy); + return emit_reuse_literals::(literals, prev, writer, reset_idx, strategy); } let mut counts = [0usize; 256]; @@ -3557,7 +3584,7 @@ fn compress_literals( if size_format == 0 { encoder.encode(literals, new_table) } else { - encoder.encode4x(literals, new_table) + encoder.encode4x::(literals, new_table) }; let encoded_len = (writer.index() - index_before) / 8; writer.change_bits(size_index, encoded_len as u64, size_bits); diff --git a/zstd/src/encoding/blocks/compressed/literal_runs.rs b/zstd/src/encoding/blocks/compressed/literal_runs.rs index f03c5070..19fa96a3 100644 --- a/zstd/src/encoding/blocks/compressed/literal_runs.rs +++ b/zstd/src/encoding/blocks/compressed/literal_runs.rs @@ -7,7 +7,7 @@ //! block, so no copy below the dispatch asks which kernel to use. use crate::decoding::simd_copy; -use crate::encoding::fastpath::FastpathKernel; +use crate::encoding::fastpath::{EncodeTier, FastpathKernel}; use crate::encoding::workspace::RegionVec; use super::RawSequence; @@ -18,16 +18,12 @@ use super::RawSequence; const LITERAL_INLINE_COPY_MAX: usize = 2048; /// Append the literal runs of `sequences`, then `tail` trailing literal bytes, -/// from `block` to `dst`. -/// -/// `kernel` is the tier resolved when the compressor was built; this is the -/// only place it is looked at, once per block. +/// from `block` to `dst`, with the copies of the tier `K`. /// /// # Panics /// If the sequences and the tail reach past `block`, which a matcher that /// reports its sequences correctly never produces. -pub(super) fn gather_literals( - kernel: FastpathKernel, +pub(super) fn gather_literals( block: &[u8], sequences: &[RawSequence], tail: usize, @@ -39,7 +35,7 @@ pub(super) fn gather_literals( if dst.capacity() - dst.len() < block.len() { crate::encoding::workspace::capacity_exhausted(); } - match kernel { + match K::TIER { #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-avx2" diff --git a/zstd/src/encoding/blocks/compressed/tests.rs b/zstd/src/encoding/blocks/compressed/tests.rs index 901e2562..64426599 100644 --- a/zstd/src/encoding/blocks/compressed/tests.rs +++ b/zstd/src/encoding/blocks/compressed/tests.rs @@ -424,12 +424,15 @@ fn estimator_literals_section_mirrors_emit_for_short_inputs() { let est = estimate_block_parts_size(&mut est_state, &literals, &[], &mut workspace); let mut emitted: Vec = Vec::new(); let mut codes_ws = Workspace::new(); - encode_block_parts( - &mut emit_state, - &literals, - &mut [], - &mut code_buffer(&mut codes_ws, 0), - &mut emitted, + crate::encoding::fastpath::with_encode_tier!( + emit_state.copy_kernel, + K => encode_block_parts::<_, K>( + &mut emit_state, + &literals, + &mut [], + &mut code_buffer(&mut codes_ws, 0), + &mut emitted, + ) ); assert_eq!( est, @@ -493,12 +496,15 @@ fn a_section_with_flat_ends_costs_what_the_emitter_writes_for_it() { let est = estimate_block_parts_size(&mut est_state, &literals, &[], &mut workspace); let mut emitted: Vec = Vec::new(); let mut codes_ws = Workspace::new(); - encode_block_parts( - &mut emit_state, - &literals, - &mut [], - &mut code_buffer(&mut codes_ws, 0), - &mut emitted, + crate::encoding::fastpath::with_encode_tier!( + emit_state.copy_kernel, + K => encode_block_parts::<_, K>( + &mut emit_state, + &literals, + &mut [], + &mut code_buffer(&mut codes_ws, 0), + &mut emitted, + ) ); assert_eq!( @@ -795,12 +801,15 @@ fn estimator_and_emitter_agree_on_a_block_with_sequences() { let mut to_emit = sequences.clone(); let mut codes_ws = Workspace::new(); let mut codes = code_buffer(&mut codes_ws, to_emit.len()); - encode_block_parts( - &mut emit_state, - &literals, - &mut to_emit, - &mut codes, - &mut emitted, + crate::encoding::fastpath::with_encode_tier!( + emit_state.copy_kernel, + K => encode_block_parts::<_, K>( + &mut emit_state, + &literals, + &mut to_emit, + &mut codes, + &mut emitted, + ) ); // Both sides must have advanced the repeat-offset history the same @@ -870,13 +879,16 @@ fn raw_partition_fallback_restores_repeat_offset_history() { compressed: &mut compressed_scratch, codes: &mut code_scratch, }; - let emitted_raw = emit_single_sequence_block( - &mut state, - true, - source.len(), - &[], - &mut sequences, - &mut emit_buffers, + let emitted_raw = crate::encoding::fastpath::with_encode_tier!( + state.copy_kernel, + K => emit_single_sequence_block::<_, K>( + &mut state, + true, + source.len(), + &[], + &mut sequences, + &mut emit_buffers, + ) ); if emitted_raw { output.extend_from_slice(&source); @@ -1206,8 +1218,7 @@ fn literals_buffer(capacity: usize) -> (Workspace, RegionVec) { #[should_panic(expected = "workspace buffer sized below what the frame put in it")] fn literals_past_the_buffer_stop_instead_of_overflowing() { let (_ws, mut literals) = literals_buffer(4); - super::literal_runs::gather_literals( - crate::encoding::fastpath::FastpathKernel::Scalar, + super::literal_runs::gather_literals::( &[0xA5; 8], &[], 8, @@ -1227,8 +1238,7 @@ fn sequences_past_the_block_are_refused() { ml: 60, off_base: 1, }]; - super::literal_runs::gather_literals( - crate::encoding::fastpath::FastpathKernel::Scalar, + super::literal_runs::gather_literals::( &[0x5A; 32], &sequences, 0, @@ -1306,11 +1316,82 @@ fn gathered_literals_are_the_runs_in_order() { expected.extend_from_slice(&block[pos..]); for kernel in runnable_fastpath_kernels() { let (_ws, mut literals) = literals_buffer(block_len); - super::literal_runs::gather_literals(kernel, &block, &sequences, 13, &mut literals); + crate::encoding::fastpath::with_encode_tier!( + kernel, + K => super::literal_runs::gather_literals::(&block, &sequences, 13, &mut literals) + ); assert_eq!(&literals[..], &expected[..], "{kernel:?}"); } } +/// Every tier this CPU can run writes the same block, byte for byte, as the +/// scalar tier: the tiers differ in the instructions of the Huffman burst and +/// the sequence-section loop (BMI2 against base shifts), never in their output. +/// The block carries a Huffman-coded literals section and thousands of +/// sequences, so both loops run on every tier. +#[test] +fn every_tier_writes_the_block_the_scalar_tier_writes() { + use super::{CompressedBlockScratch, EntropyOnlyMatcher, encode_block_parts}; + // Skewed literals, so the section is Huffman-coded rather than raw. + let literals: Vec = (0..24_000u32) + .map(|i| b"eeeettaoinshrdlucmfwyp"[(i.wrapping_mul(2654435761) >> 27) as usize % 22]) + .collect(); + // Lengths and offsets across their extra-bit ranges, so the field widths + // the loop shifts by vary from one sequence to the next. + let sequences: Vec = (0..4_000u32) + .map(|i| { + let mix = i.wrapping_mul(0x9E37_79B9); + RawSequence { + ll: (mix >> 7) % 70, + ml: 3 + (mix >> 13) % 300, + off_base: 1 + (mix >> 3) % 100_000, + } + }) + .collect(); + let encode = |kernel| { + let mut state = CompressState:: { + matcher: EntropyOnlyMatcher, + sequence_capture: None, + copy_kernel: kernel, + last_huff_table: None, + huff_table_spare: None, + huff_weights: Default::default(), + seen_content: Default::default(), + fse_tables: FseTables::new(), + block_scratch: CompressedBlockScratch::new(), + workspace: Workspace::new(), + offset_hist: [1, 4, 8], + strategy_tag: StrategyTag::Lazy, + pre_split: None, + huf_optimal_search: true, + literal_compression_disabled: false, + post_split_allowed: true, + }; + let mut emitted = Vec::new(); + let mut to_emit = sequences.clone(); + let mut codes_ws = Workspace::new(); + let mut codes = code_buffer(&mut codes_ws, to_emit.len()); + crate::encoding::fastpath::with_encode_tier!( + kernel, + K => encode_block_parts::<_, K>( + &mut state, + &literals, + &mut to_emit, + &mut codes, + &mut emitted, + ) + ); + emitted + }; + let scalar = encode(crate::encoding::fastpath::FastpathKernel::Scalar); + // The literals section is Huffman-coded (Literals_Block_Type 2 in the low + // two bits of its first byte, RFC 8878 3.1.1.3.1.1), so the burst ran. + assert_eq!(scalar[0] & 0b11, 2, "literals section is not Huffman-coded"); + for kernel in runnable_fastpath_kernels() { + assert_eq!(encode(kernel), scalar, "{kernel:?}"); + } +} + #[test] fn choose_table_without_previous_does_not_unwrap_none() { let only_zero_one_table = build_table_from_symbol_counts(&[1, 1], 5, false); diff --git a/zstd/src/encoding/fastpath/mod.rs b/zstd/src/encoding/fastpath/mod.rs index 3c76f398..3fab4f29 100644 --- a/zstd/src/encoding/fastpath/mod.rs +++ b/zstd/src/encoding/fastpath/mod.rs @@ -308,6 +308,153 @@ fn detect_kernel_uncached() -> FastpathKernel { FastpathKernel::Scalar } +/// A [`FastpathKernel`] tier as a type. The encoder chooses its tier once, in +/// [`with_encode_tier`], and everything below that choice is compiled per tier: +/// a leaf that runs a different kernel per tier matches on `K::TIER`, which +/// folds to the one kernel at monomorphisation, so no tier is asked at run time +/// below the choice. +pub(crate) trait EncodeTier { + /// The tier this type stands for. + const TIER: FastpathKernel; +} + +/// [`FastpathKernel::Scalar`] as a type. +pub(crate) struct ScalarTier; +impl EncodeTier for ScalarTier { + const TIER: FastpathKernel = FastpathKernel::Scalar; +} + +/// [`FastpathKernel::Neon`] as a type. +#[cfg(all( + target_arch = "aarch64", + target_endian = "little", + feature = "kernel-neon" +))] +pub(crate) struct NeonTier; +#[cfg(all( + target_arch = "aarch64", + target_endian = "little", + feature = "kernel-neon" +))] +impl EncodeTier for NeonTier { + const TIER: FastpathKernel = FastpathKernel::Neon; +} + +/// [`FastpathKernel::Sse2`] as a type. +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" +))] +pub(crate) struct Sse2Tier; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" +))] +impl EncodeTier for Sse2Tier { + const TIER: FastpathKernel = FastpathKernel::Sse2; +} + +/// [`FastpathKernel::Sse42`] as a type. +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" +))] +pub(crate) struct Sse42Tier; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" +))] +impl EncodeTier for Sse42Tier { + const TIER: FastpathKernel = FastpathKernel::Sse42; +} + +/// [`FastpathKernel::Avx2Bmi2`] as a type. +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" +))] +pub(crate) struct Avx2Bmi2Tier; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" +))] +impl EncodeTier for Avx2Bmi2Tier { + const TIER: FastpathKernel = FastpathKernel::Avx2Bmi2; +} + +/// [`FastpathKernel::Simd128`] as a type. +#[cfg(all( + target_arch = "wasm32", + target_feature = "simd128", + feature = "kernel-simd128" +))] +pub(crate) struct Simd128Tier; +#[cfg(all( + target_arch = "wasm32", + target_feature = "simd128", + feature = "kernel-simd128" +))] +impl EncodeTier for Simd128Tier { + const TIER: FastpathKernel = FastpathKernel::Simd128; +} + +/// The encoder's one choice of tier: evaluate `$body` with `$k` naming the +/// [`EncodeTier`] type of the tier `$tier` holds. +macro_rules! with_encode_tier { + ($tier:expr, $k:ident => $body:expr) => {{ + use $crate::encoding::fastpath as tiers; + match $tier { + tiers::FastpathKernel::Scalar => { + type $k = tiers::ScalarTier; + $body + } + #[cfg(all( + target_arch = "aarch64", + target_endian = "little", + feature = "kernel-neon" + ))] + tiers::FastpathKernel::Neon => { + type $k = tiers::NeonTier; + $body + } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" + ))] + tiers::FastpathKernel::Sse2 => { + type $k = tiers::Sse2Tier; + $body + } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" + ))] + tiers::FastpathKernel::Sse42 => { + type $k = tiers::Sse42Tier; + $body + } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" + ))] + tiers::FastpathKernel::Avx2Bmi2 => { + type $k = tiers::Avx2Bmi2Tier; + $body + } + #[cfg(all( + target_arch = "wasm32", + target_feature = "simd128", + feature = "kernel-simd128" + ))] + tiers::FastpathKernel::Simd128 => { + type $k = tiers::Simd128Tier; + $body + } + } + }}; +} +pub(crate) use with_encode_tier; + /// Public entry point for raw-pointer prefix-length scans (BT byte compare, /// repcode extend, etc.): resolves the tier via [`select_kernel`] on every /// call, so hot loops should cache the kernel and call diff --git a/zstd/src/huff0/huf_cstream.rs b/zstd/src/huff0/huf_cstream.rs index 199826ab..31585449 100644 --- a/zstd/src/huff0/huf_cstream.rs +++ b/zstd/src/huff0/huf_cstream.rs @@ -25,6 +25,8 @@ use alloc::vec::Vec; +use crate::encoding::fastpath::EncodeTier; + /// Upstream zstd `HUF_BITS_IN_CONTAINER = sizeof(size_t) * 8`. We hard-code 64 /// regardless of target pointer width. Upstream zstd's `MEM_32bits()` branch /// switches the container to `u32` on 32-bit hosts; this crate's CI @@ -111,24 +113,6 @@ pub(crate) struct HufCStream<'a> { overflow: bool, } -/// Whether to run the BMI2-targeted burst-encode kernel. BMI2 lets the codegen -/// use `shrx`/`shlx` for the bit-container's variable shifts (no CL dependency, -/// no flag stall) instead of `shr`/`shl`; the emitted instruction stream differs -/// but the logic — and therefore the compressed output — is identical. Detected -/// at the per-stream entry, never inside the symbol loop. -#[cfg(all(feature = "std", any(target_arch = "x86", target_arch = "x86_64")))] -#[inline] -fn huf_encode_use_bmi2() -> bool { - use crate::cpu_kernel::{CpuLevel, cpu_allows}; - cpu_allows(CpuLevel::Bmi2) && std::arch::is_x86_feature_detected!("bmi2") -} -#[cfg(all(not(feature = "std"), any(target_arch = "x86", target_arch = "x86_64")))] -#[inline] -fn huf_encode_use_bmi2() -> bool { - use crate::cpu_kernel::{CpuLevel, cpu_allows}; - cpu_allows(CpuLevel::Bmi2) && cfg!(target_feature = "bmi2") -} - /// One symbol into bit container `$bc` / bit position `$bp`. Mirrors /// `add_bits`; `$fast` (a `const`-valued bool) const-folds. Module-level so the /// burst-loop body macro can expand it inside both the scalar and the @@ -367,8 +351,13 @@ impl<'a> HufCStream<'a> { /// `2 * K_UNROLL`, (3) dual-container main loop processing /// `2 * K_UNROLL` symbols per iteration. Symbols consumed in reverse /// (`data[--n]`). + /// + /// The kernel is the encoder tier `K`'s: BMI2 lets the container's variable + /// shifts be `shrx`/`shlx` (no `CL` dependency, no flag stall) instead of + /// `shr`/`shl`. The instructions differ, the output does not. #[inline] pub(crate) fn encode_unrolled< + K: EncodeTier, const K_UNROLL: usize, const K_FAST_FLUSH: bool, const K_LAST_FAST: bool, @@ -377,20 +366,19 @@ impl<'a> HufCStream<'a> { table: &[u64], data: &[u8], ) { - // Pick the burst kernel ONCE per stream, never inside the symbol loop. - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - if huf_encode_use_bmi2() { - // SAFETY: `huf_encode_use_bmi2()` returned true, so the running - // CPU has BMI2; the body adds no unsafe beyond the 8-byte store - // already justified in `huf_flush!`. - unsafe { - self.encode_unrolled_bmi2::(table, data); - } - return; - } + match K::TIER { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" + ))] + // SAFETY: the tier is only selected after confirming BMI2 (and + // AVX2); the body adds no unsafe beyond the 8-byte store already + // justified in `huf_flush!`. + crate::encoding::fastpath::FastpathKernel::Avx2Bmi2 => unsafe { + self.encode_unrolled_bmi2::(table, data) + }, + _ => self.encode_unrolled_scalar::(table, data), } - self.encode_unrolled_scalar::(table, data); } /// Portable burst-encode kernel (no target feature). Bit-identical output to @@ -412,8 +400,12 @@ impl<'a> HufCStream<'a> { /// the container's variable shifts lower to `shrx`/`shlx`. /// /// # Safety - /// The caller must have verified BMI2 is available (`huf_encode_use_bmi2()`). - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + /// The caller must have verified BMI2 is available: the encoder tier + /// running it is [`crate::encoding::fastpath::FastpathKernel::Avx2Bmi2`]. + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" + ))] #[target_feature(enable = "bmi2")] unsafe fn encode_unrolled_bmi2< const K_UNROLL: usize, diff --git a/zstd/src/huff0/huf_cstream/tests.rs b/zstd/src/huff0/huf_cstream/tests.rs index e180a72d..1458c19d 100644 --- a/zstd/src/huff0/huf_cstream/tests.rs +++ b/zstd/src/huff0/huf_cstream/tests.rs @@ -66,7 +66,7 @@ fn encode_unrolled_dual_container_size_is_deterministic() { // Every symbol maps to the same 4-bit code (value 0b1010). let table = [pack_huf_celt(0b1010, 4); 256]; let data = [0u8; 16]; - s.encode_unrolled::<4, false, false>(&table, &data); + s.encode_unrolled::(&table, &data); let n = s.close(); assert_eq!( n, 9, diff --git a/zstd/src/huff0/huff0_encoder.rs b/zstd/src/huff0/huff0_encoder.rs index 260e62e8..64a07d4f 100644 --- a/zstd/src/huff0/huff0_encoder.rs +++ b/zstd/src/huff0/huff0_encoder.rs @@ -100,6 +100,7 @@ impl Clone for WeightDescription { use crate::{ bit_io::BitWriter, + encoding::fastpath::EncodeTier, fse::fse_encoder::{self, FSEEncoder}, histogram, }; @@ -155,7 +156,9 @@ impl>> HuffmanEncoder<'_, '_, V> { /// (`huf_compress.c:1077-1108`) so each instantiation keeps the /// container's per-symbol nb_bits sum below 64 - 4 (the upstream zstd "≥ /// 4 free bits" precondition for `kFast=1`). - pub fn encode4x(&mut self, data: &[u8], with_table: bool) { + /// + /// The streams are written with the burst kernel of the encoder tier `K`. + pub fn encode4x(&mut self, data: &[u8], with_table: bool) { assert!( data.len() >= 12, "upstream zstd HUF_compress4X requires srcSize >= 12" @@ -203,7 +206,7 @@ impl>> HuffmanEncoder<'_, '_, V> { "HufCStream::new returned None — dst_capacity (from \ huf_tight_compress_bound) must be > 8 for a non-empty segment", ); - Self::encode_one_stream(packed_codes, &mut bit_c, segment, table_log); + Self::encode_one_stream::(packed_codes, &mut bit_c, segment, table_log); bit_c.close() }); // Upstream zstd `HUF_CStream_close` returns 0 on overflow (the @@ -249,7 +252,7 @@ impl>> HuffmanEncoder<'_, '_, V> { /// Dispatch on table_log to pick `kUnroll`, `kFastFlush`, /// `kLastFast` matching upstream zstd's 64-bit branch in /// `huf_compress.c:1092-1110`. - fn encode_one_stream( + fn encode_one_stream( table: &[u64], bit_c: &mut super::huf_cstream::HufCStream<'_>, data: &[u8], @@ -262,14 +265,14 @@ impl>> HuffmanEncoder<'_, '_, V> { // routes here, which clevels.h does not produce for L1-L22 // Fast/DFast workloads but is correct to handle defensively. match table_log { - 11 => Self::encode_one_stream_unrolled::<5, true, false>(table, bit_c, data), - 10 => Self::encode_one_stream_unrolled::<5, true, true>(table, bit_c, data), - 9 => Self::encode_one_stream_unrolled::<6, true, false>(table, bit_c, data), - 8 => Self::encode_one_stream_unrolled::<7, true, false>(table, bit_c, data), - 7 => Self::encode_one_stream_unrolled::<8, true, false>(table, bit_c, data), + 11 => Self::encode_one_stream_unrolled::(table, bit_c, data), + 10 => Self::encode_one_stream_unrolled::(table, bit_c, data), + 9 => Self::encode_one_stream_unrolled::(table, bit_c, data), + 8 => Self::encode_one_stream_unrolled::(table, bit_c, data), + 7 => Self::encode_one_stream_unrolled::(table, bit_c, data), // tableLog ∈ {1..=6, 12} — upstream zstd's "default" branch falls // here too. kUnroll=4 keeps per-flush bit budget safe. - _ => Self::encode_one_stream_unrolled::<4, false, false>(table, bit_c, data), + _ => Self::encode_one_stream_unrolled::(table, bit_c, data), } } @@ -300,6 +303,7 @@ impl>> HuffmanEncoder<'_, '_, V> { /// each unroll group. Safe when `K_UNROLL * max_nb_bits + 4 <= /// 64`, i.e. `K_UNROLL <= (60 / table_log)`. fn encode_one_stream_unrolled< + K: EncodeTier, const K_UNROLL: usize, const K_FAST_FLUSH: bool, const K_LAST_FAST: bool, @@ -312,7 +316,7 @@ impl>> HuffmanEncoder<'_, '_, V> { // which keeps the bit containers / positions / cursor // register-resident (upstream zstd's `HUF_CStream_t` shape) instead of // reloading them from `*bit_c` per symbol. Byte-identical output. - bit_c.encode_unrolled::(table, data); + bit_c.encode_unrolled::(table, data); } /// Encode one stream and pad it to fill the last byte @@ -2035,7 +2039,10 @@ pub(crate) fn huf_encode4x_for_test(data: &[u8]) -> Vec { { let mut writer = BitWriter::from(&mut encoded); let mut encoder = HuffmanEncoder::new(&table, &mut writer); - encoder.encode4x(data, true); + crate::encoding::fastpath::with_encode_tier!( + crate::encoding::fastpath::select_kernel(), + K => encoder.encode4x::(data, true) + ); writer.flush(); } encoded From 2d4d73949c4f4cacb7442c6b81199e2d72bbff5b Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 24 Oct 2026 21:06:33 +0300 Subject: [PATCH 05/15] refactor(encode): pick the kernel tier once per call Every public encoder entry (one-shot frame, Read-driven frame, each streaming write / flush / finish / start / block call, the legacy trainer) now chooses the CPU tier exactly once with `with_encode_tier!` and runs everything below as a monomorph over `K: EncodeTier`: frame setup, dictionary priming, block cut, the matchers, LDM, the literal gather and the entropy stage. - Drop the per-backend cached `kernel` fields (Fast, Dfast, Row, the BT/HC table) and the `select_kernel()` calls behind them; tier branches are `match K::TIER`, folded at monomorphisation. - Prefix compares call `K::common_prefix_len_ptr` directly; remove `dispatch_common_prefix_len_ptr(_with_kernel)`, which chose the kernel on every compare. - The built-in driver is reached through a hidden `Matcher::builtin_driver()`, so the encoder calls its tier-generic methods; the trait methods keep a single dispatch for external matchers driving it directly. - The row probe helpers that only unit tests reach are `cfg(test)` instead of `allow(dead_code)`. - Tests run backends on this CPU's tier through `with_host_tier!`. Output is unchanged: the tier only selects among kernels that produce identical results. --- zstd/src/dictionary/legacy.rs | 71 ++-- zstd/src/dictionary/legacy/tests.rs | 4 +- zstd/src/encoding/blocks/compressed.rs | 47 +-- .../blocks/compressed/literal_runs.rs | 15 +- .../dfast/extend_with_repcode_tests.rs | 21 +- zstd/src/encoding/dfast/mod.rs | 95 +++--- zstd/src/encoding/fastpath/mod.rs | 223 +++++------- zstd/src/encoding/frame_compressor.rs | 96 ++++-- zstd/src/encoding/hc/generator.rs | 55 +-- zstd/src/encoding/hc/generator/tests.rs | 9 +- zstd/src/encoding/hc/hc_tests.rs | 14 +- zstd/src/encoding/hc/mod.rs | 41 ++- zstd/src/encoding/hc/optimal.rs | 31 +- zstd/src/encoding/ldm/mod.rs | 4 +- zstd/src/encoding/ldm/search.rs | 4 +- zstd/src/encoding/ldm/search/tests.rs | 30 +- zstd/src/encoding/ldm/tests.rs | 13 +- zstd/src/encoding/levels/fastest.rs | 39 ++- zstd/src/encoding/levels/fastest/tests.rs | 4 +- .../encoding/match_generator/dict_prime.rs | 6 +- .../encoding/match_generator/long_distance.rs | 33 +- zstd/src/encoding/match_generator/mod.rs | 318 +++++++++++------- zstd/src/encoding/match_generator/tests.rs | 169 +++++----- zstd/src/encoding/match_table/helpers.rs | 39 +-- zstd/src/encoding/match_table/storage.rs | 77 +++-- .../match_table/storage/storage_tests.rs | 6 +- zstd/src/encoding/mod.rs | 9 + zstd/src/encoding/row/mod.rs | 138 +++++--- .../src/encoding/simple/fast_kernel/kernel.rs | 21 +- .../simple/fast_kernel/kernel/tests.rs | 20 +- zstd/src/encoding/simple/fast_matcher.rs | 17 +- zstd/src/encoding/streaming_encoder.rs | 83 +++-- 32 files changed, 979 insertions(+), 773 deletions(-) diff --git a/zstd/src/dictionary/legacy.rs b/zstd/src/dictionary/legacy.rs index 698be1d5..f3b6b917 100644 --- a/zstd/src/dictionary/legacy.rs +++ b/zstd/src/dictionary/legacy.rs @@ -13,9 +13,7 @@ use alloc::vec; use alloc::vec::Vec; use super::suffix_array::suffix_array; -use crate::encoding::fastpath::{ - FastpathKernel, dispatch_common_prefix_len_ptr_with_kernel, select_kernel, -}; +use crate::encoding::fastpath::{EncodeTier, select_kernel}; /// Fewest repetitions that make a prefix a candidate (`MINRATIO`). const MIN_RATIO: u32 = 4; @@ -80,19 +78,20 @@ fn noise_band() -> [u8; NOISE_LENGTH] { /// the two. The reference copies the whole corpus into a buffer one band /// longer (`ZDICT_trainFromBuffer_legacy`), a second copy of the samples while /// the caller still holds them for the finalizer. -struct Corpus<'a> { +/// +/// `K` is the compare kernel tier, chosen once for the whole run. +struct Corpus<'a, K: EncodeTier> { samples: &'a [u8], noise: [u8; NOISE_LENGTH], - /// The compare kernel for this CPU, resolved once for the whole run. - kernel: FastpathKernel, + tier: core::marker::PhantomData, } -impl<'a> Corpus<'a> { +impl<'a, K: EncodeTier> Corpus<'a, K> { fn new(samples: &'a [u8]) -> Self { Self { samples, noise: noise_band(), - kernel: select_kernel(), + tier: core::marker::PhantomData, } } @@ -156,14 +155,7 @@ impl<'a> Corpus<'a> { let base = samples.as_ptr(); // SAFETY: `a, b <= max(a, b)` and `in_samples == samples.len() - // max(a, b)`, so both runs of `in_samples` bytes lie in the samples. - let n = unsafe { - dispatch_common_prefix_len_ptr_with_kernel( - self.kernel, - base.add(a), - base.add(b), - in_samples, - ) - }; + let n = unsafe { K::common_prefix_len_ptr(base.add(a), base.add(b), in_samples) }; if n < in_samples { return n; } @@ -318,8 +310,24 @@ pub(crate) fn train_legacy_raw( kept -= 1; used -= sample_sizes[kept]; } - let corpus = Corpus::new(&samples[..used]); - find_segments(&mut list, &corpus, used, min_rep); + // The run's one choice of compare kernel: the analysis below is compiled + // for it. + crate::encoding::fastpath::with_encode_tier!( + select_kernel(), + K => select_content::(&mut list, &samples[..used], min_rep, dict_size) + ) +} + +/// Analyse `samples` and assemble the dictionary content from the best +/// segments that fit in `dict_size`, comparing through the tier `K`. +fn select_content( + list: &mut [DictItem], + samples: &[u8], + min_rep: u32, + dict_size: usize, +) -> Result, TooSmall> { + let corpus = Corpus::::new(samples); + find_segments(list, &corpus, samples.len(), min_rep); let segments = list[0].pos as usize; let content_size: usize = list[1..segments].iter().map(|d| d.length as usize).sum(); @@ -353,7 +361,12 @@ pub(crate) fn train_legacy_raw( /// `ZDICT_trainBuffer_legacy`: walk every uncovered position of the corpus in /// text order and insert the segment its suffix neighbourhood yields. -fn find_segments(list: &mut [DictItem], corpus: &Corpus<'_>, len: usize, min_rep: u32) { +fn find_segments( + list: &mut [DictItem], + corpus: &Corpus<'_, K>, + len: usize, + min_rep: u32, +) { let min_ratio = min_rep.max(MIN_RATIO); debug_assert_eq!(corpus.samples.len(), len); let sa = suffix_array(corpus.samples); @@ -410,11 +423,11 @@ fn mark_run(done: &mut [bool], at: usize, len: usize) { /// `ZDICT_analyzePos`: the segment the suffix of rank `start` leads to, or an /// empty one. Every position the analysis settles is marked in `done`. -fn analyze_position( +fn analyze_position( done: &mut [bool], suffixes: &Suffixes, mut start: i64, - corpus: &Corpus<'_>, + corpus: &Corpus<'_, K>, min_ratio: u32, ) -> DictItem { let mut pos = suffixes.at(start); @@ -575,7 +588,7 @@ fn analyze_position( /// Whether the `length` bytes at `a` equal those at `b` (`isIncluded`); both /// runs must lie inside the corpus. Compares `length` bytes and no more, where /// `common` would run on to the end of the shared prefix. -fn is_included(corpus: &Corpus<'_>, a: usize, b: usize, length: usize) -> bool { +fn is_included(corpus: &Corpus<'_, K>, a: usize, b: usize, length: usize) -> bool { let samples = corpus.samples; if let (Some(x), Some(y)) = (samples.get(a..a + length), samples.get(b..b + length)) { return x == y; @@ -597,7 +610,12 @@ fn promote(list: &mut [DictItem], mut at: usize) -> usize { /// `ZDICT_tryMerge`: fold `elt` into an entry it overlaps, skipping entry /// `skip`. Returns the merged entry's index, or 0 when nothing merged. -fn try_merge(list: &mut [DictItem], elt: DictItem, skip: usize, corpus: &Corpus<'_>) -> usize { +fn try_merge( + list: &mut [DictItem], + elt: DictItem, + skip: usize, + corpus: &Corpus<'_, K>, +) -> usize { let size = list[0].pos as usize; let elt_end = elt.pos + elt.length; @@ -682,7 +700,12 @@ fn remove_item(list: &mut [DictItem], id: usize) { /// `ZDICT_insertDictItem`: merge `elt` into the table if it overlaps an entry, /// and keep merging while the merged entry overlaps another; otherwise insert /// it in savings order, dropping the last entry when the table is full. -fn insert_item(list: &mut [DictItem], max_size: u32, elt: DictItem, corpus: &Corpus<'_>) { +fn insert_item( + list: &mut [DictItem], + max_size: u32, + elt: DictItem, + corpus: &Corpus<'_, K>, +) { let mut merge_id = try_merge(list, elt, 0, corpus); if merge_id != 0 { loop { diff --git a/zstd/src/dictionary/legacy/tests.rs b/zstd/src/dictionary/legacy/tests.rs index 8bf76c46..8de709a1 100644 --- a/zstd/src/dictionary/legacy/tests.rs +++ b/zstd/src/dictionary/legacy/tests.rs @@ -12,7 +12,7 @@ fn neighbour_walks_stop_at_the_noise_slots() { // The corpus is the band itself, and its one suffix is the only rank in // the array: every walk leaves the array on its first step. let band = noise_band(); - let corpus = Corpus::new(&band); + let corpus = Corpus::::new(&band); let suffixes = Suffixes::new(vec![0], band.len()); let mut done = vec![false; band.len() + 16]; let solution = analyze_position(&mut done, &suffixes, 0, &corpus, MIN_RATIO); @@ -33,7 +33,7 @@ fn the_measuring_walk_stops_at_the_noise_slot() { .cycle() .take(4 * NOISE_LENGTH) .collect(); - let corpus = Corpus::new(&corpus_bytes); + let corpus = Corpus::::new(&corpus_bytes); // In suffix order: each copy's suffix is a prefix of the one before it. let suffixes = Suffixes::new(vec![96, 64, 32, 0], corpus_bytes.len()); let mut done = vec![false; corpus_bytes.len() + 16]; diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index 141f96eb..684bc30b 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -6,7 +6,7 @@ use crate::{ bit_io::BitWriter, blocks::block::BlockType, encoding::block_header::BlockHeader, - encoding::fastpath::{EncodeTier, with_encode_tier}, + encoding::fastpath::EncodeTier, encoding::frame_compressor::{CompressState, FseTables, PreviousFseTable, SharedFseTable}, encoding::workspace::{RegionVec, Workspace, region_bytes}, encoding::{Matcher, Sequence}, @@ -431,15 +431,7 @@ impl Matcher for EntropyOnlyMatcher { /// returned for the caller to [`commit_block_tables`] if it keeps the block, /// or [`discard_block_tables`] if it writes the block raw instead. #[must_use = "the block's tables have to be committed or discarded"] -pub(crate) fn compress_block( - state: &mut CompressState, - output: &mut Vec, -) -> BlockTables { - with_encode_tier!(state.copy_kernel, K => compress_block_with::(state, output)) -} - -/// [`compress_block`] compiled for the tier `K`. -fn compress_block_with( +pub(crate) fn compress_block( state: &mut CompressState, output: &mut Vec, ) -> BlockTables { @@ -456,29 +448,7 @@ fn compress_block_with( tables } -pub(crate) fn compress_block_with_post_split( - state: &mut CompressState, - last_block: bool, - output: &mut Vec, - #[cfg(feature = "lsm")] block_decompressed_sizes: Option<&mut Vec>, - #[cfg(all(feature = "lsm", feature = "hash"))] block_checksums: Option<&mut Vec>, -) { - with_encode_tier!( - state.copy_kernel, - K => compress_block_with_post_split_with::( - state, - last_block, - output, - #[cfg(feature = "lsm")] - block_decompressed_sizes, - #[cfg(all(feature = "lsm", feature = "hash"))] - block_checksums, - ) - ) -} - -/// [`compress_block_with_post_split`] compiled for the tier `K`. -fn compress_block_with_post_split_with( +pub(crate) fn compress_block_with_post_split( state: &mut CompressState, last_block: bool, output: &mut Vec, @@ -713,9 +683,14 @@ fn collect_block_parts( let code_offsets = uses_fast_offset_codes(state.strategy_tag); let block_start_hist = state.offset_hist; let mut hist = state.offset_hist; - state.matcher.start_matching(|seq| { - record_sequence(seq, &mut tail, sequences, code_offsets, &mut hist); - }); + let record = |seq| record_sequence(seq, &mut tail, sequences, code_offsets, &mut hist); + // The built-in matcher searches compiled for the frame's tier; a matcher + // from outside the crate knows nothing of tiers. The branch folds per + // matcher type. + match state.matcher.builtin_driver() { + Some(driver) => driver.start_matching_with::(record), + None => state.matcher.start_matching(record), + } if code_offsets { state.offset_hist = hist; } diff --git a/zstd/src/encoding/blocks/compressed/literal_runs.rs b/zstd/src/encoding/blocks/compressed/literal_runs.rs index 19fa96a3..94a5ed1d 100644 --- a/zstd/src/encoding/blocks/compressed/literal_runs.rs +++ b/zstd/src/encoding/blocks/compressed/literal_runs.rs @@ -7,7 +7,7 @@ //! block, so no copy below the dispatch asks which kernel to use. use crate::decoding::simd_copy; -use crate::encoding::fastpath::{EncodeTier, FastpathKernel}; +use crate::encoding::fastpath::EncodeTier; use crate::encoding::workspace::RegionVec; use super::RawSequence; @@ -41,13 +41,16 @@ pub(super) fn gather_literals( feature = "kernel-avx2" ))] // SAFETY: the tier is only selected after confirming AVX2 (and BMI2). - FastpathKernel::Avx2Bmi2 => unsafe { gather_avx2(block, sequences, tail, dst) }, + crate::encoding::fastpath::FastpathKernel::Avx2Bmi2 => unsafe { + gather_avx2(block, sequences, tail, dst) + }, #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-sse" ))] // SAFETY: both tiers are only selected after confirming SSE2. - FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { + crate::encoding::fastpath::FastpathKernel::Sse2 + | crate::encoding::fastpath::FastpathKernel::Sse42 => unsafe { gather_sse2(block, sequences, tail, dst) }, #[cfg(all( @@ -55,13 +58,15 @@ pub(super) fn gather_literals( target_endian = "little", feature = "kernel-neon" ))] - FastpathKernel::Neon => gather_neon(block, sequences, tail, dst), + crate::encoding::fastpath::FastpathKernel::Neon => gather_neon(block, sequences, tail, dst), #[cfg(all( target_arch = "wasm32", target_feature = "simd128", feature = "kernel-simd128" ))] - FastpathKernel::Simd128 => gather_simd128(block, sequences, tail, dst), + crate::encoding::fastpath::FastpathKernel::Simd128 => { + gather_simd128(block, sequences, tail, dst) + } _ => gather_scalar(block, sequences, tail, dst), } } diff --git a/zstd/src/encoding/dfast/extend_with_repcode_tests.rs b/zstd/src/encoding/dfast/extend_with_repcode_tests.rs index 7360071f..643c87f8 100644 --- a/zstd/src/encoding/dfast/extend_with_repcode_tests.rs +++ b/zstd/src/encoding/dfast/extend_with_repcode_tests.rs @@ -10,6 +10,7 @@ use alloc::vec; use alloc::vec::Vec; use super::*; +use crate::encoding::fastpath::with_host_tier; /// Capture every sequence the matcher emits into an owned record, so the /// assertions can match on `lit_len` / `offset` / `match_len` shape directly. @@ -81,13 +82,13 @@ fn dfast_repcode_extension_emits_zero_literal_rep_on_constant_run() { let mut seqs = Vec::new(); let new_pos = { let mut rec = record_seq(&mut seqs); - dfast.extend_with_repcode_after_match( + with_host_tier!(K => dfast.extend_with_repcode_after_match::( current_abs_start, current_len, pos, &mut literals_start, &mut rec, - ) + )) }; assert!( @@ -153,13 +154,13 @@ fn dfast_repcode_extension_walks_into_retained_history() { let mut seqs = Vec::new(); let new_pos = { let mut rec = record_seq(&mut seqs); - dfast.extend_with_repcode_after_match( + with_host_tier!(K => dfast.extend_with_repcode_after_match::( current_abs_start, current_len, pos, &mut literals_start, &mut rec, - ) + )) }; assert!( @@ -226,13 +227,13 @@ fn dfast_repcode_extension_accepts_exactly_four_byte_rep() { let mut seqs = Vec::new(); let new_pos = { let mut rec = record_seq(&mut seqs); - dfast.extend_with_repcode_after_match( + with_host_tier!(K => dfast.extend_with_repcode_after_match::( current_abs_start, current_len, pos, &mut literals_start, &mut rec, - ) + )) }; // Helper must emit a single 4-byte rep, then stop because @@ -371,12 +372,12 @@ fn dfast_borrowed_window_matches_owned_sequence_stream() { owned.commit_input(&whole[..split]); { let mut rec = record_seq(&mut owned_seqs); - owned.start_matching(&mut rec); + with_host_tier!(K => owned.start_matching::(&mut rec)); } owned.commit_input(&whole[split..]); { let mut rec = record_seq(&mut owned_seqs); - owned.start_matching(&mut rec); + with_host_tier!(K => owned.start_matching::(&mut rec)); } // Borrowed: same bytes, scanned in place by absolute block range. @@ -388,11 +389,11 @@ fn dfast_borrowed_window_matches_owned_sequence_stream() { unsafe { borrowed.set_borrowed_window(&whole) }; { let mut rec = record_seq(&mut borrowed_seqs); - borrowed.start_matching_borrowed(0, split, &mut rec); + with_host_tier!(K => borrowed.start_matching_borrowed::(0, split, &mut rec)); } { let mut rec = record_seq(&mut borrowed_seqs); - borrowed.start_matching_borrowed(split, whole.len(), &mut rec); + with_host_tier!(K => borrowed.start_matching_borrowed::(split, whole.len(), &mut rec)); } assert_eq!( diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs index e8ff8c2c..4dccc7fb 100644 --- a/zstd/src/encoding/dfast/mod.rs +++ b/zstd/src/encoding/dfast/mod.rs @@ -16,7 +16,7 @@ use core::convert::TryInto; use super::Sequence; use super::blocks::encode_offset_with_history; use super::dict_attach::{DictAttach, DictKeep}; -use super::fastpath::{FastpathKernel, select_kernel}; +use super::fastpath::EncodeTier; use super::incompressible::block_looks_incompressible; use super::levels::config::MIN_WINDOW_LOG; use super::match_generator::{ @@ -24,7 +24,7 @@ use super::match_generator::{ DFAST_REBASE_GUARD_BAND, DFAST_SHORT_HASH_BITS_DELTA, DFAST_SHORT_HASH_LOOKAHEAD, DFAST_SKIP_STEP_GROWTH_INTERVAL, }; -use super::match_table::helpers::{common_prefix_len_with_kernel, extend_backwards_shared}; +use super::match_table::helpers::extend_backwards_shared; use super::match_table::storage::{REBASE_RESET_FLOOR_CEILING, check_stream_abs_headroom}; use super::opt::types::MatchCandidate; use super::util::hash_top_bits; @@ -174,10 +174,6 @@ pub(crate) struct DfastMatchGenerator { /// (the CDict's `hashLog` / `chainLog`), set by the driver; `None` uses /// the live widths. dict_table_bits: Option<(usize, usize)>, - /// CPU kernel for the `common_prefix_len` / repcode byte-compares, - /// resolved once at construction so the per-byte match-finder scans skip - /// the `select_kernel()` `OnceLock` atomic on every call. - pub(crate) kernel: FastpathKernel, /// Borrowed one-shot input window, registered once per frame: /// `(input_base_ptr, total_len)`. When `Some`, the fast loop sources /// bytes from the caller's in-place input slice with absolute input @@ -330,7 +326,6 @@ impl DfastMatchGenerator { #[cfg(test)] dict_table_builds: (0, 0), dict_table_bits: None, - kernel: select_kernel(), borrowed_input: None, borrowed_block: None, dict_resident: false, @@ -1248,7 +1243,10 @@ impl DfastMatchGenerator { } } - pub(crate) fn start_matching(&mut self, mut handle_sequence: impl FnMut(Sequence)) { + pub(crate) fn start_matching( + &mut self, + mut handle_sequence: impl FnMut(Sequence), + ) { self.ensure_hash_tables(); let current_len = self.window_blocks.back().copied().unwrap_or(0); @@ -1272,7 +1270,7 @@ impl DfastMatchGenerator { // dfast is the upstream zstd's greedy double-fast at every level (no lazy // variant exists — lazy parsing is the separate `ZSTD_lazy`/`lazy2` // strategy), so there is a single match path. - self.start_matching_fast_loop(current_abs_start, current_len, &mut handle_sequence); + self.start_matching_fast_loop::(current_abs_start, current_len, &mut handle_sequence); } /// Register the caller's in-place input buffer for the borrowed one-shot @@ -1327,7 +1325,7 @@ impl DfastMatchGenerator { /// offsets, candidate reads land in the same buffer, and the seam re-seed /// re-hashes the prior block's short-key tail exactly as the owned loop /// does once `history` spans it. - pub(crate) fn start_matching_borrowed( + pub(crate) fn start_matching_borrowed( &mut self, block_start: usize, block_end: usize, @@ -1350,7 +1348,7 @@ impl DfastMatchGenerator { if backfill_start < current_abs_start { self.insert_positions(backfill_start, current_abs_start); } - self.start_matching_fast_loop(current_abs_start, current_len, &mut handle_sequence); + self.start_matching_fast_loop::(current_abs_start, current_len, &mut handle_sequence); } /// Borrowed one-shot equivalent of [`Self::skip_matching`]: stage the @@ -1423,7 +1421,7 @@ impl DfastMatchGenerator { /// updated by this helper — the caller routes through /// `emit_candidate` which inserts via `insert_positions` over the /// emitted range, exactly like the inner loop's hit path. - fn probe_tail_ip0_only( + fn probe_tail_ip0_only( &self, current_abs_start: usize, current_len: usize, @@ -1533,9 +1531,7 @@ impl DfastMatchGenerator { unsafe { let lhs = history_base_ptr.add(history_start_offset + cand_idx + 8); let rhs = history_base_ptr.add(history_start_offset + concat_idx0 + 8); - let ext = crate::encoding::fastpath::dispatch_common_prefix_len_ptr( - lhs, rhs, max_fwd, - ); + let ext = K::common_prefix_len_ptr(lhs, rhs, max_fwd); match_len += ext; } let concat = unsafe { @@ -1573,9 +1569,7 @@ impl DfastMatchGenerator { unsafe { let lhs = history_base_ptr.add(history_start_offset + cand_idx_s + 4); let rhs = history_base_ptr.add(history_start_offset + concat_idx0 + 4); - let ext = crate::encoding::fastpath::dispatch_common_prefix_len_ptr( - lhs, rhs, max_fwd, - ); + let ext = K::common_prefix_len_ptr(lhs, rhs, max_fwd); s_match_len += ext; } let concat = unsafe { @@ -1623,7 +1617,7 @@ impl DfastMatchGenerator { /// rep is a net win over re-running the full hash search. Returns /// the new value of `pos` and updates `literals_start` in place to /// the post-rep-chain anchor. - fn extend_with_repcode_after_match( + fn extend_with_repcode_after_match( &mut self, current_abs_start: usize, current_len: usize, @@ -1688,8 +1682,15 @@ impl DfastMatchGenerator { if !gate_eq { break; } - let match_len = - common_prefix_len_with_kernel(self.kernel, &concat[cand_idx..], &concat[cur_idx..]); + // SAFETY: `cand_idx < cur_idx < concat.len()` (above), and the + // compare stops at the shorter of the two tails. + let match_len = unsafe { + K::common_prefix_len_ptr( + concat.as_ptr().add(cand_idx), + concat.as_ptr().add(cur_idx), + concat.len() - cur_idx, + ) + }; if match_len < DFAST_REP_MIN_MATCH_LEN { break; } @@ -2368,7 +2369,7 @@ fn short_hash_key(data: &[u8]) -> u64 { /// cpl calls inline under one umbrella and `select_kernel()` is resolved ONCE /// per block in the bare dispatcher, never per cpl call. macro_rules! start_matching_fast_loop_body { - ($self:ident, $current_abs_start:ident, $current_len:ident, $handle_sequence:ident, $cpl:path, $borrowed:expr, $tagged:expr) => {{ + ($self:ident, $current_abs_start:ident, $current_len:ident, $handle_sequence:ident, $cpl:path, $tier:ty, $borrowed:expr, $tagged:expr) => {{ // Behaviour change vs the pre-refactor `start_matching_general`: // this fast loop deliberately drops the strict-incompressible // early-skip path (the `block_looks_incompressible_strict` short @@ -2520,7 +2521,7 @@ macro_rules! start_matching_fast_loop_body { // they're skipped — upstream zstd accepts that exact tradeoff at // the iend boundary). if ip1 + HASH_READ_SIZE > $current_len { - if let Some(committed) = $self.probe_tail_ip0_only( + if let Some(committed) = $self.probe_tail_ip0_only::<$tier>( $current_abs_start, $current_len, ip0, @@ -2535,7 +2536,7 @@ macro_rules! start_matching_fast_loop_body { $handle_sequence, ); pos = start + committed.match_len; - pos = $self.extend_with_repcode_after_match( + pos = $self.extend_with_repcode_after_match::<$tier>( $current_abs_start, $current_len, pos, @@ -3272,7 +3273,7 @@ macro_rules! start_matching_fast_loop_body { $handle_sequence, ); pos = start + candidate.match_len; - pos = $self.extend_with_repcode_after_match( + pos = $self.extend_with_repcode_after_match::<$tier>( $current_abs_start, $current_len, pos, @@ -3336,7 +3337,7 @@ macro_rules! start_matching_fast_loop_body { /// slightly FEWER than it does — what is left there is execution density, not /// work. macro_rules! start_matching_dict_loop_body { - ($self:ident, $current_abs_start:ident, $current_len:ident, $handle_sequence:ident, $cpl:path) => {{ + ($self:ident, $current_abs_start:ident, $current_len:ident, $handle_sequence:ident, $cpl:path, $tier:ty) => {{ debug_assert!($current_len > 0, "dict_loop precondition: $current_len > 0"); $self.ensure_room_for($current_abs_start + $current_len - 1); const PRIME: u64 = 0xCF1BBCDCB7A56463_u64; @@ -3758,7 +3759,7 @@ macro_rules! start_matching_dict_loop_body { $handle_sequence, ); pos = start + candidate.match_len; - pos = $self.extend_with_repcode_after_match( + pos = $self.extend_with_repcode_after_match::<$tier>( $current_abs_start, $current_len, pos, @@ -3788,13 +3789,12 @@ enum DfastInnerExit { } impl DfastMatchGenerator { - /// Dispatcher for the per-kernel dfast fast loop: resolve the tier ONCE - /// per block via `select_kernel()` and call the matching - /// `start_matching_fast_loop_` wrapper, so every per-position + /// The dfast loop of the encoder tier `K`: its + /// `start_matching_fast_loop_` wrapper, so every per-position /// `common_prefix_len_ptr` inlines under one `#[target_feature]` umbrella - /// (mirrors `build_optimal_plan_impl`). Replaces the prior per-cpl - /// `dispatch_common_prefix_len_ptr` runtime dispatch. - fn start_matching_fast_loop( + /// (mirrors `build_optimal_plan_impl`). The match on `K::TIER` folds to + /// that one call. + fn start_matching_fast_loop( &mut self, current_abs_start: usize, current_len: usize, @@ -3867,7 +3867,7 @@ impl DfastMatchGenerator { // target can check the two against each other — the branch is // `cfg(test)` and does not exist in a shipped build. #[cfg(test)] - if use_dict && self.kernel == crate::encoding::fastpath::FastpathKernel::Scalar { + if use_dict && K::TIER == crate::encoding::fastpath::FastpathKernel::Scalar { return self.start_matching_dict_loop_scalar( current_abs_start, current_len, @@ -3879,10 +3879,7 @@ impl DfastMatchGenerator { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - // Use the matcher-level cached kernel (resolved once in `new()`), - // not a per-block `select_kernel()` — the cache only helps if the - // hot path actually reads it. Mirrors the Row backend. - match self.kernel { + match K::TIER { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { dispatch_dict!( @@ -3910,7 +3907,7 @@ impl DfastMatchGenerator { // compile time here, so without it this target could never run the // scalar dictionary loop to check the SIMD one against. #[cfg(test)] - if use_dict && self.kernel == crate::encoding::fastpath::FastpathKernel::Scalar { + if use_dict && K::TIER == crate::encoding::fastpath::FastpathKernel::Scalar { return self.start_matching_dict_loop_scalar( current_abs_start, current_len, @@ -3964,6 +3961,7 @@ impl DfastMatchGenerator { current_len, handle_sequence, crate::encoding::fastpath::neon::common_prefix_len_ptr, + crate::encoding::fastpath::NeonTier, BORROWED, TAGGED ) @@ -3986,6 +3984,7 @@ impl DfastMatchGenerator { current_len, handle_sequence, crate::encoding::fastpath::sse2::common_prefix_len_ptr, + crate::encoding::fastpath::Sse2Tier, BORROWED, TAGGED ) @@ -4008,6 +4007,7 @@ impl DfastMatchGenerator { current_len, handle_sequence, crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr, + crate::encoding::fastpath::Avx2Bmi2Tier, BORROWED, TAGGED ) @@ -4031,6 +4031,7 @@ impl DfastMatchGenerator { current_len, handle_sequence, crate::encoding::fastpath::simd128::common_prefix_len_ptr, + crate::encoding::fastpath::Simd128Tier, BORROWED, TAGGED ) @@ -4061,6 +4062,7 @@ impl DfastMatchGenerator { current_len, handle_sequence, crate::encoding::fastpath::scalar::common_prefix_len_ptr, + crate::encoding::fastpath::ScalarTier, BORROWED, TAGGED ) @@ -4083,7 +4085,8 @@ impl DfastMatchGenerator { current_abs_start, current_len, handle_sequence, - crate::encoding::fastpath::neon::common_prefix_len_ptr + crate::encoding::fastpath::neon::common_prefix_len_ptr, + crate::encoding::fastpath::NeonTier ) } @@ -4103,7 +4106,8 @@ impl DfastMatchGenerator { current_abs_start, current_len, handle_sequence, - crate::encoding::fastpath::sse2::common_prefix_len_ptr + crate::encoding::fastpath::sse2::common_prefix_len_ptr, + crate::encoding::fastpath::Sse2Tier ) } @@ -4123,7 +4127,8 @@ impl DfastMatchGenerator { current_abs_start, current_len, handle_sequence, - crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr + crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr, + crate::encoding::fastpath::Avx2Bmi2Tier ) } @@ -4144,7 +4149,8 @@ impl DfastMatchGenerator { current_abs_start, current_len, handle_sequence, - crate::encoding::fastpath::simd128::common_prefix_len_ptr + crate::encoding::fastpath::simd128::common_prefix_len_ptr, + crate::encoding::fastpath::Simd128Tier ) } @@ -4180,7 +4186,8 @@ impl DfastMatchGenerator { current_abs_start, current_len, handle_sequence, - crate::encoding::fastpath::scalar::common_prefix_len_ptr + crate::encoding::fastpath::scalar::common_prefix_len_ptr, + crate::encoding::fastpath::ScalarTier ) } } diff --git a/zstd/src/encoding/fastpath/mod.rs b/zstd/src/encoding/fastpath/mod.rs index 3fab4f29..76d847ea 100644 --- a/zstd/src/encoding/fastpath/mod.rs +++ b/zstd/src/encoding/fastpath/mod.rs @@ -316,87 +316,87 @@ fn detect_kernel_uncached() -> FastpathKernel { pub(crate) trait EncodeTier { /// The tier this type stands for. const TIER: FastpathKernel; -} - -/// [`FastpathKernel::Scalar`] as a type. -pub(crate) struct ScalarTier; -impl EncodeTier for ScalarTier { - const TIER: FastpathKernel = FastpathKernel::Scalar; -} - -/// [`FastpathKernel::Neon`] as a type. -#[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" -))] -pub(crate) struct NeonTier; -#[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" -))] -impl EncodeTier for NeonTier { - const TIER: FastpathKernel = FastpathKernel::Neon; -} -/// [`FastpathKernel::Sse2`] as a type. -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" -))] -pub(crate) struct Sse2Tier; -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" -))] -impl EncodeTier for Sse2Tier { - const TIER: FastpathKernel = FastpathKernel::Sse2; + /// Length of the common prefix of `lhs` and `rhs`, at most `max`, with this + /// tier's compare. + /// + /// # Safety + /// `lhs` / `rhs` must each point to at least `max` initialized bytes, and + /// the CPU must support the tier, which holds for the tier the encoder + /// chose. + unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize; } -/// [`FastpathKernel::Sse42`] as a type. -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" -))] -pub(crate) struct Sse42Tier; -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" -))] -impl EncodeTier for Sse42Tier { - const TIER: FastpathKernel = FastpathKernel::Sse42; -} +/// One tier type: the struct, its [`FastpathKernel`] and its compare kernel, +/// under the cfg that builds the tier. +macro_rules! encode_tier { + ($(#[$cfg:meta])* $name:ident, $tier:ident, $prefix:path) => { + #[doc = concat!("[`FastpathKernel::", stringify!($tier), "`] as a type.")] + $(#[$cfg])* + pub(crate) struct $name; + $(#[$cfg])* + impl EncodeTier for $name { + const TIER: FastpathKernel = FastpathKernel::$tier; -/// [`FastpathKernel::Avx2Bmi2`] as a type. -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-avx2" -))] -pub(crate) struct Avx2Bmi2Tier; -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-avx2" -))] -impl EncodeTier for Avx2Bmi2Tier { - const TIER: FastpathKernel = FastpathKernel::Avx2Bmi2; + #[inline(always)] + unsafe fn common_prefix_len_ptr(lhs: *const u8, rhs: *const u8, max: usize) -> usize { + // SAFETY: forwarded from the caller. + unsafe { $prefix(lhs, rhs, max) } + } + } + }; } -/// [`FastpathKernel::Simd128`] as a type. -#[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" -))] -pub(crate) struct Simd128Tier; -#[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" -))] -impl EncodeTier for Simd128Tier { - const TIER: FastpathKernel = FastpathKernel::Simd128; -} +encode_tier!(ScalarTier, Scalar, scalar::common_prefix_len_ptr); +encode_tier!( + #[cfg(all( + target_arch = "aarch64", + target_endian = "little", + feature = "kernel-neon" + ))] + NeonTier, + Neon, + neon::common_prefix_len_ptr +); +// Both SSE tiers compare with the same SSE2 kernel; only the optimal parser's +// price set differs between them. +encode_tier!( + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" + ))] + Sse2Tier, + Sse2, + sse2::common_prefix_len_ptr +); +encode_tier!( + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" + ))] + Sse42Tier, + Sse42, + sse2::common_prefix_len_ptr +); +encode_tier!( + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" + ))] + Avx2Bmi2Tier, + Avx2Bmi2, + avx2_bmi2::common_prefix_len_ptr +); +encode_tier!( + #[cfg(all( + target_arch = "wasm32", + target_feature = "simd128", + feature = "kernel-simd128" + ))] + Simd128Tier, + Simd128, + simd128::common_prefix_len_ptr +); /// The encoder's one choice of tier: evaluate `$body` with `$k` naming the /// [`EncodeTier`] type of the tier `$tier` holds. @@ -455,68 +455,19 @@ macro_rules! with_encode_tier { } pub(crate) use with_encode_tier; -/// Public entry point for raw-pointer prefix-length scans (BT byte compare, -/// repcode extend, etc.): resolves the tier via [`select_kernel`] on every -/// call, so hot loops should cache the kernel and call -/// [`dispatch_common_prefix_len_ptr_with_kernel`] instead. -/// -/// # Safety -/// `lhs` / `rhs` must each point to at least `max` initialized bytes. -#[inline] -pub(crate) unsafe fn dispatch_common_prefix_len_ptr( - lhs: *const u8, - rhs: *const u8, - max: usize, -) -> usize { - // Cold-path shim: resolves the kernel via `select_kernel()` on every call. - // Hot match-finder loops resolve the kernel once per block and call - // [`dispatch_common_prefix_len_ptr_with_kernel`] directly. - unsafe { dispatch_common_prefix_len_ptr_with_kernel(select_kernel(), lhs, rhs, max) } -} - -/// Prefix-length scan against an already-resolved [`FastpathKernel`], so a hot -/// loop pays the kernel-select once per block (caller-cached) instead of the -/// `OnceLock` atomic + branch on every byte-compare. -/// -/// # Safety -/// `lhs` / `rhs` must each point to at least `max` initialized bytes. -#[inline(always)] -pub(crate) unsafe fn dispatch_common_prefix_len_ptr_with_kernel( - kernel: FastpathKernel, - lhs: *const u8, - rhs: *const u8, - max: usize, -) -> usize { - match kernel { - FastpathKernel::Scalar => unsafe { scalar::common_prefix_len_ptr(lhs, rhs, max) }, - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - FastpathKernel::Neon => unsafe { neon::common_prefix_len_ptr(lhs, rhs, max) }, - // Both x86 tiers share this kernel: it is SSE2 throughout, and only - // the optimal parser's price set differs between them. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" - ))] - FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { - sse2::common_prefix_len_ptr(lhs, rhs, max) - }, - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-avx2" - ))] - FastpathKernel::Avx2Bmi2 => unsafe { avx2_bmi2::common_prefix_len_ptr(lhs, rhs, max) }, - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ))] - FastpathKernel::Simd128 => unsafe { simd128::common_prefix_len_ptr(lhs, rhs, max) }, - } +/// [`with_encode_tier!`] on this CPU's tier, the one the encoder picks: unit +/// tests drive a backend directly through the kernels it would ship with. +#[cfg(test)] +macro_rules! with_host_tier { + ($k:ident => $body:expr) => { + $crate::encoding::fastpath::with_encode_tier!( + $crate::encoding::fastpath::select_kernel(), + $k => $body + ) + }; } +#[cfg(test)] +pub(crate) use with_host_tier; #[cfg(test)] mod tests; diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index f561c2dc..d0e8397f 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -9,8 +9,8 @@ use core::hash::Hasher; use super::{ CompressionLevel, LiteralCompressionMode, Matcher, block_header::BlockHeader, - frame_header::FrameHeader, levels::*, match_generator::MatchGeneratorDriver, - workspace::IngestBuffer, + fastpath::EncodeTier, frame_header::FrameHeader, levels::*, + match_generator::MatchGeneratorDriver, workspace::IngestBuffer, }; use crate::common::MAX_BLOCK_SIZE; use crate::fse::fse_encoder::{FSETable, default_ll_table, default_ml_table, default_of_table}; @@ -1207,7 +1207,7 @@ pub(crate) fn optimal_block_size_with( /// again would commit it into the window a second time. Otherwise it is /// primed, or restored from the matcher's copy-mode snapshot when the source /// is past the strategy's attach cutoff. -pub(crate) fn load_frame_dictionary( +pub(crate) fn load_frame_dictionary( state: &mut CompressState, level: CompressionLevel, dict: &EncoderDictionary, @@ -1265,9 +1265,14 @@ pub(crate) fn load_frame_dictionary( ); let restored = prefer_copy_snapshot && state.matcher.restore_primed_dictionary(level); if !restored { - state - .matcher - .prime_with_dictionary(dict.inner.dict_content.as_slice(), dict.inner.offset_hist); + let content = dict.inner.dict_content.as_slice(); + // The built-in matcher indexes the dictionary under the frame's tier. + match state.matcher.builtin_driver() { + Some(driver) => driver.prime_with_dictionary_with::(content, dict.inner.offset_hist), + None => state + .matcher + .prime_with_dictionary(content, dict.inner.offset_hist), + } if prefer_copy_snapshot { state.matcher.capture_primed_dictionary(level); } @@ -2026,7 +2031,12 @@ impl FrameCompressor { /// an in-place `&[u8]` cursor. Returns `total_uncompressed`; the caller /// emits the frame header (before this call, when the content size is /// known) or the drain tail. - fn run_one_frame(&mut self, input: &[u8], prep: &FramePrep, out: &mut Vec) -> u64 { + fn run_one_frame( + &mut self, + input: &[u8], + prep: &FramePrep, + out: &mut Vec, + ) -> u64 { if self.borrowed_eligible(input.len(), prep) { // A reused Dfast matcher's tables hold earlier frames, which its // borrowed kernel (numbering this input from zero) would read as @@ -2041,10 +2051,10 @@ impl FrameCompressor { if !self.state.matcher.borrowed_frame_is_independent() { self.state.matcher.forget_earlier_frames(); } - self.run_borrowed_block_loop(input, out) + self.run_borrowed_block_loop::(input, out) } else { let mut cursor: &[u8] = input; - self.run_owned_block_loop(&mut cursor, prep.initial_size_hint, true, out) + self.run_owned_block_loop::<_, K>(&mut cursor, prep.initial_size_hint, true, out) } } @@ -2092,11 +2102,23 @@ impl FrameCompressor { /// Panics on encoder error, matching [`Self::compress`] and /// [`compress_slice_to_vec`](crate::encoding::compress_slice_to_vec). pub fn compress_independent_frame_into(&mut self, input: &[u8], out: &mut Vec) { + // The frame's one choice of kernel tier: its setup, every block's cut, + // search and entropy stage run compiled for it. + crate::encoding::fastpath::with_encode_tier!( + self.state.copy_kernel, + K => self.compress_independent_frame_with::(input, out) + ) + } + + /// [`Self::compress_independent_frame_into`] compiled for the kernel tier + /// `K`. + fn compress_independent_frame_with(&mut self, input: &[u8], out: &mut Vec) { // Size the next frame from the actual payload, not a stale hint a // previous call may have left behind (a wrong hint would change the // resolved window/header and could flip borrowed eligibility). self.source_size_hint = Some(input.len() as u64); - let prep = self.prepare_frame(crate::encoding::workspace::IngestPlan::Slice(input.len())); + let prep = + self.prepare_frame::(crate::encoding::workspace::IngestPlan::Slice(input.len())); // Content size is known up front (one-shot), so write the frame // header FIRST and emit blocks STRAIGHT into `out` — no separate // `all_blocks` accumulator and no header+blocks copy (which was the @@ -2120,7 +2142,7 @@ impl FrameCompressor { out.reserve(18 + first_block_bound + checksum_len); self.append_frame_header(total_uncompressed, &prep, out); let header_len = out.len(); - let _ = self.run_one_frame(input, &prep, out); + let _ = self.run_one_frame::(input, &prep, out); #[cfg(feature = "hash")] if self.content_checksum { out.extend_from_slice(&(self.hasher.finish() as u32).to_le_bytes()); @@ -2161,7 +2183,7 @@ impl FrameCompressor { /// total_uncompressed)`. Caller guarantees Fast backend + no /// dictionary; over-window inputs are fine (matches are bounded by /// `window_low` exactly as the owned evicting path). - fn run_borrowed_block_loop(&mut self, input: &[u8], out: &mut Vec) -> u64 { + fn run_borrowed_block_loop(&mut self, input: &[u8], out: &mut Vec) -> u64 { // Blocks are appended to `out` starting here. `out` may already hold // the frame header (the one-shot compress-into-Vec path writes it // first, since the content size is known up front, and the loop @@ -2265,7 +2287,7 @@ impl FrameCompressor { if self.content_checksum { self.hasher.write(block); } - crate::encoding::levels::compress_block_encoded_borrowed( + crate::encoding::levels::compress_block_encoded_borrowed::( &mut *guard.0, self.compression_level, last_block, @@ -2696,10 +2718,20 @@ impl FrameCompressor self.compress_with::() + ) + } + + /// [`Self::compress`] compiled for the kernel tier `K`. + fn compress_with(&mut self) { // `prepare_frame` / `finish_frame` are shared with the borrowed // one-shot path, so both run the same reset / dict-prime / // entropy-seed setup and frame tail. - let prep = self.prepare_frame(crate::encoding::workspace::IngestPlan::Stream); + let prep = self.prepare_frame::(crate::encoding::workspace::IngestPlan::Stream); // Take the reader out so `run_owned_block_loop` can borrow it // mutably alongside `&mut self` (the rest of the loop touches // `self.state` / `self.hasher`, disjoint from the reader). Restored @@ -2725,7 +2757,7 @@ impl FrameCompressor( &mut block_source, prep.initial_size_hint, false, @@ -2749,7 +2781,16 @@ impl FrameCompressor) { - let prep = self.prepare_frame(crate::encoding::workspace::IngestPlan::Stream); + crate::encoding::fastpath::with_encode_tier!( + self.state.copy_kernel, + K => self.compress_known_into_with::(total, out) + ) + } + + /// [`Self::compress_known_into`] compiled for the kernel tier `K`. + #[cfg(feature = "dict-builder")] + fn compress_known_into_with(&mut self, total: u64, out: &mut Vec) { + let prep = self.prepare_frame::(crate::encoding::workspace::IngestPlan::Stream); let mut source = self .uncompressed_data .take() @@ -2758,7 +2799,12 @@ impl FrameCompressor( + &mut block_source, + prep.initial_size_hint, + false, + out, + ); self.uncompressed_data = Some(source); assert_eq!( read, total, @@ -2780,7 +2826,10 @@ impl FrameCompressor FramePrep { + fn prepare_frame( + &mut self, + ingest: crate::encoding::workspace::IngestPlan, + ) -> FramePrep { // The raw-skip's memory of what this frame has already emitted. Frames // are independent, so carrying it over would let one frame's content // hold the skip off for the next; the allocation is kept. @@ -2881,7 +2930,7 @@ impl FrameCompressor( &mut self.state, self.compression_level, dict, @@ -2979,7 +3028,7 @@ impl FrameCompressor( + fn run_owned_block_loop( &mut self, source: &mut S, initial_size_hint: Option, @@ -3064,7 +3113,10 @@ impl FrameCompressor driver.settle_block_cut_with::(available, split_at), + None => self.state.matcher.settle_block_cut(available, split_at), + } } else { split_at }; @@ -3150,7 +3202,7 @@ impl FrameCompressor( &mut self.state, self.compression_level, last_block, diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs index 08d7e5d2..0b8986f5 100644 --- a/zstd/src/encoding/hc/generator.rs +++ b/zstd/src/encoding/hc/generator.rs @@ -10,6 +10,7 @@ use crate::encoding::Sequence; use crate::encoding::blocks::encode_offset_with_history; use crate::encoding::cost_model::HC_PREDEF_THRESHOLD; +use crate::encoding::fastpath::EncodeTier; use crate::encoding::hc::{HC_MIN_MATCH_LEN, HcMatch, MAX_HC_SEARCH_DEPTH}; use crate::encoding::levels::config::HcConfig; use crate::encoding::match_generator::{HC_SEARCH_DEPTH, HC_TARGET_LEN, HcBackend}; @@ -1528,8 +1529,8 @@ impl HcMatchGenerator { /// Backfill positions from the tail of the previous slice that couldn't be /// hashed at the time (insert_position needs 4 bytes of lookahead). - pub(crate) fn skip_matching(&mut self, incompressible_hint: Option) { - self.table.skip_matching(incompressible_hint); + pub(crate) fn skip_matching(&mut self, incompressible_hint: Option) { + self.table.skip_matching::(incompressible_hint); } /// Runtime-dispatched entry kept only for in-crate tests. Production @@ -1538,7 +1539,16 @@ impl HcMatchGenerator { /// which pick the lazy / optimal arm from `S::USE_BT` at /// monomorphisation time. #[cfg(test)] - pub(crate) fn start_matching(&mut self, mut handle_sequence: impl FnMut(Sequence)) { + pub(crate) fn start_matching(&mut self, handle_sequence: impl FnMut(Sequence)) { + crate::encoding::fastpath::with_encode_tier!( + crate::encoding::fastpath::select_kernel(), + K => self.start_matching_with::(handle_sequence) + ) + } + + /// [`Self::start_matching`] compiled for the tier `K`. + #[cfg(test)] + fn start_matching_with(&mut self, mut handle_sequence: impl FnMut(Sequence)) { use crate::encoding::strategy::{self, StrategyTag}; // Dispatch on the mirrored `strategy_tag` so each test runs // under the same monomorphisation production would pick. @@ -1547,17 +1557,17 @@ impl HcMatchGenerator { // BtOpt from BtUltra. match self.strategy_tag { StrategyTag::Fast | StrategyTag::Dfast | StrategyTag::Greedy | StrategyTag::Lazy => { - self.start_matching_lazy(&mut handle_sequence) + self.start_matching_lazy::(&mut handle_sequence) } StrategyTag::Btlazy2 => unreachable!("btlazy2 runs on the lazy (Row) backend"), StrategyTag::BtOpt => { - self.start_matching_optimal::(&mut handle_sequence) + self.start_matching_optimal::(&mut handle_sequence) } StrategyTag::BtUltra => { - self.start_matching_optimal::(&mut handle_sequence) + self.start_matching_optimal::(&mut handle_sequence) } StrategyTag::BtUltra2 => { - self.start_matching_optimal::(&mut handle_sequence) + self.start_matching_optimal::(&mut handle_sequence) } } } @@ -1572,7 +1582,7 @@ impl HcMatchGenerator { /// `HcMatchGenerator::start_matching` reaches the same arms by /// runtime-matching on `self.strategy_tag` (the parse-mode field /// has been removed); production never invokes that path. - pub(crate) fn start_matching_strategy( + pub(crate) fn start_matching_strategy( &mut self, handle_sequence: &mut impl FnMut(Sequence), ) { @@ -1582,9 +1592,9 @@ impl HcMatchGenerator { "Strategy::USE_BT disagrees with runtime table.uses_bt at HC dispatch" ); if S::USE_BT { - self.start_matching_optimal::(handle_sequence) + self.start_matching_optimal::(handle_sequence) } else { - self.start_matching_lazy(handle_sequence) + self.start_matching_lazy::(handle_sequence) } } @@ -1592,15 +1602,18 @@ impl HcMatchGenerator { /// (attach-mode dictionary), else the no-dict monomorph. Mirrors upstream's /// compile-time `dictMode` split — the `DICT = false` body carries no dms /// code at all, so the no-dict hot path is unaffected by the dict search. - pub(crate) fn start_matching_lazy(&mut self, handle_sequence: impl FnMut(Sequence)) { + pub(crate) fn start_matching_lazy( + &mut self, + handle_sequence: impl FnMut(Sequence), + ) { if self.table.dms.is_primed() { - self.start_matching_lazy_impl::(handle_sequence); + self.start_matching_lazy_impl::(handle_sequence); } else { - self.start_matching_lazy_impl::(handle_sequence); + self.start_matching_lazy_impl::(handle_sequence); } } - pub(crate) fn start_matching_lazy_impl( + pub(crate) fn start_matching_lazy_impl( &mut self, mut handle_sequence: impl FnMut(Sequence), ) { @@ -1635,7 +1648,7 @@ impl HcMatchGenerator { let current_abs_end = current_abs_start + current_len; self.table - .backfill_boundary_positions(current_abs_start, current_abs_end); + .backfill_boundary_positions::(current_abs_start, current_abs_end); let mut pos = 0usize; let mut literals_start = 0usize; @@ -1660,7 +1673,7 @@ impl HcMatchGenerator { // forward `(offset, length)` in registers (`HcMatch`, 16 bytes). let best = match carried.take() { Some(m) => m, - None => self.hc.find_best_match::( + None => self.hc.find_best_match::( concat, dms_primed, &self.table, @@ -1674,7 +1687,7 @@ impl HcMatchGenerator { // position during its own search, before the depth loop probes // the next one. self.table.insert_position(abs_pos); - if let Some(carry) = self.hc.lazy_decide_carry::( + if let Some(carry) = self.hc.lazy_decide_carry::( concat, dms_primed, &self.table, @@ -1791,26 +1804,26 @@ impl HcMatchGenerator { /// parse reads its range via `current_block_range()` and its bytes via /// `get_last_space()` / `live_history()`, all borrowed-aware, so the block /// is scanned in place with the per-position window_low offset cap. - pub(crate) fn start_matching_lazy_borrowed( + pub(crate) fn start_matching_lazy_borrowed( &mut self, block_start: usize, block_end: usize, handle_sequence: impl FnMut(Sequence), ) { self.table.stage_borrowed_block(block_start, block_end); - self.start_matching_lazy(handle_sequence); + self.start_matching_lazy::(handle_sequence); } /// Borrowed (no-copy) equivalent of the lazy `skip_matching`: stage the /// in-place block, then seed positions without an owned-history append. - pub(crate) fn skip_matching_borrowed( + pub(crate) fn skip_matching_borrowed( &mut self, block_start: usize, block_end: usize, incompressible_hint: Option, ) { self.table.stage_borrowed_block(block_start, block_end); - self.table.skip_matching(incompressible_hint); + self.table.skip_matching::(incompressible_hint); } } diff --git a/zstd/src/encoding/hc/generator/tests.rs b/zstd/src/encoding/hc/generator/tests.rs index eed86e4a..532e0597 100644 --- a/zstd/src/encoding/hc/generator/tests.rs +++ b/zstd/src/encoding/hc/generator/tests.rs @@ -6,6 +6,7 @@ use crate::encoding::CompressionLevel; use crate::encoding::Matcher; #[cfg(test)] use crate::encoding::dfast::DfastMatchGenerator; +use crate::encoding::fastpath::with_host_tier; #[cfg(test)] use crate::encoding::match_generator::MatchGeneratorDriver; use crate::encoding::test_support::BlockReplay; @@ -22,13 +23,13 @@ fn dfast_matches_roundtrip_multi_block_pattern() { matcher.commit_input(&first_block); let mut history = Vec::new(); let mut replay = BlockReplay::new(&first_block); - matcher.start_matching(|seq| replay.apply(&mut history, seq)); + with_host_tier!(K => matcher.start_matching::(|seq| replay.apply(&mut history, seq))); assert_eq!(history, first_block); matcher.commit_input(&second_block); let prefix_len = history.len(); let mut replay = BlockReplay::new(&second_block); - matcher.start_matching(|seq| replay.apply(&mut history, seq)); + with_host_tier!(K => matcher.start_matching::(|seq| replay.apply(&mut history, seq))); assert_eq!(&history[prefix_len..], second_block.as_slice()); } @@ -82,7 +83,7 @@ fn dfast_accepts_exact_five_byte_match() { let mut saw_five_byte_match = false; let mut saw_longer_match = false; - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(|seq| { if let Sequence::Triple { offset, match_len, .. } = seq @@ -93,7 +94,7 @@ fn dfast_accepts_exact_five_byte_match() { saw_longer_match = true; } } - }); + })); assert!( saw_five_byte_match, diff --git a/zstd/src/encoding/hc/hc_tests.rs b/zstd/src/encoding/hc/hc_tests.rs index a0322230..f51139b6 100644 --- a/zstd/src/encoding/hc/hc_tests.rs +++ b/zstd/src/encoding/hc/hc_tests.rs @@ -3,6 +3,7 @@ //! helpers, chain-walk self-loop branch, and the lazy-pick //! "next match is better" decline paths. use super::*; +use crate::encoding::fastpath::with_host_tier; use crate::encoding::match_table::storage::MatchTable; fn table_with_history(buf: &[u8]) -> MatchTable { @@ -26,7 +27,7 @@ fn repcode_candidate_returns_none_when_suffix_too_short() { let mut t = table_with_history(b"abc"); t.offset_hist = [1, 2, 3]; // current_idx + HC_MIN_MATCH_LEN > concat.len() → early no-match. - assert!(!HcMatcher::repcode_candidate(&t, 0, 1).is_match()); + assert!(!with_host_tier!(K => HcMatcher::repcode_candidate::(&t, 0, 1)).is_match()); } #[test] @@ -37,7 +38,7 @@ fn repcode_candidate_skips_rep_at_history_boundary() { t.offset_hist = [5, 6, 7]; // No match possible at abs_pos=4 because every rep aims past // history start. - let result = HcMatcher::repcode_candidate(&t, 4, 1); + let result = with_host_tier!(K => HcMatcher::repcode_candidate::(&t, 4, 1)); assert!(!result.is_match(), "no rep can land in-range"); } @@ -46,7 +47,7 @@ fn find_best_match_returns_none_for_short_suffix() { let hc = HcMatcher::new(2, 4, 32); let t = table_with_history(b"abc"); assert!( - !hc.find_best_match::(t.live_history(), false, &t, 0, 1) + !with_host_tier!(K => hc.find_best_match::(t.live_history(), false, &t, 0, 1)) .is_match() ); } @@ -85,7 +86,8 @@ fn hash_chain_candidate_picks_longest_forward_over_shorter_with_backward_room() let hc = HcMatcher::new(2, 16, 64); // The forward winner (idx 12, forward 8) has no backward room. - let c0 = hc.hash_chain_candidate::(t.live_history(), false, &t, 24); + let c0 = + with_host_tier!(K => hc.hash_chain_candidate::(t.live_history(), false, &t, 24)); assert!(c0.is_match(), "forward match must be found"); assert_eq!(c0.match_len, 8, "longest forward match is 8 (idx 12)"); assert_eq!( @@ -135,7 +137,9 @@ fn hash_chain_candidate_forward_ties_keep_first_visited() { t.chain_table_mut()[18 & chain_mask] = HC_EMPTY; let hc = HcMatcher::new(2, 16, 64); - let cand = hc.hash_chain_candidate::(t.live_history(), false, &t, abs_pos); + let cand = with_host_tier!( + K => hc.hash_chain_candidate::(t.live_history(), false, &t, abs_pos) + ); assert!(cand.is_match(), "walk must still produce a match"); assert_eq!( cand.match_len, 8, diff --git a/zstd/src/encoding/hc/mod.rs b/zstd/src/encoding/hc/mod.rs index 52d7a7b6..38429c0c 100644 --- a/zstd/src/encoding/hc/mod.rs +++ b/zstd/src/encoding/hc/mod.rs @@ -20,6 +20,7 @@ pub(crate) mod optimal; pub(crate) mod priceset; use super::cost_model::{HC_FORMAT_MINMATCH, HC_OPT_NUM, HcOptimalCostProfile}; +use super::fastpath::EncodeTier; use super::match_table::helpers::common_prefix_len; use super::match_table::storage::{HC_EMPTY, MatchTable}; use super::opt::types::MatchCandidate; @@ -141,7 +142,11 @@ impl HcMatcher { /// Probe the 3 rep-code offsets (with the upstream zstd `ll0 ↦ rep[0] − 1` /// fallback) and return the best in-range match. Pure helper — /// only reads from `MatchTable`, no HcMatcher state needed. - pub(crate) fn repcode_candidate(table: &MatchTable, abs_pos: usize, lit_len: usize) -> HcMatch { + pub(crate) fn repcode_candidate( + table: &MatchTable, + abs_pos: usize, + lit_len: usize, + ) -> HcMatch { let reps = if lit_len == 0 { [ Some(table.offset_hist[1] as usize), @@ -195,7 +200,8 @@ impl HcMatcher { { continue; } - let match_len = common_prefix_len(&concat[candidate_idx..], &concat[current_idx..]); + let match_len = + common_prefix_len::(&concat[candidate_idx..], &concat[current_idx..]); if match_len >= HC_MIN_MATCH_LEN { best = Self::better_candidate( best, @@ -223,7 +229,7 @@ impl HcMatcher { /// `ZSTD_HcFindBestMatch` shape — reached by a single call per position, not /// the find->chain->dms call chain that an out-of-line chain walk adds. #[inline(always)] - pub(crate) fn hash_chain_candidate( + pub(crate) fn hash_chain_candidate( &self, concat: &[u8], dms_primed: bool, @@ -358,7 +364,7 @@ impl HcMatcher { }; if gate_ok { let current_ml = - common_prefix_len(&concat[candidate_idx..], &concat[current_idx..]); + common_prefix_len::(&concat[candidate_idx..], &concat[current_idx..]); // `currentMl > ml`: strictly longer forward match wins; ties // keep the first (closest, the walk is newest-first). if current_ml > ml { @@ -401,7 +407,7 @@ impl HcMatcher { // dms runs only in the `DICT = true` monomorph, which uses the // `ml >= MIN_MATCH` match test, so the walk tracks `(ml, best_idx)` // and never the `found` flag. - let walk = self.dms_chain_walk( + let walk = self.dms_chain_walk::( table, concat, current_idx, @@ -450,7 +456,7 @@ impl HcMatcher { /// path. #[inline] #[allow(clippy::too_many_arguments)] - fn dms_chain_walk( + fn dms_chain_walk( &self, table: &MatchTable, concat: &[u8], @@ -524,7 +530,8 @@ impl HcMatcher { == MatchTable::read_le_u32_ptr(base_ptr.add(current_idx + gate_off)) }; if gate_ok { - let current_ml = common_prefix_len(&concat[dict_idx..], &concat[current_idx..]); + let current_ml = + common_prefix_len::(&concat[dict_idx..], &concat[current_idx..]); if current_ml > ml { ml = current_ml; best_idx = dict_idx; @@ -557,7 +564,7 @@ impl HcMatcher { /// folded into the (out-of-line) `find_best_match` monolith so the rep is /// part of the single per-position find call, not a nested out-of-line probe. #[inline(always)] - pub(crate) fn repcode_candidate_offset1( + pub(crate) fn repcode_candidate_offset1( concat: &[u8], table: &MatchTable, abs_pos: usize, @@ -590,7 +597,7 @@ impl HcMatcher { { return HcMatch::NONE; } - let match_len = common_prefix_len(&concat[candidate_idx..], &concat[current_idx..]); + let match_len = common_prefix_len::(&concat[candidate_idx..], &concat[current_idx..]); if match_len >= HC_MIN_MATCH_LEN { HcMatch { offset: rep, @@ -616,7 +623,7 @@ impl HcMatcher { /// `rax:rdx`, so the find->loop boundary stays a register pair, not a stack /// spill. #[inline(never)] - pub(crate) fn find_best_match( + pub(crate) fn find_best_match( &self, concat: &[u8], dms_primed: bool, @@ -637,11 +644,11 @@ impl HcMatcher { // The `lit_len == 0` case keeps the full rotated probe for encode // correctness (the ll0 `rep[0]-1` rule). let rep = if lit_len == 0 { - Self::repcode_candidate(table, abs_pos, lit_len) + Self::repcode_candidate::(table, abs_pos, lit_len) } else { - Self::repcode_candidate_offset1(concat, table, abs_pos) + Self::repcode_candidate_offset1::(concat, table, abs_pos) }; - let hash = self.hash_chain_candidate::(concat, dms_primed, table, abs_pos); + let hash = self.hash_chain_candidate::(concat, dms_primed, table, abs_pos); Self::better_candidate(rep, hash) } @@ -662,7 +669,7 @@ impl HcMatcher { /// the `pos + 1` / `pos + 2` lookahead sees `abs_pos` at offset 1 — matching /// upstream, where the searched position is inserted during its own search, /// before the depth loop probes the next one. - pub(crate) fn lazy_decide_carry( + pub(crate) fn lazy_decide_carry( &self, concat: &[u8], dms_primed: bool, @@ -689,7 +696,7 @@ impl HcMatcher { history_end = history_end, min_match = HC_MIN_MATCH_LEN, search = |pos, ll| { - let m = self.find_best_match::(concat, dms_primed, table, pos, ll); + let m = self.find_best_match::(concat, dms_primed, table, pos, ll); m.is_match().then_some(m) }, ) { @@ -709,7 +716,7 @@ impl HcMatcher { #[allow(dead_code)] #[inline(always)] #[allow(clippy::too_many_arguments)] - pub(crate) fn for_each_repcode_candidate_with_reps( + pub(crate) fn for_each_repcode_candidate_with_reps( &self, table: &MatchTable, abs_pos: usize, @@ -740,7 +747,7 @@ impl HcMatcher { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match table.kernel { + match K::TIER { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { self.for_each_repcode_candidate_with_reps_avx2_bmi2( diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index 45d0a293..bbb12665 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -16,6 +16,7 @@ use super::generator::HcMatchGenerator; use crate::encoding::Sequence; use crate::encoding::blocks::encode_offset_with_history; +use crate::encoding::fastpath::EncodeTier; use crate::encoding::hc::HC_OPT_CANDIDATE_CAPACITY; use crate::encoding::hc::priceset::{ priceset_avx2, priceset_neon, priceset_scalar, priceset_simd128, priceset_sse2, priceset_sse41, @@ -1306,7 +1307,7 @@ impl HcMatchGenerator { // The scalar arm of `run_main_loop!` wraps a safe `*_scalar` call in // `unsafe` (the SIMD arms need it); allow the redundant block here. #[allow(unused_unsafe)] - pub(crate) fn start_matching_optimal( + pub(crate) fn start_matching_optimal( &mut self, mut handle_sequence: impl FnMut(Sequence), ) { @@ -1335,7 +1336,7 @@ impl HcMatchGenerator { .skip_insert_until_abs .max(self.table.history_abs_start); self.table - .backfill_boundary_positions(current_abs_start, current_abs_end); + .backfill_boundary_positions::(current_abs_start, current_abs_end); self.table.next_to_update3 = hash3_start_cursor; // Take the five DP scratch buffers ONCE for both the seed pass and the @@ -1344,7 +1345,12 @@ impl HcMatchGenerator { // separate take + restore round-trip. let mut plan_buffers = self.take_optimal_plan_buffers(); if self.should_run_btultra2_seed_pass::(current_len) { - self.run_btultra2_seed_pass(current, current_abs_start, current_len, &mut plan_buffers); + self.run_btultra2_seed_pass::( + current, + current_abs_start, + current_len, + &mut plan_buffers, + ); // The seed pass pushes the stored-index offset past everything it // inserted, so the block has to be cleared again for the main pass // against the offset it leaves behind. @@ -1498,7 +1504,7 @@ impl HcMatchGenerator { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match self.table.kernel { + match K::TIER { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => run_main_loop!(run_optimal_block_avx2_bmi2), #[cfg(feature = "kernel-sse")] @@ -1547,7 +1553,7 @@ impl HcMatchGenerator { // The scalar arm of `run_seed_loop!` wraps a safe call in `unsafe`. #[allow(unused_unsafe)] - fn run_btultra2_seed_pass( + fn run_btultra2_seed_pass( &mut self, current: &[u8], current_abs_start: usize, @@ -1699,7 +1705,7 @@ impl HcMatchGenerator { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match self.table.kernel { + match K::TIER { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => run_seed_loop!(run_optimal_block_avx2_bmi2), #[cfg(feature = "kernel-sse")] @@ -2085,7 +2091,7 @@ impl HcMatchGenerator { } #[cfg(test)] - pub(crate) fn collect_optimal_candidates( + pub(crate) fn collect_optimal_candidates( &mut self, abs_pos: usize, current_abs_end: usize, @@ -2115,7 +2121,7 @@ impl HcMatchGenerator { macro_rules! search { ($strategy:ty) => { match self.table.search_mls { - 3 => self.collect_optimal_candidates_initialized::<$strategy, 3>( + 3 => self.collect_optimal_candidates_initialized::<$strategy, K, 3>( abs_pos, current_abs_end, sufficient_match_len, @@ -2123,7 +2129,7 @@ impl HcMatchGenerator { ll0, out, ), - 5 => self.collect_optimal_candidates_initialized::<$strategy, 5>( + 5 => self.collect_optimal_candidates_initialized::<$strategy, K, 5>( abs_pos, current_abs_end, sufficient_match_len, @@ -2131,7 +2137,7 @@ impl HcMatchGenerator { ll0, out, ), - 6 => self.collect_optimal_candidates_initialized::<$strategy, 6>( + 6 => self.collect_optimal_candidates_initialized::<$strategy, K, 6>( abs_pos, current_abs_end, sufficient_match_len, @@ -2139,7 +2145,7 @@ impl HcMatchGenerator { ll0, out, ), - _ => self.collect_optimal_candidates_initialized::<$strategy, 4>( + _ => self.collect_optimal_candidates_initialized::<$strategy, K, 4>( abs_pos, current_abs_end, sufficient_match_len, @@ -2196,6 +2202,7 @@ impl HcMatchGenerator { #[inline(always)] pub(crate) fn collect_optimal_candidates_initialized< S: crate::encoding::strategy::Strategy, + K: EncodeTier, const MLS: usize, >( &mut self, @@ -2224,7 +2231,7 @@ impl HcMatchGenerator { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match self.table.kernel { + match K::TIER { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { self.collect_optimal_candidates_initialized_avx2_bmi2::( diff --git a/zstd/src/encoding/ldm/mod.rs b/zstd/src/encoding/ldm/mod.rs index 07179b8e..e375a6dc 100644 --- a/zstd/src/encoding/ldm/mod.rs +++ b/zstd/src/encoding/ldm/mod.rs @@ -238,7 +238,7 @@ impl LdmProducer { /// The producer covers the **prefix-only** path; the /// two-segment `extDict` variant is documented and deferred /// in [`search`]. - pub(crate) fn generate_into( + pub(crate) fn generate_into( &mut self, live_history: &[u8], history_abs_start: usize, @@ -361,7 +361,7 @@ impl LdmProducer { // mechanism becomes load-bearing only when the // caller drives multiple compress_block windows // through the same producer. - let best = find_best_match( + let best = find_best_match::( &self.hash_table, hash_id, checksum, diff --git a/zstd/src/encoding/ldm/search.rs b/zstd/src/encoding/ldm/search.rs index 782a4c3d..2b1b43c3 100644 --- a/zstd/src/encoding/ldm/search.rs +++ b/zstd/src/encoding/ldm/search.rs @@ -165,7 +165,7 @@ pub(crate) struct FindBestMatchInputs<'a> { /// [`LdmHashTable::bucket_mask`]. All positions in /// [`FindBestMatchInputs`] are absolute stream coordinates; the /// returned [`LdmMatch::match_pos`] is also absolute. -pub(crate) fn find_best_match( +pub(crate) fn find_best_match( table: &LdmHashTable, hash_id: u32, checksum: u32, @@ -252,7 +252,7 @@ pub(crate) fn find_best_match( // history before the current block are legitimate (upstream zstd // `pMatch < iend` is automatically true when the entry's // absolute offset is below `iend_abs`). - let forward_len = common_prefix_len( + let forward_len = common_prefix_len::( &live_history[split_idx..split_idx_end], &live_history[match_idx..], ); diff --git a/zstd/src/encoding/ldm/search/tests.rs b/zstd/src/encoding/ldm/search/tests.rs index 592de03a..0412e8dd 100644 --- a/zstd/src/encoding/ldm/search/tests.rs +++ b/zstd/src/encoding/ldm/search/tests.rs @@ -7,6 +7,18 @@ fn fresh_table() -> LdmHashTable { LdmHashTable::new(4, 2) } +/// [`find_best_match`] on this CPU's kernel tier. +fn find_best( + table: &LdmHashTable, + hash_id: u32, + checksum: u32, + inputs: FindBestMatchInputs<'_>, +) -> Option { + crate::encoding::fastpath::with_host_tier!( + K => find_best_match::(table, hash_id, checksum, inputs) + ) +} + /// `count_backwards_match` honours both lower bounds and /// stops on the first mismatch. Fixture: "XXXabc" matches /// "YYYabc" with 3 backward bytes from offset 3 in each. @@ -47,7 +59,7 @@ fn find_best_match_returns_none_on_checksum_mismatch() { let mut table = fresh_table(); table.insert_absolute(1, 4, 0x1111_1111); let history = b"abcdefghabcdefgh"; - let m = find_best_match( + let m = find_best( &table, 1, 0xDEAD_BEEF, @@ -77,7 +89,7 @@ fn find_best_match_rejects_stale_entries() { // → rejected. (At lowest_index_abs = 4 the entry would // exactly meet the floor and survive — that's the edge // case the inclusive bound is designed to preserve.) - let m = find_best_match( + let m = find_best( &table, 1, 0xCAFE, @@ -108,7 +120,7 @@ fn find_best_match_picks_longest_combined_match() { let history = b"PPPPabcdefghabcdefgh"; // split at position 12, anchor at 12 → no backward room. // The forward count should match 8 bytes ("abcdefgh"). - let m = find_best_match( + let m = find_best( &table, 1, 0xCAFE, @@ -141,7 +153,7 @@ fn find_best_match_extends_backwards_into_pre_split_bytes() { // split at 12 (start of second "abcdefgh"), anchor at 10 // → backward up to 2 bytes ("XY" at positions 10..12 vs // 0..2). Forward count: 8 bytes ("abcdefgh"). - let m = find_best_match( + let m = find_best( &table, 1, 0xCAFE, @@ -178,7 +190,7 @@ fn find_best_match_prefers_longer_total_across_slots() { // offset 8 ("abcdefgh") gives 8 bytes forward; match at // offset 4 ("abcdabcd...") gives only 4 bytes forward // because the 5th byte ('a' vs 'e') mismatches. - let m = find_best_match( + let m = find_best( &table, 1, 0xCAFE, @@ -210,7 +222,7 @@ fn find_best_match_rejects_entries_at_or_past_split() { // direct caller / test fixture / future extDict variant. table.insert_absolute(1, 12, 0xCAFE); let history = b"PPPPabcdefghabcdefgh"; - let m = find_best_match( + let m = find_best( &table, 1, 0xCAFE, @@ -230,7 +242,7 @@ fn find_best_match_rejects_entries_at_or_past_split() { // must be rejected. let mut table_after = fresh_table(); table_after.insert_absolute(1, 16, 0xCAFE); - let m_after = find_best_match( + let m_after = find_best( &table_after, 1, 0xCAFE, @@ -261,7 +273,7 @@ fn find_best_match_forward_count_is_bounded_by_iend_abs() { // iend_abs = 16 (4 bytes past the split) the match must // cap at 4. let history = b"PPPPabcdefghabcdefgh"; - let m = find_best_match( + let m = find_best( &table, 1, 0xCAFE, @@ -291,7 +303,7 @@ fn find_best_match_filters_short_forward_matches() { let history = b"PPPPabXXXXXXab"; // 2-byte forward match from split=12 vs match=4, but // min_match_length = 4 → rejected. - let m = find_best_match( + let m = find_best( &table, 1, 0xCAFE, diff --git a/zstd/src/encoding/ldm/tests.rs b/zstd/src/encoding/ldm/tests.rs index cdeea4b8..510bbda9 100644 --- a/zstd/src/encoding/ldm/tests.rs +++ b/zstd/src/encoding/ldm/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::encoding::fastpath::with_host_tier; /// `LdmParams::adjust_for` must derive a representative /// upstream zstd-btultra2 parameter set at window_log=27. Checked via @@ -45,7 +46,7 @@ fn clear_resets_rolling_hash_state() { let mut out = Vec::new(); // Feed a non-empty chunk so the rolling hash advances. let data = [0xAAu8; 256]; - producer.generate_into(&data, 0, 0, data.len(), &mut out); + with_host_tier!(K => producer.generate_into::(&data, 0, 0, data.len(), &mut out)); let advanced = producer.hash_state.rolling; assert_ne!( advanced, @@ -106,7 +107,7 @@ fn generate_into_emits_long_range_match_on_repeated_payload() { history.extend_from_slice(&payload); let mut out = Vec::new(); - producer.generate_into(&history, 0, 0, history.len(), &mut out); + with_host_tier!(K => producer.generate_into::(&history, 0, 0, history.len(), &mut out)); assert!( !out.is_empty(), @@ -199,7 +200,7 @@ fn generate_into_preserves_bucket_entries_across_history_slide() { // populates with absolute offsets inside payload #1. let mut out1 = Vec::new(); let split_at = PAYLOAD + GAP_A; - producer.generate_into(&frame[..split_at], 0, 0, split_at, &mut out1); + with_host_tier!(K => producer.generate_into::(&frame[..split_at], 0, 0, split_at, &mut out1)); // Call 2: simulate a window slide. The encoder retired // the leading `eviction` bytes from the live history; @@ -215,13 +216,13 @@ fn generate_into_preserves_bucket_entries_across_history_slide() { let block_end_abs = block_start_abs + PAYLOAD; let mut out2 = Vec::new(); - producer.generate_into( + with_host_tier!(K => producer.generate_into::( live, history_abs_start, block_start_abs, block_end_abs, &mut out2, - ); + )); // The long-range match must still fire — entries from // call 1 in the surviving tail of payload #1 @@ -288,7 +289,7 @@ fn generate_into_empty_range_is_noop() { let mut out = Vec::new(); let data = [0u8; 128]; let pre = producer.hash_state.rolling; - producer.generate_into(&data, 0, 64, 64, &mut out); + with_host_tier!(K => producer.generate_into::(&data, 0, 64, 64, &mut out)); assert!(out.is_empty()); assert_eq!(producer.hash_state.rolling, pre); } diff --git a/zstd/src/encoding/levels/fastest.rs b/zstd/src/encoding/levels/fastest.rs index 910d222b..53672895 100644 --- a/zstd/src/encoding/levels/fastest.rs +++ b/zstd/src/encoding/levels/fastest.rs @@ -8,6 +8,7 @@ use crate::{ commit_block_tables, compress_block, compress_block_with_post_split, discard_block_tables, }, + fastpath::EncodeTier, frame_compressor::CompressState, incompressible::{ block_looks_incompressible, block_looks_incompressible_strict, @@ -40,7 +41,7 @@ use alloc::vec::Vec; // emit-fn shape for no readability gain. #[allow(clippy::too_many_arguments)] #[inline] -pub(crate) fn compress_block_encoded( +pub(crate) fn compress_block_encoded( state: &mut CompressState, compression_level: CompressionLevel, last_block: bool, @@ -168,7 +169,7 @@ pub(crate) fn compress_block_encoded( sink.push(precomputed_checksum.expect("checksum is hashed whenever a sink exists")); } state.matcher.commit_filled(block_len); - state.matcher.skip_matching_with_hint(Some(false)); + skip_block::(state, false); record_unsearched_block(state, block_size); let header = BlockHeader { last_block, @@ -189,7 +190,7 @@ pub(crate) fn compress_block_encoded( sink.push(precomputed_checksum.expect("checksum is hashed whenever a sink exists")); } state.matcher.commit_filled(block_len); - state.matcher.skip_matching_with_hint(Some(true)); + skip_block::(state, true); record_unsearched_block(state, block_size); let header = BlockHeader { last_block, @@ -209,7 +210,7 @@ pub(crate) fn compress_block_encoded( // loop so the cardinality matches the decoder's per-block count // exactly. #[cfg(all(feature = "lsm", feature = "hash"))] - compress_block_with_post_split( + compress_block_with_post_split::( state, last_block, output, @@ -217,9 +218,14 @@ pub(crate) fn compress_block_encoded( block_checksums, ); #[cfg(all(feature = "lsm", not(feature = "hash")))] - compress_block_with_post_split(state, last_block, output, block_decompressed_sizes); + compress_block_with_post_split::( + state, + last_block, + output, + block_decompressed_sizes, + ); #[cfg(not(feature = "lsm"))] - compress_block_with_post_split(state, last_block, output); + compress_block_with_post_split::(state, last_block, output); return BlockType::Compressed; } #[cfg(feature = "lsm")] @@ -244,7 +250,7 @@ pub(crate) fn compress_block_encoded( let hdr_off = output.len(); output.extend_from_slice(&[0u8; 3]); let payload_off = output.len(); - let tables = compress_block(state, output); + let tables = compress_block::(state, output); let payload_len = output.len() - payload_off; // Fall back to a raw block when the compressed payload is not // smaller than the source (`payload >= block_size`) or exceeds the @@ -285,6 +291,17 @@ pub(crate) fn compress_block_encoded( } } +/// Index the committed block without searching it, under the frame's tier `K` +/// when the matcher is the built-in one (which still searches it for +/// long-distance matches). +#[inline] +fn skip_block(state: &mut CompressState, incompressible: bool) { + match state.matcher.builtin_driver() { + Some(driver) => driver.skip_matching_with_hint_with::(Some(incompressible)), + None => state.matcher.skip_matching_with_hint(Some(incompressible)), + } +} + /// A block written out unsearched parses as literals alone: one delimiter /// carrying all of it, when the parse is being recorded. #[inline] @@ -311,7 +328,7 @@ fn record_unsearched_block(state: &mut CompressState, block_size: /// strategy, and the borrowed path is gated to the fast one), so it is /// omitted. #[allow(clippy::too_many_arguments)] -pub(crate) fn compress_block_encoded_borrowed( +pub(crate) fn compress_block_encoded_borrowed( state: &mut CompressState, compression_level: CompressionLevel, last_block: bool, @@ -390,7 +407,7 @@ pub(crate) fn compress_block_encoded_borrowed( sink.push(crate::encoding::frame_compressor::xxh64_block_low32(block)); } state.matcher.set_borrowed_block(block_start, block_end); - state.matcher.skip_matching_with_hint(Some(false)); + state.matcher.skip_matching_with_hint_with::(Some(false)); record_unsearched_block(state, block_size); let header = BlockHeader { last_block, @@ -410,7 +427,7 @@ pub(crate) fn compress_block_encoded_borrowed( sink.push(crate::encoding::frame_compressor::xxh64_block_low32(block)); } state.matcher.set_borrowed_block(block_start, block_end); - state.matcher.skip_matching_with_hint(Some(true)); + state.matcher.skip_matching_with_hint_with::(Some(true)); record_unsearched_block(state, block_size); let header = BlockHeader { last_block, @@ -452,7 +469,7 @@ pub(crate) fn compress_block_encoded_borrowed( let hdr_off = output.len(); output.extend_from_slice(&[0u8; 3]); let payload_off = output.len(); - let tables = compress_block(state, output); + let tables = compress_block::(state, output); let payload_len = output.len() - payload_off; if payload_len >= MAX_BLOCK_SIZE as usize || payload_len >= block_size as usize { // Incompressible (compressed payload not smaller than the source, diff --git a/zstd/src/encoding/levels/fastest/tests.rs b/zstd/src/encoding/levels/fastest/tests.rs index c0063c68..e8a3200b 100644 --- a/zstd/src/encoding/levels/fastest/tests.rs +++ b/zstd/src/encoding/levels/fastest/tests.rs @@ -92,7 +92,7 @@ fn rle_branch_passes_compressible_hint_to_skip_matching() { (block.len(), false) }); - let emitted = compress_block_encoded( + let emitted = crate::encoding::fastpath::with_host_tier!(K => compress_block_encoded::<_, K>( &mut state, CompressionLevel::Fastest, true, @@ -103,7 +103,7 @@ fn rle_branch_passes_compressible_hint_to_skip_matching() { None, #[cfg(all(feature = "lsm", feature = "hash"))] None, - ); + )); assert_eq!(emitted, BlockType::RLE); assert_eq!( diff --git a/zstd/src/encoding/match_generator/dict_prime.rs b/zstd/src/encoding/match_generator/dict_prime.rs index dff5d4bf..fc964250 100644 --- a/zstd/src/encoding/match_generator/dict_prime.rs +++ b/zstd/src/encoding/match_generator/dict_prime.rs @@ -66,7 +66,9 @@ impl MatchGeneratorDriver { self.index_dictionary_in_place(); } - pub(super) fn prime_with_dictionary_impl( + /// [`crate::encoding::Matcher::prime_with_dictionary`] compiled for the kernel tier `K`, the one the encoder chose for the + /// frame. + pub(crate) fn prime_with_dictionary_with( &mut self, dict_content: &[u8], offset_hist: [u32; 3], @@ -244,7 +246,7 @@ impl MatchGeneratorDriver { (chunk.len(), false) }); self.commit_filled(chunk.len()); - self.skip_matching_for_dictionary_priming(dict_content.len()); + self.skip_matching_for_dictionary_priming::(dict_content.len()); committed_dict_budget += end - start; start = end; } diff --git a/zstd/src/encoding/match_generator/long_distance.rs b/zstd/src/encoding/match_generator/long_distance.rs index 757cdb17..323ee7f1 100644 --- a/zstd/src/encoding/match_generator/long_distance.rs +++ b/zstd/src/encoding/match_generator/long_distance.rs @@ -6,6 +6,7 @@ //! storage directly. use super::{MatchGeneratorDriver, Sequence, owned_sink}; +use crate::encoding::fastpath::EncodeTier; impl MatchGeneratorDriver { /// Search the input from where the last search stopped up to `to`, a @@ -17,7 +18,7 @@ impl MatchGeneratorDriver { /// window back from it, or the whole dictionary while the window has not /// yet passed the dictionary's end. Every match then holds an offset valid /// at its last byte, wherever in the range it lies. - fn search_long_distance_to(&mut self, to: usize) { + fn search_long_distance_to(&mut self, to: usize) { let Some(producer) = self.ldm.as_mut() else { return; }; @@ -36,7 +37,7 @@ impl MatchGeneratorDriver { let floor = reach_floor.max(live_start); debug_assert!(floor <= from, "a search range wider than the window"); self.ldm_found.clear(); - producer.generate_into( + producer.generate_into::( &live[floor - live_start..], floor, from, @@ -56,11 +57,11 @@ impl MatchGeneratorDriver { /// The matches of the last committed block, into [`Self::ldm_block`], /// searched first as far as the block reaches. Returns the block's length. - pub(super) fn take_long_distance_block(&mut self) -> usize { + pub(super) fn take_long_distance_block(&mut self) -> usize { let block_len = self.storage.last_block_len(); let block_end = self.ldm_committed_end; self.align_long_distance_store(block_end - block_len); - self.search_long_distance_to(block_end); + self.search_long_distance_to::(block_end); let min_match = self.reset_shape.map_or( crate::encoding::cost_model::HC_FORMAT_MINMATCH, |(params, ..)| params.min_match(), @@ -73,13 +74,17 @@ impl MatchGeneratorDriver { /// See [`crate::encoding::Matcher::settle_block_cut`]: the next `window` /// uncommitted bytes are searched now, and the cut moved out of any long /// match. - pub(super) fn settle_long_distance_cut(&mut self, window: usize, cut: usize) -> usize { + pub(super) fn settle_long_distance_cut( + &mut self, + window: usize, + cut: usize, + ) -> usize { if self.ldm.is_none() { return cut; } let start = self.ldm_committed_end; self.align_long_distance_store(start); - self.search_long_distance_to(start + window); + self.search_long_distance_to::(start + window); self.ldm_store.cut_outside_matches(window, cut) } @@ -93,12 +98,12 @@ impl MatchGeneratorDriver { /// ordinary search. With `produced`, the caller has emptied the optimal /// parser's candidates first, so a piece the producer fails on is never /// matched against an earlier block's long matches. - pub(super) fn start_matching_long_distance( + pub(super) fn start_matching_long_distance( &mut self, handle_sequence: &mut impl FnMut(Sequence), produced: bool, ) -> bool { - let block_len = self.take_long_distance_block(); + let block_len = self.take_long_distance_block::(); if self.ldm_block.is_empty() { return false; } @@ -110,11 +115,11 @@ impl MatchGeneratorDriver { let mut gap = Some(0usize); if seq.lit_length != 0 { self.storage.commit_piece(seq.lit_length); - self.match_gap(produced, &mut owned_sink(handle_sequence, &mut gap)); + self.match_gap::(produced, &mut owned_sink(handle_sequence, &mut gap)); } let gap_literals = gap.expect("the gap sink only ever writes Some"); self.storage.commit_piece(seq.match_length); - self.storage.skip_long_match(); + self.storage.skip_long_match::(); handle_sequence(Sequence::Triple { literal_len: gap_literals, offset: seq.offset, @@ -127,7 +132,7 @@ impl MatchGeneratorDriver { if tail != 0 { self.storage.commit_piece(tail); let mut forward = None; - self.match_gap(produced, &mut owned_sink(handle_sequence, &mut forward)); + self.match_gap::(produced, &mut owned_sink(handle_sequence, &mut forward)); } self.ldm_block = sequences; self.ldm_split_block = Some(block_len); @@ -136,10 +141,10 @@ impl MatchGeneratorDriver { /// Match the piece just committed between long matches: by the producer /// when `produced` and it gives a parse, by the backend otherwise. - fn match_gap(&mut self, produced: bool, sink: &mut impl FnMut(Sequence)) { - if produced && self.match_produced(sink) { + fn match_gap(&mut self, produced: bool, sink: &mut impl FnMut(Sequence)) { + if produced && self.match_produced::(sink) { return; } - self.start_matching_owned(sink); + self.start_matching_owned::(sink); } } diff --git a/zstd/src/encoding/match_generator/mod.rs b/zstd/src/encoding/match_generator/mod.rs index 1f6aaa22..95d347b5 100644 --- a/zstd/src/encoding/match_generator/mod.rs +++ b/zstd/src/encoding/match_generator/mod.rs @@ -22,6 +22,7 @@ use super::cost_model::{ #[cfg(test)] use super::cost_model::{HC_BLOCKSIZE_MAX, HC_MAX_LL, HC_MAX_ML, HC_MAX_OFF, HcOptPriceType}; use super::dfast::DfastMatchGenerator; +use super::fastpath::EncodeTier; #[cfg(test)] use super::hc::HC_MIN_MATCH_LEN; #[cfg(test)] @@ -315,12 +316,12 @@ impl MatcherStorage { /// See the backends' `skip_long_match`. #[cfg(feature = "ldm")] - fn skip_long_match(&mut self) { + fn skip_long_match(&mut self) { match self { Self::Simple(m) => m.skip_long_match(), Self::Dfast(m) => m.skip_long_match(), Self::Row(m) => m.skip_long_match(), - Self::HashChain(m) => m.table.skip_long_match(), + Self::HashChain(m) => m.table.skip_long_match::(), } } @@ -838,7 +839,10 @@ impl MatchGeneratorDriver { /// piece of one between long matches, indexing it as an unsearched one. /// Returns `false`, having done nothing, when the producer failed and the /// backend is to match it. - fn match_produced(&mut self, handle_sequence: &mut impl FnMut(Sequence)) -> bool { + fn match_produced( + &mut self, + handle_sequence: &mut impl FnMut(Sequence), + ) -> bool { let mut slot = self .producer .take() @@ -871,7 +875,7 @@ impl MatchGeneratorDriver { return false; } // The block's long matches, if any, were taken before it got here. - self.skip_backend(None); + self.skip_backend::(None); if produced { let mut literals = 0usize; for sequence in &slot.parse { @@ -1272,7 +1276,7 @@ impl MatchGeneratorDriver { .is_none_or(|log| log <= DFAST_ATTACH_DICT_CUTOFF_LOG) } - fn skip_matching_for_dictionary_priming(&mut self, dict_len: usize) { + fn skip_matching_for_dictionary_priming(&mut self, dict_len: usize) { match self.active_backend() { super::strategy::BackendTag::Simple => { // Upstream zstd `ZSTD_shouldAttachDict` mode selection for the Fast @@ -1343,7 +1347,7 @@ impl MatchGeneratorDriver { if self.hc_dict_attach_mode() { self.hc_matcher_mut().table.skip_matching_dict_bt(); } else { - self.hc_matcher_mut().skip_matching(Some(false)); + self.hc_matcher_mut().skip_matching::(Some(false)); } } } @@ -2105,7 +2109,12 @@ impl Matcher for MatchGeneratorDriver { #[inline] fn prime_with_dictionary(&mut self, dict_content: &[u8], offset_hist: [u32; 3]) { - self.prime_with_dictionary_impl(dict_content, offset_hist) + // For a caller driving the matcher through the trait; the encoder + // calls `prime_with_dictionary_with` under its frame's tier. + crate::encoding::fastpath::with_encode_tier!( + crate::encoding::fastpath::select_kernel(), + K => self.prime_with_dictionary_with::(dict_content, offset_hist) + ) } #[inline] @@ -2164,13 +2173,12 @@ impl Matcher for MatchGeneratorDriver { } fn settle_block_cut(&mut self, window: usize, cut: usize) -> usize { - #[cfg(feature = "ldm")] - return self.settle_long_distance_cut(window, cut); - #[cfg(not(feature = "ldm"))] - { - let _ = window; - cut - } + // For a caller driving the matcher through the trait; the encoder + // calls `settle_block_cut_with` under its frame's tier. + crate::encoding::fastpath::with_encode_tier!( + crate::encoding::fastpath::select_kernel(), + K => self.settle_block_cut_with::(window, cut) + ) } fn get_last_space(&mut self) -> &[u8] { @@ -2260,7 +2268,137 @@ impl Matcher for MatchGeneratorDriver { } } - fn start_matching(&mut self, mut handle_sequence: impl FnMut(Sequence)) { + fn start_matching(&mut self, handle_sequence: impl FnMut(Sequence)) { + // For a caller driving the matcher through the trait itself. The + // encoder does not come here: it reaches the driver through + // `builtin_driver` and searches under the tier it chose for the frame. + crate::encoding::fastpath::with_encode_tier!( + crate::encoding::fastpath::select_kernel(), + K => self.start_matching_with::(handle_sequence) + ) + } + + #[inline] + fn builtin_driver(&mut self) -> Option<&mut MatchGeneratorDriver> { + Some(self) + } + + fn skip_matching(&mut self) { + self.skip_matching_with_hint(None); + } + + fn skip_matching_with_hint(&mut self, incompressible_hint: Option) { + // For a caller driving the matcher through the trait; the encoder + // calls `skip_matching_with_hint_with` under its frame's tier. + crate::encoding::fastpath::with_encode_tier!( + crate::encoding::fastpath::select_kernel(), + K => self.skip_matching_with_hint_with::(incompressible_hint) + ) + } +} + +impl MatchGeneratorDriver { + /// [`Matcher::skip_matching_with_hint`] compiled for the kernel tier `K`: + /// the block is still searched for long-distance matches. + pub(crate) fn skip_matching_with_hint_with( + &mut self, + incompressible_hint: Option, + ) { + // Borrowed one-shot Fast path: a staged block range routes to the + // borrowed skip (records the range for `get_last_space`, primes + // hashes on the dict-priming hint) with no owned-history append + // and nothing to recycle. Stage is consumed. + if let Some((block_start, block_end)) = self.borrowed_pending.take() { + match self.active_backend() { + super::strategy::BackendTag::Simple => self.simple_mut().skip_matching_borrowed( + block_start, + block_end, + incompressible_hint, + ), + super::strategy::BackendTag::Dfast => self + .dfast_matcher_mut() + .skip_matching_borrowed(block_start, block_end, incompressible_hint), + super::strategy::BackendTag::Row => self.row_matcher_mut().skip_matching_borrowed( + block_start, + block_end, + incompressible_hint, + ), + super::strategy::BackendTag::HashChain => self + .hc_matcher_mut() + .skip_matching_borrowed::(block_start, block_end, incompressible_hint), + } + return; + } + // A block written out unsearched is still searched for long matches, + // as upstream searches every block it compresses, and its matches are + // taken past. + #[cfg(feature = "ldm")] + if self.ldm.is_some() { + self.take_long_distance_block::(); + } + self.skip_backend::(incompressible_hint); + } +} + +impl MatchGeneratorDriver { + /// Index the last committed block (or piece of one) in the backend + /// without searching it, leaving the long-distance state alone. + fn skip_backend(&mut self, incompressible_hint: Option) { + match self.active_backend() { + super::strategy::BackendTag::Simple => { + self.simple_mut() + .skip_matching_with_hint(incompressible_hint); + } + super::strategy::BackendTag::Dfast => { + self.dfast_matcher_mut().skip_matching(incompressible_hint) + } + super::strategy::BackendTag::Row => self + .row_matcher_mut() + .skip_matching_with_hint(incompressible_hint), + super::strategy::BackendTag::HashChain => self + .hc_matcher_mut() + .skip_matching::(incompressible_hint), + } + } +} + +/// The sink every owned match runs against: `handle_sequence`, except that a +/// block's trailing literals go to `gap_literals` when it is `Some`, as the +/// gap before a long-distance match needs them. One sink type for the plain +/// block and for those gaps compiles each backend's scan once per caller sink +/// instead of twice. +fn owned_sink<'a, F: FnMut(Sequence)>( + handle_sequence: &'a mut F, + gap_literals: &'a mut Option, +) -> impl FnMut(Sequence) + 'a { + // The guard is read only for the block's last report, never per triple. + // Each arm rebuilds the report from its fields: forwarding the matched + // value whole copies it as one 32-byte block out of the separate stores + // that wrote it, and that copy stalls on every triple. + move |sequence| match sequence { + Sequence::Triple { + literal_len, + offset, + match_len, + } => handle_sequence(Sequence::Triple { + literal_len, + offset, + match_len, + }), + Sequence::Literals { len } => match gap_literals { + Some(gap) => *gap = len, + None => handle_sequence(Sequence::Literals { len }), + }, + } +} + +impl MatchGeneratorDriver { + /// [`Matcher::start_matching`] compiled for the kernel tier `K`, the one + /// the encoder chose for the frame. Nothing below asks for a tier. + pub(crate) fn start_matching_with( + &mut self, + mut handle_sequence: impl FnMut(Sequence), + ) { // Borrowed one-shot Fast path: if the frame driver staged a // block range via `set_borrowed_block`, scan it in place against // the borrowed window instead of the owned committed block. Only @@ -2278,18 +2416,18 @@ impl Matcher for MatchGeneratorDriver { // Dict-attach borrowed scan: live matches read the // borrowed input in place, dict matches read the // committed dict prefix via the 2-segment counter. - m.start_matching_borrowed_dict(block_start, block_end, &mut sink); + m.start_matching_borrowed_dict::(block_start, block_end, &mut sink); } else { m.start_matching_borrowed(block_start, block_end, &mut sink); } } super::strategy::BackendTag::Dfast => self .dfast_matcher_mut() - .start_matching_borrowed(block_start, block_end, &mut sink), + .start_matching_borrowed::(block_start, block_end, &mut sink), super::strategy::BackendTag::Row => { // Same greedy/lazy parse split as the owned RowHash arm. let greedy = self.parse == super::strategy::ParseMode::Greedy; - self.row_matcher_mut().start_matching_borrowed( + self.row_matcher_mut().start_matching_borrowed::( block_start, block_end, greedy, @@ -2299,7 +2437,7 @@ impl Matcher for MatchGeneratorDriver { super::strategy::BackendTag::HashChain => match self.search { super::strategy::SearchMethod::HashChain => self .hc_matcher_mut() - .start_matching_lazy_borrowed(block_start, block_end, &mut sink), + .start_matching_lazy_borrowed::(block_start, block_end, &mut sink), // `borrowed_supported()` keeps the optimal parsers on the // owned path and `set_borrowed_block` asserts it. other => { @@ -2328,19 +2466,22 @@ impl Matcher for MatchGeneratorDriver { }; hc.backend.bt_mut().ldm_sequences.clear(); } - if self.start_matching_long_distance(&mut handle_sequence, true) { + if self.start_matching_long_distance::(&mut handle_sequence, true) { return; } - if self.match_produced(&mut handle_sequence) { + if self.match_produced::(&mut handle_sequence) { return; } // The producer failed with fallback on a block with no long // match, which is already taken: the backend matches it. let mut gap_literals = None; - self.start_matching_owned(&mut owned_sink(&mut handle_sequence, &mut gap_literals)); + self.start_matching_owned::(&mut owned_sink( + &mut handle_sequence, + &mut gap_literals, + )); return; } - if self.match_produced(&mut handle_sequence) { + if self.match_produced::(&mut handle_sequence) { return; } } @@ -2350,116 +2491,37 @@ impl Matcher for MatchGeneratorDriver { // below it they are taken outright (upstream // `ZSTD_ldm_blockCompress`, `zstd_ldm.c:685-749`). if matches!(self.search, super::strategy::SearchMethod::BinaryTree) { - self.take_long_distance_block(); + self.take_long_distance_block::(); let MatcherStorage::HashChain(hc) = &mut self.storage else { unreachable!("the optimal parser runs on the hash-chain backend"); }; core::mem::swap(&mut self.ldm_block, &mut hc.backend.bt_mut().ldm_sequences); - } else if self.start_matching_long_distance(&mut handle_sequence, false) { + } else if self.start_matching_long_distance::(&mut handle_sequence, false) { return; } } let mut gap_literals = None; - self.start_matching_owned(&mut owned_sink(&mut handle_sequence, &mut gap_literals)); - } - - fn skip_matching(&mut self) { - self.skip_matching_with_hint(None); + self.start_matching_owned::(&mut owned_sink(&mut handle_sequence, &mut gap_literals)); } - fn skip_matching_with_hint(&mut self, incompressible_hint: Option) { - // Borrowed one-shot Fast path: a staged block range routes to the - // borrowed skip (records the range for `get_last_space`, primes - // hashes on the dict-priming hint) with no owned-history append - // and nothing to recycle. Stage is consumed. - if let Some((block_start, block_end)) = self.borrowed_pending.take() { - match self.active_backend() { - super::strategy::BackendTag::Simple => self.simple_mut().skip_matching_borrowed( - block_start, - block_end, - incompressible_hint, - ), - super::strategy::BackendTag::Dfast => self - .dfast_matcher_mut() - .skip_matching_borrowed(block_start, block_end, incompressible_hint), - super::strategy::BackendTag::Row => self.row_matcher_mut().skip_matching_borrowed( - block_start, - block_end, - incompressible_hint, - ), - super::strategy::BackendTag::HashChain => self - .hc_matcher_mut() - .skip_matching_borrowed(block_start, block_end, incompressible_hint), - } - return; - } - // A block written out unsearched is still searched for long matches, - // as upstream searches every block it compresses, and its matches are - // taken past. + /// [`Matcher::settle_block_cut`] compiled for the kernel tier `K`. + pub(crate) fn settle_block_cut_with( + &mut self, + window: usize, + cut: usize, + ) -> usize { #[cfg(feature = "ldm")] - if self.ldm.is_some() { - self.take_long_distance_block(); - } - self.skip_backend(incompressible_hint); - } -} - -impl MatchGeneratorDriver { - /// Index the last committed block (or piece of one) in the backend - /// without searching it, leaving the long-distance state alone. - fn skip_backend(&mut self, incompressible_hint: Option) { - match self.active_backend() { - super::strategy::BackendTag::Simple => { - self.simple_mut() - .skip_matching_with_hint(incompressible_hint); - } - super::strategy::BackendTag::Dfast => { - self.dfast_matcher_mut().skip_matching(incompressible_hint) - } - super::strategy::BackendTag::Row => self - .row_matcher_mut() - .skip_matching_with_hint(incompressible_hint), - super::strategy::BackendTag::HashChain => { - self.hc_matcher_mut().skip_matching(incompressible_hint) - } + return self.settle_long_distance_cut::(window, cut); + #[cfg(not(feature = "ldm"))] + { + let _ = window; + cut } } -} -/// The sink every owned match runs against: `handle_sequence`, except that a -/// block's trailing literals go to `gap_literals` when it is `Some`, as the -/// gap before a long-distance match needs them. One sink type for the plain -/// block and for those gaps compiles each backend's scan once per caller sink -/// instead of twice. -fn owned_sink<'a, F: FnMut(Sequence)>( - handle_sequence: &'a mut F, - gap_literals: &'a mut Option, -) -> impl FnMut(Sequence) + 'a { - // The guard is read only for the block's last report, never per triple. - // Each arm rebuilds the report from its fields: forwarding the matched - // value whole copies it as one 32-byte block out of the separate stores - // that wrote it, and that copy stalls on every triple. - move |sequence| match sequence { - Sequence::Triple { - literal_len, - offset, - match_len, - } => handle_sequence(Sequence::Triple { - literal_len, - offset, - match_len, - }), - Sequence::Literals { len } => match gap_literals { - Some(gap) => *gap = len, - None => handle_sequence(Sequence::Literals { len }), - }, - } -} - -impl MatchGeneratorDriver { /// Match the last committed block of the owned history with the frame's /// backend. - fn start_matching_owned(&mut self, handle_sequence: &mut impl FnMut(Sequence)) { + fn start_matching_owned(&mut self, handle_sequence: &mut impl FnMut(Sequence)) { use super::strategy::{self, StrategyTag}; // Decoupled parse×search dispatch (fires once per block). The // search axis (`self.search`) picks the candidate-finding backend; @@ -2475,7 +2537,8 @@ impl MatchGeneratorDriver { self.simple_mut().start_matching(handle_sequence); } SearchMethod::DoubleFast => { - self.dfast_matcher_mut().start_matching(handle_sequence); + self.dfast_matcher_mut() + .start_matching::(handle_sequence); } SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => { // One upstream `lazy_generic` body for greedy (depth 0: a @@ -2483,17 +2546,22 @@ impl MatchGeneratorDriver { // / lazy2 (`lazy_depth` 1 / 2) and btlazy2 (depth 2 over the // binary-tree finder); the finder (rows / chain / tree) was // resolved at `configure`. - self.row_matcher_mut().start_matching(handle_sequence); + self.row_matcher_mut().start_matching::(handle_sequence); } SearchMethod::HashChain => { // Greedy/lazy/lazy2 all flow through the lazy parser; it // reads `hc.lazy_depth` (0 = greedy commit). - self.hc_matcher_mut().start_matching_lazy(handle_sequence); + self.hc_matcher_mut() + .start_matching_lazy::(handle_sequence); } SearchMethod::BinaryTree => match self.strategy_tag { - StrategyTag::BtOpt => self.compress_block::(handle_sequence), - StrategyTag::BtUltra => self.compress_block::(handle_sequence), - StrategyTag::BtUltra2 => self.compress_block::(handle_sequence), + StrategyTag::BtOpt => self.compress_block::(handle_sequence), + StrategyTag::BtUltra => { + self.compress_block::(handle_sequence) + } + StrategyTag::BtUltra2 => { + self.compress_block::(handle_sequence) + } _ => unreachable!( "SearchMethod::BinaryTree requires an optimal strategy tag (BtOpt/BtUltra/BtUltra2)" ), @@ -2512,7 +2580,7 @@ impl MatchGeneratorDriver { /// backends (Fast / DoubleFast / RowHash / HashChain) are dispatched /// directly off the search axis and never reach this method, so all /// strategies arriving here are HashChain-backed. - fn compress_block( + fn compress_block( &mut self, handle_sequence: &mut impl FnMut(Sequence), ) { @@ -2522,7 +2590,7 @@ impl MatchGeneratorDriver { "compress_block only handles the optimal (BT) path" ); self.hc_matcher_mut() - .start_matching_strategy::(handle_sequence); + .start_matching_strategy::(handle_sequence); } } diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index 061bdd7d..e9e5779b 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -3,6 +3,7 @@ //! optimal round-trips, and dictionary-priming behaviour. use super::*; +use crate::encoding::fastpath::with_host_tier; use crate::encoding::test_support::BlockReplay; use alloc::vec::Vec; @@ -230,12 +231,12 @@ fn row_mls_knob_gates_matches_and_roundtrips() { let mut out: Vec = Vec::with_capacity(data.len()); let mut shortest_match = usize::MAX; let mut replay = BlockReplay::new(&data); - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(|seq| { if let Sequence::Triple { match_len, .. } = seq { shortest_match = shortest_match.min(match_len); } replay.apply(&mut out, seq); - }); + })); assert_eq!(out, data, "mls={mls} round-trip diverged"); if shortest_match != usize::MAX { @@ -870,22 +871,23 @@ fn dfast_dictionary_all_kernel_tiers_emit_identical_sequences() { driver.set_source_size_hint(block.len() as u64); driver.set_dictionary_size_hint(crate::encoding::DictionarySizes::raw_content(dict.len())); driver.reset(CompressionLevel::Level(3)); - driver.prime_with_dictionary(&dict, [1, 4, 8]); - assert_eq!( - driver.active_backend(), - super::super::strategy::BackendTag::Dfast, - "level 3 with a dictionary must run the dfast backend", - ); - driver.dfast_matcher_mut().kernel = tier; - driver.commit_input(&block); let mut seqs = Vec::new(); - driver.start_matching(|seq| match seq { - Sequence::Triple { - literal_len, - offset, - match_len, - } => seqs.push((literal_len, offset, match_len)), - Sequence::Literals { len } => seqs.push((len, 0, 0)), + crate::encoding::fastpath::with_encode_tier!(tier, K => { + driver.prime_with_dictionary_with::(&dict, [1, 4, 8]); + assert_eq!( + driver.active_backend(), + super::super::strategy::BackendTag::Dfast, + "level 3 with a dictionary must run the dfast backend", + ); + driver.commit_input(&block); + driver.start_matching_with::(|seq| match seq { + Sequence::Triple { + literal_len, + offset, + match_len, + } => seqs.push((literal_len, offset, match_len)), + Sequence::Literals { len } => seqs.push((len, 0, 0)), + }); }); seqs }; @@ -1395,7 +1397,7 @@ fn hc_repcode_candidates_respect_litlen_dependent_rep_order() { let reps = [6u32, 3u32, 9u32]; let mut lit_pos_candidates = Vec::new(); - hc.hc.for_each_repcode_candidate_with_reps( + with_host_tier!(K => hc.hc.for_each_repcode_candidate_with_reps::( &hc.table, abs_pos, 1, @@ -1405,14 +1407,14 @@ fn hc_repcode_candidates_respect_litlen_dependent_rep_order() { |c| { lit_pos_candidates.push(c.offset); }, - ); + )); assert!( lit_pos_candidates.contains(&6), "when lit_len>0, rep0 should be considered and match" ); let mut ll0_candidates = Vec::new(); - hc.hc.for_each_repcode_candidate_with_reps( + with_host_tier!(K => hc.hc.for_each_repcode_candidate_with_reps::( &hc.table, abs_pos, 0, @@ -1422,7 +1424,7 @@ fn hc_repcode_candidates_respect_litlen_dependent_rep_order() { |c| { ll0_candidates.push(c.offset); }, - ); + )); assert!( !ll0_candidates.contains(&6), "when lit_len==0, rep0 is not directly eligible (ll0 semantics)" @@ -1451,7 +1453,7 @@ fn hc_collect_optimal_candidates_keeps_reps_when_chain_depth_zero() { let abs_pos = 6usize; let current_abs_end = hc.table.history.len(); let mut out = Vec::new(); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, current_abs_end, usize::MAX / 2, @@ -1461,7 +1463,7 @@ fn hc_collect_optimal_candidates_keeps_reps_when_chain_depth_zero() { ldm_candidate: None, }, &mut out, - ); + )); assert!( !out.is_empty(), "rep candidates should remain available even when chain depth is zero" @@ -1485,7 +1487,7 @@ fn hc_collect_optimal_candidates_panics_for_non_bt_strategy() { hc.table.history_abs_start = 0; hc.table.ensure_tables(); let mut out = Vec::new(); - hc.collect_optimal_candidates( + hc.collect_optimal_candidates::( 6, hc.table.history.len(), usize::MAX / 2, @@ -1500,7 +1502,7 @@ fn hc_collect_optimal_candidates_panics_for_non_bt_strategy() { #[test] fn hc_collect_optimal_candidates_dispatches_every_bt_strategy() { - use crate::encoding::fastpath::FastpathKernel; + use crate::encoding::fastpath::ScalarTier; use crate::encoding::strategy::StrategyTag; // The public dispatcher must route every BT strategy tag to the collector // AND to the specialization carrying that tag's consts — not just survive. @@ -1518,7 +1520,6 @@ fn hc_collect_optimal_candidates_dispatches_every_bt_strategy() { ] { let mut hc = HcMatchGenerator::new(64); hc.strategy_tag = tag; - hc.table.kernel = FastpathKernel::Scalar; hc.table.history = b"abcQ00000000abcZ00000000".to_vec().into(); hc.table.history_start = 0; hc.table.history_abs_start = 0; @@ -1529,7 +1530,7 @@ fn hc_collect_optimal_candidates_dispatches_every_bt_strategy() { hc.table.search_depth = 8; let abs_pos = 12usize; let mut out = Vec::new(); - hc.collect_optimal_candidates( + hc.collect_optimal_candidates::( abs_pos, hc.table.history.len(), usize::MAX / 2, @@ -1589,20 +1590,22 @@ fn hc_collect_optimal_candidates_agree_across_kernel_tiers() { let mut hc = HcMatchGenerator::new(1 << 20); hc.configure(BTULTRA2_HC_CONFIG, StrategyTag::BtUltra2, 26); hc.table.commit_input(&data); - hc.table.kernel = tier; let mut out = Vec::new(); (0..data.len() - 8) .map(|pos| { - hc.collect_optimal_candidates( - pos, - data.len(), - usize::MAX / 2, - HcCandidateQuery { - reps: [1, 4, 8], - lit_len: 1, - ldm_candidate: None, - }, - &mut out, + crate::encoding::fastpath::with_encode_tier!( + tier, + K => hc.collect_optimal_candidates::( + pos, + data.len(), + usize::MAX / 2, + HcCandidateQuery { + reps: [1, 4, 8], + lit_len: 1, + ldm_candidate: None, + }, + &mut out, + ) ); out.iter().map(|c| (c.offset, c.match_len)).collect() }) @@ -1639,7 +1642,7 @@ fn hc_collect_optimal_candidates_rep_tail_match_skips_chain_probe() { hc.table.search_depth = 32; let mut out = Vec::new(); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, hc.table.history.len(), usize::MAX / 2, @@ -1649,7 +1652,7 @@ fn hc_collect_optimal_candidates_rep_tail_match_skips_chain_probe() { ldm_candidate: None, }, &mut out, - ); + )); assert!( out.iter() @@ -1674,7 +1677,7 @@ fn hc_collect_optimal_candidates_long_chain_match_advances_skip_window() { hc.table.search_depth = 32; let mut out = Vec::new(); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, hc.table.history.len(), usize::MAX / 2, @@ -1684,7 +1687,7 @@ fn hc_collect_optimal_candidates_long_chain_match_advances_skip_window() { ldm_candidate: None, }, &mut out, - ); + )); assert!( hc.table.skip_insert_until_abs > abs_pos, @@ -1707,7 +1710,7 @@ fn hc_collect_optimal_candidates_advances_skip_window_on_plain_bt_path() { hc.table.skip_insert_until_abs = 0; hc.table.search_depth = 0; let mut out = Vec::new(); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, hc.table.history.len(), usize::MAX / 2, @@ -1717,7 +1720,7 @@ fn hc_collect_optimal_candidates_advances_skip_window_on_plain_bt_path() { ldm_candidate: None, }, &mut out, - ); + )); assert_eq!( hc.table.skip_insert_until_abs, @@ -1759,7 +1762,7 @@ fn hc_ldm_candidates_are_merged_into_optimal_candidates() { hc.table.search_depth = 0; let mut out = Vec::new(); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, current_abs_end, usize::MAX / 2, @@ -1769,7 +1772,7 @@ fn hc_ldm_candidates_are_merged_into_optimal_candidates() { ldm_candidate: Some(ldm), }, &mut out, - ); + )); assert!( out.iter().any( |candidate| candidate.offset == ldm.offset && candidate.match_len == ldm.match_len @@ -1800,7 +1803,7 @@ fn hc_ldm_candidate_survives_the_search_early_exit() { match_len: 12, }; let mut out = Vec::new(); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, hc.table.history.len(), // Below the repeat match's length, so the repeat probe ends the search. @@ -1811,7 +1814,7 @@ fn hc_ldm_candidate_survives_the_search_early_exit() { ldm_candidate: Some(ldm), }, &mut out, - ); + )); assert!( out.iter().any(|c| c.offset == 10 && c.match_len == 5), "the repeat match that ends the search is kept" @@ -1869,7 +1872,7 @@ fn btultra_and_btultra2_both_keep_dictionary_candidates() { let mut hc = HcMatchGenerator::new(256); hc.configure(test_config, StrategyTag::BtUltra2, window_log); prepare_history(&mut hc, abs_pos); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, 160, usize::MAX / 2, @@ -1879,7 +1882,7 @@ fn btultra_and_btultra2_both_keep_dictionary_candidates() { ldm_candidate: None, }, &mut out, - ); + )); assert!( out.iter().any(|candidate| candidate.offset >= 32), "btultra2 should retain dictionary candidates on upstream zstd-parity path" @@ -1888,7 +1891,7 @@ fn btultra_and_btultra2_both_keep_dictionary_candidates() { let mut hc = HcMatchGenerator::new(256); hc.configure(test_config, StrategyTag::BtUltra, window_log); prepare_history(&mut hc, abs_pos); - hc.collect_optimal_candidates( + with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, 160, usize::MAX / 2, @@ -1898,7 +1901,7 @@ fn btultra_and_btultra2_both_keep_dictionary_candidates() { ldm_candidate: None, }, &mut out, - ); + )); assert!( out.iter().any(|candidate| candidate.offset >= 32), "btultra should retain dictionary candidates" @@ -2643,13 +2646,13 @@ fn row_matches_roundtrip_multi_block_pattern() { matcher.commit_input(&first_block); let mut history = Vec::new(); let mut replay = BlockReplay::new(&first_block); - matcher.start_matching(|seq| replay.apply(&mut history, seq)); + with_host_tier!(K => matcher.start_matching::(|seq| replay.apply(&mut history, seq))); assert_eq!(history, first_block); matcher.commit_input(&second_block); let prefix_len = history.len(); let mut replay = BlockReplay::new(&second_block); - matcher.start_matching(|seq| replay.apply(&mut history, seq)); + with_host_tier!(K => matcher.start_matching::(|seq| replay.apply(&mut history, seq))); assert_eq!(&history[prefix_len..], second_block.as_slice()); @@ -2658,7 +2661,7 @@ fn row_matches_roundtrip_multi_block_pattern() { matcher.commit_input(&third_block); let third_prefix = history.len(); let mut replay = BlockReplay::new(&third_block); - matcher.start_matching(|seq| replay.apply(&mut history, seq)); + with_host_tier!(K => matcher.start_matching::(|seq| replay.apply(&mut history, seq))); assert_eq!(&history[third_prefix..], third_block.as_slice()); } @@ -2672,10 +2675,10 @@ fn row_short_block_emits_literals_only() { let mut saw_triple = false; let mut reconstructed = Vec::new(); let mut replay = BlockReplay::new(b"abcde"); - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(|seq| { saw_triple |= matches!(seq, Sequence::Triple { .. }); replay.apply(&mut reconstructed, seq); - }); + })); assert!( !saw_triple, @@ -2688,11 +2691,11 @@ fn row_short_block_emits_literals_only() { // searches (upstream zstd `lazy_generic` `ilimit`). saw_triple = false; matcher.commit_input(b"abcdeabcdeabcdeabcde-padding-past-ilimit"); - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(|seq| { if let Sequence::Triple { .. } = seq { saw_triple = true; } - }); + })); assert!( saw_triple, "row backend should emit triples on repeated data" @@ -2739,14 +2742,14 @@ fn row_backfills_previous_block_tail_for_cross_boundary_match() { matcher.commit_input(&first_block); let mut reconstructed = Vec::new(); let mut replay = BlockReplay::new(&first_block); - matcher.start_matching(|seq| replay.apply(&mut reconstructed, seq)); + with_host_tier!(K => matcher.start_matching::(|seq| replay.apply(&mut reconstructed, seq))); assert_eq!(reconstructed, first_block); matcher.commit_input(&second_block); let mut saw_cross_boundary = false; let prefix_len = reconstructed.len(); let mut replay = BlockReplay::new(&second_block); - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(|seq| { if let Sequence::Triple { literal_len: 0, offset: 3, @@ -2757,7 +2760,7 @@ fn row_backfills_previous_block_tail_for_cross_boundary_match() { saw_cross_boundary = true; } replay.apply(&mut reconstructed, seq); - }); + })); assert!( saw_cross_boundary, @@ -3719,7 +3722,7 @@ fn common_prefix_len_matches_scalar_reference_across_offsets() { let a = &base[start..]; let b = a.to_vec(); assert_eq!( - common_prefix_len(a, &b), + with_host_tier!(K => common_prefix_len::(a, &b)), scalar_reference(a, &b), "equal slices total_len={total_len} start={start}" ); @@ -3732,7 +3735,7 @@ fn common_prefix_len_matches_scalar_reference_across_offsets() { let mut altered = b.clone(); altered[mismatch] ^= 0x5A; assert_eq!( - common_prefix_len(a, &altered), + with_host_tier!(K => common_prefix_len::(a, &altered)), scalar_reference(a, &altered), "total_len={total_len} start={start} mismatch={mismatch}" ); @@ -3743,7 +3746,7 @@ fn common_prefix_len_matches_scalar_reference_across_offsets() { let mut altered = b.clone(); altered[mismatch] ^= 0xA5; assert_eq!( - common_prefix_len(a, &altered), + with_host_tier!(K => common_prefix_len::(a, &altered)), scalar_reference(a, &altered), "tail mismatch total_len={total_len} start={start} mismatch={mismatch}" ); @@ -3754,7 +3757,7 @@ fn common_prefix_len_matches_scalar_reference_across_offsets() { let long = alloc::vec![0xAB; 320]; let shorter = alloc::vec![0xAB; 137]; assert_eq!( - common_prefix_len(&long, &shorter), + with_host_tier!(K => common_prefix_len::(&long, &shorter)), scalar_reference(&long, &shorter) ); } @@ -3897,7 +3900,7 @@ fn row_repcode_skips_candidate_before_history_start() { matcher.history_abs_start = 10; matcher.offset_hist = [3, 0, 0]; - assert!(matcher.repcode_candidate(12, 1).is_none()); + assert!(with_host_tier!(K => matcher.repcode_candidate::(12, 1)).is_none()); } #[test] @@ -3909,7 +3912,7 @@ fn row_repcode_returns_none_when_position_too_close_to_history_end() { matcher.history_abs_start = 0; matcher.offset_hist = [1, 0, 0]; - assert!(matcher.repcode_candidate(4, 1).is_none()); + assert!(with_host_tier!(K => matcher.repcode_candidate::(4, 1)).is_none()); } #[test] @@ -4051,7 +4054,7 @@ fn hc_sparse_skip_matching_preserves_tail_cross_block_match() { let tail_start = first.len() - tail.len(); first[tail_start..].copy_from_slice(tail); matcher.table.commit_input(&first); - matcher.skip_matching(Some(true)); + with_host_tier!(K => matcher.skip_matching::(Some(true))); let mut second = tail.to_vec(); second.extend_from_slice(b"after-tail-literals"); @@ -4102,7 +4105,7 @@ fn btultra2_sparse_skip_matching_preserves_tail_cross_block_match() { let tail_start = first.len() - tail.len(); first[tail_start..].copy_from_slice(tail); matcher.table.commit_input(&first); - matcher.skip_matching(Some(true)); + with_host_tier!(K => matcher.skip_matching::(Some(true))); let mut second = tail.to_vec(); second.extend_from_slice(b"after-tail-literals"); @@ -4145,7 +4148,7 @@ fn hc_sparse_skip_matching_does_not_reinsert_sparse_tail_positions() { let mut matcher = HcMatchGenerator::new(1 << 22); let first = deterministic_high_entropy_bytes(0xC2B2_AE3D_27D4_EB4F, 4096); matcher.table.commit_input(&first); - matcher.skip_matching(Some(true)); + with_host_tier!(K => matcher.skip_matching::(Some(true))); let current_len = first.len(); let current_abs_start = @@ -4440,7 +4443,7 @@ fn hc_rebases_positions_after_u32_boundary() { // Simulate a long-running stream where absolute history positions crossed // the u32 range. Before #51 this disabled HC inserts entirely. matcher.table.history_abs_start = history_abs_start; - matcher.skip_matching(None); + with_host_tier!(K => matcher.skip_matching::(None)); assert_eq!( matcher.table.position_base, matcher.table.history_abs_start, "rebase should anchor to the oldest live absolute position" @@ -4567,7 +4570,7 @@ fn dfast_skip_matching_handles_window_eviction() { let mut reconstructed = alloc::vec![7, 8, 9, 10, 11, 12]; let mut replay = BlockReplay::new(&[7, 8, 9, 10, 11, 12]); - matcher.start_matching(|seq| replay.apply(&mut reconstructed, seq)); + with_host_tier!(K => matcher.start_matching::(|seq| replay.apply(&mut reconstructed, seq))); assert_eq!(reconstructed, [7, 8, 9, 10, 11, 12, 7, 8, 9, 10, 11, 12]); } @@ -4687,18 +4690,18 @@ fn dfast_inserts_tail_positions_for_next_block_matching() { matcher.commit_input(b"012345bcdea"); let mut history = Vec::new(); let mut replay = BlockReplay::new(b"012345bcdea"); - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(|seq| { assert!( matches!(seq, Sequence::Literals { .. }), "first block should not match history" ); replay.apply(&mut history, seq); - }); + })); assert_eq!(history, b"012345bcdea"); matcher.commit_input(b"bcdeabcdeab"); let mut saw_first_sequence = false; - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(|seq| { assert!(!saw_first_sequence, "expected a single cross-block match"); saw_first_sequence = true; match seq { @@ -4720,7 +4723,7 @@ fn dfast_inserts_tail_positions_for_next_block_matching() { } } } - }); + })); assert!( saw_first_sequence, @@ -4833,7 +4836,7 @@ fn dfast_dense_skip_matching_backfills_previous_tail_for_next_block() { matcher.commit_input(&second); let mut first_sequence = None; - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(|seq| { if first_sequence.is_some() { return; } @@ -4845,7 +4848,7 @@ fn dfast_dense_skip_matching_backfills_previous_tail_for_next_block() { match_len, } => (literal_len, offset, match_len), }); - }); + })); let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); assert_eq!( @@ -4879,7 +4882,7 @@ fn dfast_sparse_skip_matching_preserves_tail_cross_block_match() { matcher.commit_input(&second); let mut first_sequence = None; - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(|seq| { if first_sequence.is_some() { return; } @@ -4891,7 +4894,7 @@ fn dfast_sparse_skip_matching_preserves_tail_cross_block_match() { match_len, } => (literal_len, offset, match_len), }); - }); + })); let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); assert_eq!( @@ -5104,7 +5107,7 @@ fn dfast_sparse_skip_matching_backfills_previous_tail_for_consecutive_sparse_blo matcher.commit_input(&third); let mut first_sequence = None; - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(|seq| { if first_sequence.is_some() { return; } @@ -5116,7 +5119,7 @@ fn dfast_sparse_skip_matching_backfills_previous_tail_for_consecutive_sparse_blo match_len, } => (literal_len, offset, match_len), }); - }); + })); let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); assert_eq!( diff --git a/zstd/src/encoding/match_table/helpers.rs b/zstd/src/encoding/match_table/helpers.rs index 258686a1..ad36dc66 100644 --- a/zstd/src/encoding/match_table/helpers.rs +++ b/zstd/src/encoding/match_table/helpers.rs @@ -15,7 +15,7 @@ //! change is `pub(crate)` visibility on the moved items. use super::super::opt::types::MatchCandidate; -use crate::encoding::fastpath::FastpathKernel; +use crate::encoding::fastpath::EncodeTier; /// Minimum match length emitted by the Simple / level-1 matcher (and the /// upstream zstd `ZSTD_fast.c` baseline). Lives here so every matcher and the @@ -67,36 +67,15 @@ pub(crate) const INCOMPRESSIBLE_SKIP_STEP: usize = crate::encoding::incompressible::RAW_SKIP_INDEX_STEP; /// Length of the common prefix of two byte slices, capped at -/// `min(a.len(), b.len())`. Hot path on every match finder; dispatches to -/// the per-CPU `common_prefix_len_ptr` kernel (NEON / SSE4.2 / AVX2+BMI2 / -/// scalar) so the SIMD/CRC heavy lifting lives in one place. See -/// [`crate::encoding::fastpath`] for the per-CPU implementations. +/// `min(a.len(), b.len())`, with the compare kernel of the encoder tier `K`. +/// See [`crate::encoding::fastpath`] for the per-tier implementations. #[inline(always)] -pub(crate) fn common_prefix_len(a: &[u8], b: &[u8]) -> usize { +pub(crate) fn common_prefix_len(a: &[u8], b: &[u8]) -> usize { let max = a.len().min(b.len()); // SAFETY: slice `a` / `b` guarantee at least their `len()` initialized // bytes; `max` is the minimum so both pointers are valid for `max` - // bytes. - unsafe { - crate::encoding::fastpath::dispatch_common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) - } -} - -/// As [`common_prefix_len`] but against an already-resolved [`FastpathKernel`]. -/// Hot match-finder loops resolve the kernel once per block and pass it in, -/// so each byte-compare skips the per-call `select_kernel()` `OnceLock` atomic. -#[inline(always)] -pub(crate) fn common_prefix_len_with_kernel(kernel: FastpathKernel, a: &[u8], b: &[u8]) -> usize { - let max = a.len().min(b.len()); - // SAFETY: as `common_prefix_len` — both pointers are valid for `max` bytes. - unsafe { - crate::encoding::fastpath::dispatch_common_prefix_len_ptr_with_kernel( - kernel, - a.as_ptr(), - b.as_ptr(), - max, - ) - } + // bytes. The tier is the one the encoder chose for the running CPU. + unsafe { K::common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) } } /// Pick the better of two candidate matches: longer wins, ties go to @@ -166,8 +145,7 @@ pub(crate) fn extend_backwards_shared( /// invoked once per encoded byte on lazy / Dfast / Row matchers /// (~10% exclusive on the default-level profile). #[inline(always)] -pub(crate) fn repcode_candidate_shared( - kernel: FastpathKernel, +pub(crate) fn repcode_candidate_shared( concat: &[u8], history_abs_start: usize, offset_hist: [u32; 3], @@ -233,8 +211,7 @@ pub(crate) fn repcode_candidate_shared( == core::ptr::read_unaligned(base.add(current_idx).cast::()) }; if gate { - let match_len = common_prefix_len_with_kernel( - kernel, + let match_len = common_prefix_len::( &concat[candidate_idx..], &concat[current_idx..], ); diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index 2753866c..61ed24f1 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -22,6 +22,7 @@ use super::super::Sequence; use super::super::blocks::encode_offset_with_history; use super::super::cost_model::HC_OPT_NUM; use super::super::dict_attach::DictAttach; +use super::super::fastpath::EncodeTier; use super::super::hc::HC_MIN_MATCH_LEN; use super::super::opt::types::{HcOptimalSequence, MatchCandidate}; use super::super::workspace::{HistoryBuf, Table, Workspace, region_bytes}; @@ -323,13 +324,6 @@ pub(crate) struct MatchTable { /// Active borrowed block range `[start, end)` within `borrowed_input`, /// staged before each borrowed scan. pub(crate) borrowed_block: Option<(usize, usize)>, - /// SIMD kernel tier, resolved ONCE at construction (upstream-style - /// once-per-encoder-invoke detection: the CPU does not change mid-run, so - /// `select_kernel()`'s CPUID/`OnceLock` read is paid a single time here). - /// The per-block strategy entries and the BT walker dispatch off this - /// cached value instead of re-detecting in the hot path. Mirrors the - /// `kernel` field the Fast / Row / Dfast matchers already cache. - pub(crate) kernel: crate::encoding::fastpath::FastpathKernel, /// Set by [`Self::retire_history`] ahead of a layout and taken by the next /// [`Self::reset`], which would otherwise read the floor off a history /// already cut down to what it keeps. @@ -379,7 +373,6 @@ impl Clone for MatchTable { dms: self.dms.clone(), borrowed_input: self.borrowed_input, borrowed_block: self.borrowed_block, - kernel: self.kernel, retired: self.retired, } } @@ -425,7 +418,6 @@ impl Clone for MatchTable { self.block_live = (core::ptr::null(), 0); self.borrowed_input = source.borrowed_input; self.borrowed_block = source.borrowed_block; - self.kernel = source.kernel; self.retired = source.retired; } } @@ -560,7 +552,6 @@ impl MatchTable { dms: DictAttach::new(), borrowed_input: None, borrowed_block: None, - kernel: crate::encoding::fastpath::select_kernel(), retired: None, } } @@ -1743,7 +1734,7 @@ impl MatchTable { /// `target_feature` umbrella. Every position in the range must already be /// representable (see [`Self::arm_block_positions`]). #[inline(always)] - pub(crate) fn bt_insert_range( + pub(crate) fn bt_insert_range( &mut self, from: usize, stop: usize, @@ -1761,7 +1752,7 @@ impl MatchTable { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match self.kernel { + match K::TIER { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { self.bt_insert_range_avx2_bmi2(from, stop, current_abs_end, target_abs) @@ -1776,7 +1767,7 @@ impl MatchTable { } } // wasm has no runtime CPU detection: `simd128` is resolved at compile - // time, so the tier is picked by `cfg` rather than by `self.kernel`. + // time, so the tier is picked by `cfg` rather than by `K`. #[cfg(all( target_arch = "wasm32", target_feature = "simd128", @@ -1950,9 +1941,13 @@ impl MatchTable { /// BT-side history replay after [`Self::begin_rebase`]. Re-walks /// `history_start..abs_pos` through the tree insertion so the pointer-pair /// table is consistent with the freshly reset `position_base`. - pub(crate) fn replay_history_for_rebase_bt(&mut self, history_start: usize, abs_pos: usize) { + pub(crate) fn replay_history_for_rebase_bt( + &mut self, + history_start: usize, + abs_pos: usize, + ) { let rebuild_end = self.history_abs_end(); - let _ = self.bt_insert_range(history_start, abs_pos, rebuild_end, abs_pos); + let _ = self.bt_insert_range::(history_start, abs_pos, rebuild_end, abs_pos); } /// Insert every position from the insertion frontier up to `abs_pos` @@ -1964,7 +1959,11 @@ impl MatchTable { /// covered stay out of the tree. Clamping to `abs_pos` inserted them and /// gave the tree candidates upstream never surfaces (a longer but farther /// match the optimal parser then forced over a cheaper closer one). - pub(crate) fn bt_update_tree_until(&mut self, abs_pos: usize, current_abs_end: usize) { + pub(crate) fn bt_update_tree_until( + &mut self, + abs_pos: usize, + current_abs_end: usize, + ) { if self.skip_insert_until_abs < self.history_abs_start { self.skip_insert_until_abs = self.history_abs_start; } @@ -1972,7 +1971,7 @@ impl MatchTable { if update_abs < abs_pos { if self.can_skip_rebase_check(abs_pos) { // Every position up to the target is representable: one run. - let _ = self.bt_insert_range(update_abs, abs_pos, current_abs_end, abs_pos); + let _ = self.bt_insert_range::(update_abs, abs_pos, current_abs_end, abs_pos); } else { // A rebase may be due inside the run, and it moves the // coordinates the range hoists, so go one position at a time. @@ -1980,8 +1979,12 @@ impl MatchTable { if !self.can_skip_rebase_check(abs_pos) { self.maybe_rebase_positions(update_abs); } - update_abs = - self.bt_insert_range(update_abs, update_abs + 1, current_abs_end, abs_pos); + update_abs = self.bt_insert_range::( + update_abs, + update_abs + 1, + current_abs_end, + abs_pos, + ); } } } @@ -2147,8 +2150,16 @@ impl MatchTable { let history_start = self.history_abs_start; // Rebuild only the already-inserted prefix. The caller inserts abs_pos // immediately after this, and later positions are added in-order. + // + // With the scalar compare, fixed: this runs once per ~4 GiB of stream, + // below every position insert, and the tiers' compares return the same + // lengths, so the tree is the same and only this rare pass's speed + // depends on it. if self.uses_bt { - self.replay_history_for_rebase_bt(history_start, abs_pos); + self.replay_history_for_rebase_bt::( + history_start, + abs_pos, + ); } else { self.replay_history_for_rebase_hc(history_start, abs_pos); } @@ -2321,7 +2332,7 @@ impl MatchTable { /// Backfill the last `< 4` bytes of the previous slice (which couldn't /// be hashed at the time because `insert_position` needs 4 bytes of /// lookahead) before the next slice's matching pass. - pub(crate) fn backfill_boundary_positions( + pub(crate) fn backfill_boundary_positions( &mut self, current_abs_start: usize, current_abs_end: usize, @@ -2331,7 +2342,7 @@ impl MatchTable { .max(self.history_abs_start); if backfill_start < current_abs_start { if self.uses_bt { - self.bt_update_tree_until(current_abs_start, current_abs_end); + self.bt_update_tree_until::(current_abs_start, current_abs_end); } else { self.insert_positions(backfill_start, current_abs_start); } @@ -2356,7 +2367,7 @@ impl MatchTable { /// (which threads the binary tree) and then densely inserts the /// final `HC_MIN_MATCH_LEN + INCOMPRESSIBLE_SKIP_STEP` positions so /// the very last hashable starts are present for the next slice. - pub(crate) fn bt_insert_sparse_incompressible_block( + pub(crate) fn bt_insert_sparse_incompressible_block( &mut self, current_abs_start: usize, current_abs_end: usize, @@ -2364,7 +2375,7 @@ impl MatchTable { let mut pos = current_abs_start; while pos < current_abs_end { self.maybe_rebase_positions(pos); - let _ = self.bt_insert_range(pos, pos + 1, current_abs_end, current_abs_end); + let _ = self.bt_insert_range::(pos, pos + 1, current_abs_end, current_abs_end); self.insert_hash3_only_no_rebase(pos); let next = pos.saturating_add(INCOMPRESSIBLE_SKIP_STEP); if next <= pos { @@ -2383,7 +2394,7 @@ impl MatchTable { continue; } self.maybe_rebase_positions(pos); - let _ = self.bt_insert_range(pos, pos + 1, current_abs_end, current_abs_end); + let _ = self.bt_insert_range::(pos, pos + 1, current_abs_end, current_abs_end); self.insert_hash3_only_no_rebase(pos); } @@ -2444,7 +2455,7 @@ impl MatchTable { /// catches up on after one (`ZSTD_ldm_limitTableUpdate` moves /// `nextToUpdate` to it), since the match itself is never searched. #[cfg(feature = "ldm")] - pub(crate) fn skip_long_match(&mut self) { + pub(crate) fn skip_long_match(&mut self) { self.ensure_tables(); let (start, len) = self.current_block_range(); let end = start + len; @@ -2456,7 +2467,7 @@ impl MatchTable { self.next_to_update3 = self.next_to_update3.max(self.skip_insert_until_abs); return; } - self.backfill_boundary_positions(start, end); + self.backfill_boundary_positions::(start, end); self.insert_positions( end - len.min(crate::encoding::ldm::LONG_MATCH_INDEXED_TAIL), end, @@ -2474,7 +2485,7 @@ impl MatchTable { ); } - pub(crate) fn skip_matching(&mut self, incompressible_hint: Option) { + pub(crate) fn skip_matching(&mut self, incompressible_hint: Option) { self.ensure_tables(); let (current_abs_start, current_len) = self.current_block_range(); let current_abs_end = current_abs_start + current_len; @@ -2482,13 +2493,13 @@ impl MatchTable { // indexed only from its floor on. let floor = (self.history_abs_start + self.dict_index_floor).min(current_abs_end); let current_abs_start = current_abs_start.max(floor); - self.backfill_boundary_positions(current_abs_start, current_abs_end); + self.backfill_boundary_positions::(current_abs_start, current_abs_end); if self.uses_bt { if incompressible_hint == Some(true) { - self.bt_insert_sparse_incompressible_block(current_abs_start, current_abs_end); + self.bt_insert_sparse_incompressible_block::(current_abs_start, current_abs_end); return; } - self.bt_update_tree_until(current_abs_end, current_abs_end); + self.bt_update_tree_until::(current_abs_end, current_abs_end); return; } if incompressible_hint == Some(true) { @@ -2518,7 +2529,7 @@ impl MatchTable { /// `collect_optimal_candidates_initialized_` umbrella. #[allow(dead_code)] #[inline(always)] - pub(crate) fn hash3_candidate( + pub(crate) fn hash3_candidate( &self, abs_pos: usize, current_abs_end: usize, @@ -2535,7 +2546,7 @@ impl MatchTable { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match self.kernel { + match K::TIER { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { self.hash3_candidate_avx2_bmi2(abs_pos, current_abs_end, min_match_len) diff --git a/zstd/src/encoding/match_table/storage/storage_tests.rs b/zstd/src/encoding/match_table/storage/storage_tests.rs index 30ade717..8324e97c 100644 --- a/zstd/src/encoding/match_table/storage/storage_tests.rs +++ b/zstd/src/encoding/match_table/storage/storage_tests.rs @@ -84,7 +84,7 @@ fn skip_matching_bt_incompressible_routes_through_sparse_block() { t.is_btultra2 = false; t.search_depth = 4; let before_skip_until = t.skip_insert_until_abs; - t.skip_matching(Some(true)); + crate::encoding::fastpath::with_host_tier!(K => t.skip_matching::(Some(true))); // BT + incompressible path must take the // `bt_insert_sparse_incompressible_block` branch and advance // `skip_insert_until_abs` to current_abs_end. @@ -101,7 +101,7 @@ fn skip_matching_bt_dense_routes_through_bt_update_tree() { t.is_btultra2 = false; t.search_depth = 4; // `incompressible_hint = None` → dense bt_update_tree_until path - t.skip_matching(None); + crate::encoding::fastpath::with_host_tier!(K => t.skip_matching::(None)); assert_eq!(t.skip_insert_until_abs, t.history_abs_start + t.window_size); } @@ -126,7 +126,7 @@ fn replay_history_for_rebase_bt_walks_inserted_prefix() { // ground-truth observation is "some hash slots are no longer // HC_EMPTY". assert!(t.hash_table().iter().all(|&v| v == HC_EMPTY)); - t.replay_history_for_rebase_bt(0, 32); + crate::encoding::fastpath::with_host_tier!(K => t.replay_history_for_rebase_bt::(0, 32)); assert!( t.hash_table().iter().any(|&v| v != HC_EMPTY), "BT replay must populate hash table" diff --git a/zstd/src/encoding/mod.rs b/zstd/src/encoding/mod.rs index e968e022..7c8d45f1 100644 --- a/zstd/src/encoding/mod.rs +++ b/zstd/src/encoding/mod.rs @@ -537,6 +537,15 @@ pub trait Matcher { /// holds anything there. #[doc(hidden)] fn leave_workspace(&mut self) {} + /// The built-in matcher, when this is it. The encoder then runs its + /// backends compiled for the kernel tier it chose for the block, rather + /// than through [`start_matching`](Self::start_matching), which leaves the + /// tier to the matcher. Every other matcher keeps the default. + #[doc(hidden)] + #[inline] + fn builtin_driver(&mut self) -> Option<&mut match_generator::MatchGeneratorDriver> { + None + } /// Provide a hint about the total uncompressed size for the next frame. /// /// Implementations may use this to select smaller hash tables and windows diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index e977388b..87acbbb5 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -287,20 +287,21 @@ impl RowDictShape { && self.use_bt == other.use_bt } } +use super::match_table::helpers::INCOMPRESSIBLE_SKIP_STEP; +#[cfg(test)] use super::match_table::helpers::{ - INCOMPRESSIBLE_SKIP_STEP, best_len_offset_candidate, extend_backwards_shared, - repcode_candidate_shared, + best_len_offset_candidate, extend_backwards_shared, repcode_candidate_shared, }; use super::match_table::storage::REBASE_RESET_FLOOR_CEILING; +#[cfg(test)] use super::opt::types::MatchCandidate; -// The row probe reuses the shared `fastpath::FastpathKernel` selection so each -// per-tier `#[target_feature]` probe can inline BOTH the tag-match mask AND the -// matching `fastpath::::common_prefix_len_ptr` (the tiers must share a -// feature set for the cpl to inline — `Sse42` is a superset of the SSE2 mask -// intrinsics, `Avx2Bmi2` of the AVX2 mask). `select_kernel()` does the runtime -// detect once per process via a `OnceLock`. -use super::fastpath::FastpathKernel; +// The row probe runs under the encoder's tier so each per-tier +// `#[target_feature]` probe can inline BOTH the tag-match mask AND the matching +// `fastpath::::common_prefix_len_ptr` (the tiers must share a feature set +// for the cpl to inline — `Sse42` is a superset of the SSE2 mask intrinsics, +// `Avx2Bmi2` of the AVX2 mask). +use super::fastpath::{EncodeTier, FastpathKernel}; /// Compile-time row tag-match kernel. Each ZST monomorphises the per-row /// tag compare so the search hot loop drops the runtime `RowTagKernel` @@ -311,6 +312,10 @@ use super::fastpath::FastpathKernel; /// supports its ISA — the same contract `RowTagKernel::detect` upholds for /// the enum's `unsafe` SIMD calls. pub(crate) trait RowTags: Copy { + /// The encoder tier this tag kernel runs under, whose byte compare the + /// probe's callers use. + type Tier: EncodeTier; + /// Run the row match probe (live row + dict dual-probe) for this kernel. /// Forwards to the matcher's per-tier `#[target_feature]` probe method whose /// body expands the tier's `row_tag_mask_*!` SIMD inline (no function call), @@ -324,6 +329,7 @@ pub(crate) trait RowTags: Copy { /// The caller (via `dispatch_tag_kernel!`) only selects a kernel whose ISA /// `RowTagKernel::detect` confirmed present, upholding the per-tier /// `#[target_feature]` contract. + #[cfg(test)] unsafe fn probe( matcher: &RowMatchGenerator, abs_pos: usize, @@ -346,6 +352,9 @@ pub(crate) trait RowTags: Copy { #[derive(Copy, Clone)] struct ScalarTags; impl RowTags for ScalarTags { + type Tier = crate::encoding::fastpath::ScalarTier; + + #[cfg(test)] #[inline] unsafe fn probe( matcher: &RowMatchGenerator, @@ -369,6 +378,10 @@ struct Sse2Tags; feature = "kernel-sse" ))] impl RowTags for Sse2Tags { + // `Sse42` shares this tag kernel and the SSE2 compare. + type Tier = crate::encoding::fastpath::Sse2Tier; + + #[cfg(test)] #[inline] unsafe fn probe( matcher: &RowMatchGenerator, @@ -394,6 +407,9 @@ struct Avx2Bmi2Tags; feature = "kernel-avx2" ))] impl RowTags for Avx2Bmi2Tags { + type Tier = crate::encoding::fastpath::Avx2Bmi2Tier; + + #[cfg(test)] #[inline] unsafe fn probe( matcher: &RowMatchGenerator, @@ -419,6 +435,9 @@ struct NeonTags; feature = "kernel-neon" ))] impl RowTags for NeonTags { + type Tier = crate::encoding::fastpath::NeonTier; + + #[cfg(test)] #[inline] unsafe fn probe( matcher: &RowMatchGenerator, @@ -448,6 +467,9 @@ struct Simd128Tags; feature = "kernel-simd128" ))] impl RowTags for Simd128Tags { + type Tier = crate::encoding::fastpath::Simd128Tier; + + #[cfg(test)] #[inline] unsafe fn probe( matcher: &RowMatchGenerator, @@ -461,12 +483,13 @@ impl RowTags for Simd128Tags { } } -/// Resolve the runtime `tag_kernel` (`FastpathKernel`) to a `RowTags` ZST once, -/// then call a `*_k::` method that binds the `row_log` const. The kernel -/// branch runs once per block (cold), so the per-position hot loop is fully -/// monomorphised over the selected tier — no runtime kernel enum inside. +/// Map the encoder tier `$tier` to its `RowTags` ZST, then call a `*_k::` +/// method that binds the `row_log` const. The match on the tier's constant +/// folds at monomorphisation, so the per-position hot loop is fully +/// monomorphised over the tier with no kernel choice anywhere. +#[cfg(test)] macro_rules! dispatch_tag_kernel { - ($self:ident . $k_method:ident ( $($arg:expr),* )) => {{ + ($tier:ty, $self:ident . $k_method:ident ( $($arg:expr),* )) => {{ // wasm32 has no runtime CPUID: when the build enables `simd128`, the // tier is resolved at compile time straight to `Simd128Tags`, so the // runtime `FastpathKernel` match (native-only) is cfg'd out entirely. @@ -484,7 +507,7 @@ macro_rules! dispatch_tag_kernel { feature = "kernel-simd128" )))] { - match $self.tag_kernel { + match <$tier as EncodeTier>::TIER { #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-avx2" @@ -1802,6 +1825,7 @@ macro_rules! dispatch_lazy { /// Bind the runtime `row_log` (clamped 4..=6) to the const `ROW_LOG` of a /// `*_rl::` hot loop. Mirrors the upstream zstd's per-rowLog variant /// table; the branch is cold (once per block / call). +#[cfg(test)] macro_rules! dispatch_row_log { ($self:ident . $rl_method:ident :: <$k:ty> ( $($arg:expr),* )) => { match $self.row_log { @@ -1990,6 +2014,7 @@ macro_rules! row_tag_mask_simd128 { /// spills per position. Early exits use the labeled block (`break 'probe`) /// because a `return` inside a macro body would return from the EXPANSION /// SITE's function. +#[cfg(test)] macro_rules! row_probe_body { ($m:expr, $abs_pos:expr, $lit_len:expr, $hash:expr, $seed:expr, $rl:expr, $use_mask:literal, $maskmac:ident, $cpl:path) => {{ #[allow(unused_labels)] @@ -2289,6 +2314,7 @@ macro_rules! row_probe_body { }}; } +#[cfg(test)] macro_rules! gen_row_probe { ($name:ident, $use_mask:literal, $maskmac:ident, $cpl:path $(, $tf:literal)?) => { $(#[target_feature(enable = $tf)])? @@ -2466,10 +2492,6 @@ pub(crate) struct RowMatchGenerator { /// indexing, `0` for all of it (see [`Self::set_dictionary_index_floor`]). dict_index_floor: usize, pub(crate) lazy_depth: u8, - /// Cached fastpath kernel for the per-candidate `common_prefix_len` - /// compare, resolved once per matcher so the rep probe skips the - /// `select_kernel()` atomic on every input byte. - pub(crate) cpl_kernel: crate::encoding::fastpath::FastpathKernel, /// Single backing allocation for every `u32` index table this matcher /// owns (the cwksp analog the dfast backend already uses for its long / /// short pair). The row finder and the chain / tree finder are mutually @@ -2516,19 +2538,6 @@ pub(crate) struct RowMatchGenerator { // [`Self::rebase_positions`]), keeping positions representable without // capping frame length. The tags and the slot cursors live in the byte // tail of that same buffer; see [`Self::rows_len`]. - /// Cached tag-match SIMD kernel; CPU features are fixed per process, so - /// resolve once instead of querying per `row_candidate` call. On - /// wasm32+simd128 the tier is compile-time (`dispatch_tag_kernel!` selects - /// `Simd128Tags` directly), so the field is unread there. - #[cfg_attr( - all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ), - allow(dead_code) - )] - tag_kernel: FastpathKernel, /// Attached immutable dictionary row index (upstream zstd `dictMatchState`). `Some` /// activates the bounded dict probe in `row_candidate_rl`; built once and /// cached across frames via `DictAttach`, invalidated on eviction / resize. @@ -2603,11 +2612,9 @@ impl RowMatchGenerator { lazy_reps: None, dict_index_floor: 0, lazy_depth: 1, - cpl_kernel: crate::encoding::fastpath::select_kernel(), rows_len: 0, heads_len: 0, tags_len: 0, - tag_kernel: crate::encoding::fastpath::select_kernel(), dict: DictAttach::new(), dict_keep: DictKeep::new(), #[cfg(test)] @@ -3374,7 +3381,9 @@ impl RowMatchGenerator { /// deviate here — both the rep probe at `abs_pos + 1` *and* the /// regular `row_candidate(abs_pos, ..)` are evaluated each /// iteration, and the longer match wins (ties go to rep for - /// cheaper encoding via [`best_len_offset_candidate`]). Upstream zstd + /// cheaper encoding via + /// [`best_len_offset_candidate`](super::match_table::helpers::best_len_offset_candidate)). + /// Upstream zstd /// can afford pure commit-on-first-rep because it recovers any /// ratio loss via superblock-level entropy sharing, which we /// don't replicate yet, so the hybrid form avoids a measured @@ -3397,7 +3406,8 @@ impl RowMatchGenerator { /// `offset_2` for back-to-back hits. It is omitted here: the /// main-loop rep probe at `abs_pos + 1` already evaluates all /// three rep slots (rep1, rep2, rep3 + the upstream zstd `ll0` fallback) - /// via [`repcode_candidate_shared`], so the inner-loop slot + /// via [`repcode_candidate_shared`](super::match_table::helpers::repcode_candidate_shared), + /// so the inner-loop slot /// upstream zstd's single-rep design would catch is already covered by /// the next main-loop iteration. Confirmed dead-on-arrival via a /// `panic!` probe across the full 528-test suite + benchmark @@ -3726,6 +3736,7 @@ impl RowMatchGenerator { &self.history[self.history_start..self.history.len() - self.uncommitted_len] } + #[cfg(test)] fn history_abs_end(&self) -> usize { self.history_abs_start + self.live_history().len() } @@ -3925,22 +3936,23 @@ impl RowMatchGenerator { .max(self.history_abs_start) } - /// Used only by the dead-code [`Self::start_matching`] (lazy-style - /// row parse). Kept paired with that method so reviving the lazy - /// path doesn't have to re-derive the rep+row best-of-two pick. + /// The rep+row best-of-two pick at one position, for unit tests that probe + /// the row search outside the production monolith. + #[cfg(test)] #[inline(always)] pub(crate) fn best_match_rl( &self, abs_pos: usize, lit_len: usize, ) -> Option { - let rep = self.repcode_candidate(abs_pos, lit_len); + let rep = self.repcode_candidate::(abs_pos, lit_len); // SAFETY: `K` selected by `dispatch_tag_kernel!` after `detect` confirmed // its ISA; `K::probe` upholds the per-tier feature contract. let row = unsafe { K::probe::(self, abs_pos, lit_len, None) }; best_len_offset_candidate(rep, row) } + #[cfg(test)] #[inline(always)] pub(crate) fn pick_lazy_match_rl( &self, @@ -3970,15 +3982,14 @@ impl RowMatchGenerator { } } - #[allow(dead_code)] + #[cfg(test)] #[inline(always)] - pub(crate) fn repcode_candidate( + pub(crate) fn repcode_candidate( &self, abs_pos: usize, lit_len: usize, ) -> Option { - repcode_candidate_shared( - self.cpl_kernel, + repcode_candidate_shared::( self.live_history(), self.history_abs_start, self.offset_hist, @@ -3995,7 +4006,7 @@ impl RowMatchGenerator { // (driver + tests) use these bare names; the hot loops call the `_rl` // siblings directly with the type and const already bound. `skip` does // no tag compare, so it dispatches on `row_log` only. - pub(crate) fn start_matching(&mut self, handle_sequence: impl FnMut(Sequence)) { + pub(crate) fn start_matching(&mut self, handle_sequence: impl FnMut(Sequence)) { // SAFETY: same per-tier umbrella contract as `start_matching_greedy`. #[cfg(all( target_arch = "wasm32", @@ -4012,7 +4023,7 @@ impl RowMatchGenerator { feature = "kernel-simd128" )))] { - match self.tag_kernel { + match K::TIER { #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-avx2" @@ -4161,7 +4172,7 @@ impl RowMatchGenerator { /// into the owned mirror). `history_abs_start` was forced /// to 0 in `set_borrowed_window`, so positions stay absolute input /// offsets and the window-low candidate cap bounds offsets to the window. - pub(crate) fn start_matching_borrowed( + pub(crate) fn start_matching_borrowed( &mut self, block_start: usize, block_end: usize, @@ -4171,7 +4182,7 @@ impl RowMatchGenerator { self.stage_borrowed_block(block_start, block_end); // Greedy runs the same lazy monolith at depth 0. let _ = greedy; - self.start_matching(handle_sequence); + self.start_matching::(handle_sequence); } /// Borrowed equivalent of [`Self::skip_matching_with_hint`]: stage the @@ -4187,23 +4198,31 @@ impl RowMatchGenerator { self.skip_matching_with_hint(incompressible_hint); } - #[allow(dead_code)] + #[cfg(test)] pub(crate) fn best_match(&self, abs_pos: usize, lit_len: usize) -> Option { - dispatch_tag_kernel!(self.best_match_k(abs_pos, lit_len)) + crate::encoding::fastpath::with_encode_tier!( + crate::encoding::fastpath::select_kernel(), + T => dispatch_tag_kernel!(T, self.best_match_k(abs_pos, lit_len)) + ) } + #[cfg(test)] fn best_match_k(&self, abs_pos: usize, lit_len: usize) -> Option { dispatch_row_log!(self.best_match_rl::(abs_pos, lit_len)) } - #[allow(dead_code)] + #[cfg(test)] pub(crate) fn pick_lazy_match( &self, abs_pos: usize, lit_len: usize, best: Option, ) -> Option { - dispatch_tag_kernel!(self.pick_lazy_match_k(abs_pos, lit_len, best)) + crate::encoding::fastpath::with_encode_tier!( + crate::encoding::fastpath::select_kernel(), + T => dispatch_tag_kernel!(T, self.pick_lazy_match_k(abs_pos, lit_len, best)) + ) } + #[cfg(test)] fn pick_lazy_match_k( &self, abs_pos: usize, @@ -4217,10 +4236,14 @@ impl RowMatchGenerator { // `dispatch_tag_kernel!`; the selected tier's `row_probe_*` method is the // monomorphised per-position hot loop with the SIMD tag-match inlined under // its `#[target_feature]` umbrella (no dispatcher branch inside the loop). - #[allow(dead_code)] + #[cfg(test)] pub(crate) fn row_candidate(&self, abs_pos: usize, lit_len: usize) -> Option { - dispatch_tag_kernel!(self.row_candidate_k(abs_pos, lit_len)) + crate::encoding::fastpath::with_encode_tier!( + crate::encoding::fastpath::select_kernel(), + T => dispatch_tag_kernel!(T, self.row_candidate_k(abs_pos, lit_len)) + ) } + #[cfg(test)] fn row_candidate_k( &self, abs_pos: usize, @@ -4242,6 +4265,8 @@ impl RowMatchGenerator { // probe's: the SSE2 umbrella covers the 128-bit mask intrinsics, // AVX2+BMI2 ⊇ the AVX2 mask). // Scalar uses the on-the-fly per-slot byte compare (`use_mask = false`). + // The probes serve the unit tests' row search; production runs the monolith. + #[cfg(test)] gen_row_probe!( row_probe_scalar, false, @@ -4249,6 +4274,7 @@ impl RowMatchGenerator { crate::encoding::fastpath::scalar::common_prefix_len_ptr ); #[cfg(all( + test, any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-sse" ))] @@ -4260,6 +4286,7 @@ impl RowMatchGenerator { "sse2" ); #[cfg(all( + test, any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-avx2" ))] @@ -4271,6 +4298,7 @@ impl RowMatchGenerator { "avx2,bmi2" ); #[cfg(all( + test, target_arch = "aarch64", target_endian = "little", feature = "kernel-neon" @@ -4287,6 +4315,7 @@ impl RowMatchGenerator { // wasm tier's behavior of vectorising only the tag scan, with the portable // prefix-length kernel. #[cfg(all( + test, target_arch = "wasm32", target_feature = "simd128", feature = "kernel-simd128" @@ -4298,6 +4327,7 @@ impl RowMatchGenerator { crate::encoding::fastpath::scalar::common_prefix_len_ptr ); + #[cfg(test)] fn extend_backwards( &self, candidate_pos: usize, diff --git a/zstd/src/encoding/simple/fast_kernel/kernel.rs b/zstd/src/encoding/simple/fast_kernel/kernel.rs index fcf1e72b..fdab6d97 100644 --- a/zstd/src/encoding/simple/fast_kernel/kernel.rs +++ b/zstd/src/encoding/simple/fast_kernel/kernel.rs @@ -2050,12 +2050,15 @@ fast_dict_borrowed_wrapper!( crate::encoding::fastpath::simd128::common_prefix_len_ptr ); -/// Dispatch the borrowed dual-base dict kernel to the resolved per-tier SIMD -/// wrapper. `kernel` is the matcher's once-resolved -/// [`FastpathKernel`](crate::encoding::fastpath::FastpathKernel); the -/// scalar arm carries no `#[target_feature]` (generic word-at-a-time count). +/// The borrowed dual-base dict kernel compiled for the encoder tier `K`: its +/// per-tier SIMD wrapper, or for the scalar tier the plain body (generic +/// word-at-a-time count, no `#[target_feature]`). #[allow(clippy::too_many_arguments)] -pub(crate) fn compress_block_fast_dict_borrowed( +pub(crate) fn compress_block_fast_dict_borrowed< + K: crate::encoding::fastpath::EncodeTier, + const MLS: u32, + const USE_CMOV: bool, +>( inp: &[u8], dict: &[u8], block_start: usize, @@ -2066,7 +2069,6 @@ pub(crate) fn compress_block_fast_dict_borrowed FastBlockResult { // Used by the per-tier match arms below; on a target with no SIMD tier // (e.g. wasm32 without simd128) only the scalar fallback compiles and the @@ -2099,7 +2101,7 @@ pub(crate) fn compress_block_fast_dict_borrowed unsafe { cbfd_borrowed_avx2_bmi2::( @@ -2142,7 +2144,7 @@ pub(crate) fn compress_block_fast_dict_borrowed unsafe { cbfd_borrowed_neon::( inp, @@ -2169,7 +2171,7 @@ pub(crate) fn compress_block_fast_dict_borrowed unsafe { cbfd_borrowed_simd128::( inp, @@ -2205,7 +2207,6 @@ pub(crate) fn compress_block_fast_dict_borrowed( + let result = compress_block_fast_dict_borrowed::( &inp, &dict, 0, @@ -656,7 +656,6 @@ fn borrowed_dict_kernel_reconstructs_via_dual_base() { [0, 0], 2, &mut handle, - FastpathKernel::Scalar, ); // Reconstruct against the logical [dict][input] window: start from the @@ -706,7 +705,7 @@ fn borrowed_dict_kernel_reconstructs_via_dual_base() { /// four-byte gate has to hold there too: the match is real and must be found. #[test] fn borrowed_dict_kernel_takes_a_repcode_whose_candidate_is_in_the_dictionary() { - use crate::encoding::fastpath::FastpathKernel; + use crate::encoding::fastpath::ScalarTier; let hash_log = 12u32; const MLS: u32 = 4; @@ -731,7 +730,7 @@ fn borrowed_dict_kernel_takes_a_repcode_whose_candidate_is_in_the_dictionary() { let mut replay = BlockReplay::new(&inp); let mut handle = |seq: Sequence| tuples.push(capture(&mut replay, seq)); - let result = compress_block_fast_dict_borrowed::( + let result = compress_block_fast_dict_borrowed::( &inp, &dict, 0, @@ -745,7 +744,6 @@ fn borrowed_dict_kernel_takes_a_repcode_whose_candidate_is_in_the_dictionary() { [offset_1, 0], 2, &mut handle, - FastpathKernel::Scalar, ); let mut window = dict.clone(); @@ -783,7 +781,7 @@ fn borrowed_dict_kernel_takes_a_repcode_whose_candidate_is_in_the_dictionary() { /// failed four-byte gate would have rejected it. #[test] fn borrowed_dict_kernel_rejects_a_short_repcode_in_the_dictionary_tail() { - use crate::encoding::fastpath::FastpathKernel; + use crate::encoding::fastpath::ScalarTier; let hash_log = 12u32; const MLS: u32 = 4; @@ -801,7 +799,7 @@ fn borrowed_dict_kernel_rejects_a_short_repcode_in_the_dictionary_tail() { let mut replay = BlockReplay::new(&inp); let mut handle = |seq: Sequence| tuples.push(capture(&mut replay, seq)); - let result = compress_block_fast_dict_borrowed::( + let result = compress_block_fast_dict_borrowed::( &inp, &dict, 0, @@ -815,7 +813,6 @@ fn borrowed_dict_kernel_rejects_a_short_repcode_in_the_dictionary_tail() { [3, 0], 2, &mut handle, - FastpathKernel::Scalar, ); assert!( @@ -836,7 +833,7 @@ fn borrowed_dict_kernel_rejects_a_short_repcode_in_the_dictionary_tail() { /// must still be found, which is what the gate must not cost. #[test] fn borrowed_dict_kernel_finds_a_match_starting_in_the_dictionary_tail() { - use crate::encoding::fastpath::FastpathKernel; + use crate::encoding::fastpath::ScalarTier; let hash_log = 12u32; const MLS: u32 = 4; @@ -868,7 +865,7 @@ fn borrowed_dict_kernel_finds_a_match_starting_in_the_dictionary_tail() { let mut replay = BlockReplay::new(&inp); let mut handle = |seq: Sequence| tuples.push(capture(&mut replay, seq)); - let result = compress_block_fast_dict_borrowed::( + let result = compress_block_fast_dict_borrowed::( &inp, &dict, 0, @@ -882,7 +879,6 @@ fn borrowed_dict_kernel_finds_a_match_starting_in_the_dictionary_tail() { [0, 0], 2, &mut handle, - FastpathKernel::Scalar, ); let mut window = dict.clone(); diff --git a/zstd/src/encoding/simple/fast_matcher.rs b/zstd/src/encoding/simple/fast_matcher.rs index 84b194a3..612f8f13 100644 --- a/zstd/src/encoding/simple/fast_matcher.rs +++ b/zstd/src/encoding/simple/fast_matcher.rs @@ -248,14 +248,6 @@ pub(crate) struct FastKernelMatcher { /// `adjust_params_for_source_size` clamps `window_log` below /// the upstream zstd default of 19, flipping `use_cmov` on). use_cmov: bool, - /// Cached per-tier SIMD kernel selection (resolved once via - /// [`crate::encoding::fastpath::select_kernel`] at construction / reset), - /// mirroring the Dfast/Row backends. Drives the `#[target_feature]` - /// umbrella dispatch in the borrowed dual-base dict scan so the - /// match-length `common_prefix_len_ptr` is the tier's 32-byte AVX2 / - /// 16-byte SSE4.2 / NEON / wasm-simd128 compare instead of the generic - /// word-at-a-time `count_forward`. - kernel: crate::encoding::fastpath::FastpathKernel, /// Initial step the kernel uses for the 4-cursor body's skip /// schedule. Upstream zstd `stepSize = targetLength + !(targetLength) + /// 1` (min 2). Negative-level frames set this to 2..8 to @@ -358,7 +350,6 @@ impl Clone for FastKernelMatcher { max_window_size: self.max_window_size, window_log: self.window_log, use_cmov: self.use_cmov, - kernel: self.kernel, step_size: self.step_size, staged_block: self.staged_block, last_block_start: self.last_block_start, @@ -388,7 +379,6 @@ impl Clone for FastKernelMatcher { self.max_window_size = source.max_window_size; self.window_log = source.window_log; self.use_cmov = source.use_cmov; - self.kernel = source.kernel; self.step_size = source.step_size; self.staged_block = source.staged_block; self.last_block_start = source.last_block_start; @@ -626,7 +616,6 @@ impl FastKernelMatcher { max_window_size: 1usize << window_log, window_log, use_cmov: window_log < 19, - kernel: crate::encoding::fastpath::select_kernel(), step_size, staged_block: None, borrowed: None, @@ -1450,7 +1439,7 @@ impl FastKernelMatcher { /// lookahead, dense fills, backward extension, immediate repcode-2 loop) — /// the prior scalar greedy scan had none of these and was +68% slower. /// Validated by roundtrip + cross-validation + the FFI ratio gate. - pub(crate) fn start_matching_borrowed_dict( + pub(crate) fn start_matching_borrowed_dict( &mut self, block_start: usize, block_end: usize, @@ -1484,7 +1473,6 @@ impl FastKernelMatcher { let use_cmov = self.use_cmov; let step_size = self.step_size; let rep_in = self.rep; - let kernel = self.kernel; // Window bounds in VIRTUAL `[dict][input]` coords, so the kernel's gates // match the owned flat dict kernel: `window_low` is the absolute floor; @@ -1539,7 +1527,7 @@ impl FastKernelMatcher { macro_rules! run { ($mls:literal, $cmov:literal) => { - compress_block_fast_dict_borrowed::<$mls, $cmov>( + compress_block_fast_dict_borrowed::( inp, dict, block_start, @@ -1550,7 +1538,6 @@ impl FastKernelMatcher { rep_in, step_size, &mut handle_sequence, - kernel, ) }; } diff --git a/zstd/src/encoding/streaming_encoder.rs b/zstd/src/encoding/streaming_encoder.rs index 069fcdb9..6604928f 100644 --- a/zstd/src/encoding/streaming_encoder.rs +++ b/zstd/src/encoding/streaming_encoder.rs @@ -10,6 +10,7 @@ use core::hash::Hasher; #[cfg(feature = "hash")] use twox_hash::XxHash64; +use crate::encoding::fastpath::EncodeTier; use crate::encoding::levels::compress_block_encoded; use crate::encoding::{ CompressionLevel, EncoderDictionary, MatchGeneratorDriver, Matcher, block_header::BlockHeader, @@ -725,6 +726,20 @@ impl CompressionContext { /// set with [`set_pledged_content_size`](Self::set_pledged_content_size) /// allows less. pub fn write(&mut self, drain: &mut D, buf: &[u8]) -> Result { + // The call's one choice of kernel tier: the frame start and every block + // it emits run compiled for it. + crate::encoding::fastpath::with_encode_tier!( + self.state.copy_kernel, + K => self.write_with::(drain, buf) + ) + } + + /// [`Self::write`] compiled for the kernel tier `K`. + fn write_with( + &mut self, + drain: &mut D, + buf: &[u8], + ) -> Result { self.ensure_open()?; if buf.is_empty() { return Ok(0); @@ -741,7 +756,7 @@ impl CompressionContext { )); } - self.ensure_frame_started(drain)?; + self.ensure_frame_started::(drain)?; // Enforce pledged upper bound: truncate the accepted slice to the // remaining allowance so that partial-write semantics are honored @@ -773,7 +788,10 @@ impl CompressionContext { remaining = &remaining[to_take..]; consumed += to_take; - if self.emit_full_pending_block(drain, block_capacity).is_err() { + if self + .emit_full_pending_block::(drain, block_capacity) + .is_err() + { // The bytes this call took are reported; the failure, sticky // on the context now, is what the next call returns. break; @@ -789,8 +807,13 @@ impl CompressionContext { if self.pending_len() == 0 { return drain.flush().map_err(|err| self.fail(err)); } - self.ensure_frame_started(drain)?; - self.emit_pending_block(drain, false)?; + crate::encoding::fastpath::with_encode_tier!( + self.state.copy_kernel, + K => { + self.ensure_frame_started::(drain)?; + self.emit_pending_block::(drain, false) + } + )?; drain.flush().map_err(|err| self.fail(err)) } @@ -812,10 +835,15 @@ impl CompressionContext { )); } - self.ensure_frame_started(drain)?; // An empty frame, or one whose input ended on a block boundary, closes // with an empty last block. - self.emit_pending_block(drain, true)?; + crate::encoding::fastpath::with_encode_tier!( + self.state.copy_kernel, + K => { + self.ensure_frame_started::(drain)?; + self.emit_pending_block::(drain, true) + } + )?; #[cfg(feature = "hash")] if self.content_checksum { @@ -881,7 +909,10 @@ impl CompressionContext { /// ``` pub fn start_frame(&mut self, drain: &mut D) -> Result<(), Error> { self.ensure_open()?; - self.ensure_frame_started(drain) + crate::encoding::fastpath::with_encode_tier!( + self.state.copy_kernel, + K => self.ensure_frame_started::(drain) + ) } /// The largest block the open frame cuts: the block size limits held to @@ -935,7 +966,11 @@ impl CompressionContext { // so the post-split, which would emit several physical blocks, stays // off (upstream `ZSTD_compressBlock` never splits either). let post_split = core::mem::replace(&mut self.state.post_split_allowed, false); - let encoded = self.encode_block(payload, block.len(), false); + // The context's one choice of kernel tier for this block. + let encoded = crate::encoding::fastpath::with_encode_tier!( + self.state.copy_kernel, + K => self.encode_block::<_, K>(payload, block.len(), false) + ); self.state.post_split_allowed = post_split; encoded.map_err(|err| self.fail(err))?; self.bytes_consumed += block.len() as u64; @@ -1021,7 +1056,10 @@ impl CompressionContext { } } - fn ensure_frame_started(&mut self, drain: &mut D) -> Result<(), Error> { + fn ensure_frame_started( + &mut self, + drain: &mut D, + ) -> Result<(), Error> { if self.frame_started { return Ok(()); } @@ -1119,7 +1157,7 @@ impl CompressionContext { self.state.offset_hist = [1, 4, 8]; if use_dictionary_state && let Some(dict) = self.dictionary.as_ref() { self.state.offset_hist = dict.inner.offset_hist; - crate::encoding::frame_compressor::load_frame_dictionary( + crate::encoding::frame_compressor::load_frame_dictionary::( &mut self.state, self.compression_level, dict, @@ -1268,7 +1306,7 @@ impl CompressionContext { /// `remaining` is the input still to come as far as the splitter knows: /// a full block again while writes continue, the buffered bytes at the /// end of the frame. - fn pre_split_len(&mut self, block_capacity: usize, remaining: usize) -> usize { + fn pre_split_len(&mut self, block_capacity: usize, remaining: usize) -> usize { if matches!(self.compression_level, CompressionLevel::Uncompressed) { return self.pending_len(); } @@ -1285,7 +1323,10 @@ impl CompressionContext { // The splitter sees byte statistics only; a cut it proposes inside a // match the matcher already knows of is moved out. if cut < window { - self.state.matcher.settle_block_cut(window, cut) + match self.state.matcher.builtin_driver() { + Some(driver) => driver.settle_block_cut_with::(window, cut), + None => self.state.matcher.settle_block_cut(window, cut), + } } else { cut } @@ -1294,7 +1335,7 @@ impl CompressionContext { /// Emit the pending block once it is full, cut where the pre-splitter /// says; the suffix stays pending and heads the next block. A failure /// leaves the context failed. - fn emit_full_pending_block( + fn emit_full_pending_block( &mut self, drain: &mut D, block_capacity: usize, @@ -1302,12 +1343,12 @@ impl CompressionContext { if self.pending_len() != block_capacity { return Ok(()); } - let block_len = self.pre_split_len(block_capacity, block_capacity); - self.encode_block(drain, block_len, false) + let block_len = self.pre_split_len::(block_capacity, block_capacity); + self.encode_block::(drain, block_len, false) .map_err(|err| self.fail(err)) } - fn emit_pending_block( + fn emit_pending_block( &mut self, drain: &mut D, last_block: bool, @@ -1318,16 +1359,16 @@ impl CompressionContext { // path splits it with `remaining = len`); each cut prefix goes // out as a non-last block and the suffix is re-examined. while self.pending_len() == block_capacity { - let block_len = self.pre_split_len(block_capacity, block_capacity); + let block_len = self.pre_split_len::(block_capacity, block_capacity); if block_len == block_capacity { break; } - self.encode_block(drain, block_len, false) + self.encode_block::(drain, block_len, false) .map_err(|err| self.fail(err))?; } } let block_len = self.pending_len(); - self.encode_block(drain, block_len, last_block) + self.encode_block::(drain, block_len, last_block) .map_err(|err| self.fail(err)) } @@ -1347,7 +1388,7 @@ impl CompressionContext { /// Writes the first `block_len` pending bytes to `drain` as one block, /// `last_block` if it closes the frame. - fn encode_block( + fn encode_block( &mut self, drain: &mut D, block_len: usize, @@ -1384,7 +1425,7 @@ impl CompressionContext { } let dict_active = self.dictionary.is_some() && self.state.matcher.supports_dictionary_priming(); - compress_block_encoded( + compress_block_encoded::( &mut self.state, self.compression_level, last_block, From 9ce90b49ec04d7f1755ab6383185288be2902b2d Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 24 Oct 2026 21:09:49 +0300 Subject: [PATCH 06/15] test(encode): run the BT tier-identity test per tier The x86-only test forced a tier through the matcher's former `kernel` field; it now runs the search under each tier the CPU supports. --- zstd/src/encoding/hc/generator.rs | 5 +++- zstd/src/encoding/match_generator/tests.rs | 32 +++++++++++----------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs index 0b8986f5..2474356c 100644 --- a/zstd/src/encoding/hc/generator.rs +++ b/zstd/src/encoding/hc/generator.rs @@ -1548,7 +1548,10 @@ impl HcMatchGenerator { /// [`Self::start_matching`] compiled for the tier `K`. #[cfg(test)] - fn start_matching_with(&mut self, mut handle_sequence: impl FnMut(Sequence)) { + pub(crate) fn start_matching_with( + &mut self, + mut handle_sequence: impl FnMut(Sequence), + ) { use crate::encoding::strategy::{self, StrategyTag}; // Dispatch on the mirrored `strategy_tag` so each test runs // under the same monomorphisation production would pick. diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index e9e5779b..cbc8fa1f 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -729,13 +729,11 @@ fn btultra2_seed_pass_initializes_opt_state() { ); } -/// Every per-CPU kernel tier emits a bit-identical sequence stream. Forcing -/// `table.kernel` runs each tier's monomorphized BT-collect / DP wrapper on -/// one machine (only the runtime-selected tier would otherwise execute), which -/// both pins the scalar-vs-SIMD bit-identity invariant and exercises the -/// per-tier wrappers the runtime dispatch leaves cold. x86-only: the aarch64 -/// path dispatches NEON unconditionally, so the cached field is read only in -/// the x86 `cfg` block. +/// Every per-CPU kernel tier emits a bit-identical sequence stream. Running +/// the search under each tier the CPU supports executes each monomorphized +/// BT-collect / DP wrapper on one machine (only the detected tier would +/// otherwise run), which pins the scalar-vs-SIMD bit-identity invariant. +/// x86-only: aarch64 has a single NEON tier. #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] #[test] fn bt_optimal_all_kernel_tiers_emit_identical_sequences() { @@ -778,16 +776,18 @@ fn bt_optimal_all_kernel_tiers_emit_identical_sequences() { 26, ); hc.table.commit_input(&data); - hc.table.kernel = tier; let mut seqs = Vec::new(); - hc.start_matching(|seq| match seq { - Sequence::Triple { - literal_len, - offset, - match_len, - } => seqs.push((literal_len, offset, match_len)), - Sequence::Literals { len } => seqs.push((len, 0, 0)), - }); + crate::encoding::fastpath::with_encode_tier!( + tier, + K => hc.start_matching_with::(|seq| match seq { + Sequence::Triple { + literal_len, + offset, + match_len, + } => seqs.push((literal_len, offset, match_len)), + Sequence::Literals { len } => seqs.push((len, 0, 0)), + }) + ); seqs }; From da7827a8cc6689021565832d13a8c918a0fd82da Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 24 Oct 2026 23:55:22 +0300 Subject: [PATCH 07/15] fix(encode): route the wasm row search by tier - The row lazy search on wasm32 with simd128 bypassed `K::TIER` behind a cfg block of its own, which left `FastpathKernel` unused there and failed the wasm clippy job. It is now one more arm of the same tier match every other backend uses. - `BitBurst::flush` stores the accumulator with one unaligned `u64` write instead of copying it through a byte array first. --- zstd/src/bit_io/bit_writer.rs | 7 ++- zstd/src/encoding/row/mod.rs | 84 ++++++++++++++++------------------- 2 files changed, 44 insertions(+), 47 deletions(-) diff --git a/zstd/src/bit_io/bit_writer.rs b/zstd/src/bit_io/bit_writer.rs index a8d03489..9a198e82 100644 --- a/zstd/src/bit_io/bit_writer.rs +++ b/zstd/src/bit_io/bit_writer.rs @@ -415,9 +415,12 @@ impl>> BitBurst<'_, V> { self.cursor.wrapping_add(8) <= self.end, "a burst flushed past the room it reserved", ); - // SAFETY: the cursor is 8 bytes short of the reserved end (above). + // SAFETY: the cursor is 8 bytes short of the reserved end (above), and + // an unaligned store has no alignment requirement. unsafe { - core::ptr::copy_nonoverlapping(self.partial.to_le_bytes().as_ptr(), self.cursor, 8); + self.cursor + .cast::() + .write_unaligned(self.partial.to_le()); self.cursor = self.cursor.add(nb_bytes); } // A full accumulator (64 bits) leaves nothing: shifting a `u64` by 64 diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 87acbbb5..79b91121 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -4007,51 +4007,45 @@ impl RowMatchGenerator { // siblings directly with the type and const already bound. `skip` does // no tag compare, so it dispatches on `row_log` only. pub(crate) fn start_matching(&mut self, handle_sequence: impl FnMut(Sequence)) { - // SAFETY: same per-tier umbrella contract as `start_matching_greedy`. - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ))] - { - // SAFETY: simd128 is a compile-time feature here; no runtime gate. - unsafe { dispatch_lazy!(self.lazy_simd128::(handle_sequence)) } - } - #[cfg(not(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - )))] - { - match K::TIER { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-avx2" - ))] - FastpathKernel::Avx2Bmi2 => unsafe { - dispatch_lazy!(self.lazy_avx2bmi2::(handle_sequence)) - }, - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" - ))] - FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { - dispatch_lazy!(self.lazy_sse2::(handle_sequence)) - }, - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - FastpathKernel::Neon => unsafe { - dispatch_lazy!(self.lazy_neon::(handle_sequence)) - }, - // SAFETY: the scalar kernel has no `#[target_feature]`; the - // fn is `unsafe` only for macro uniformity. - FastpathKernel::Scalar => unsafe { - dispatch_lazy!(self.lazy_scalar::(handle_sequence)) - }, - } + // SAFETY: same per-tier umbrella contract as `start_matching_greedy`: + // `K` names a tier the encoder confirmed this CPU runs. + match K::TIER { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" + ))] + FastpathKernel::Avx2Bmi2 => unsafe { + dispatch_lazy!(self.lazy_avx2bmi2::(handle_sequence)) + }, + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" + ))] + FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { + dispatch_lazy!(self.lazy_sse2::(handle_sequence)) + }, + #[cfg(all( + target_arch = "aarch64", + target_endian = "little", + feature = "kernel-neon" + ))] + FastpathKernel::Neon => unsafe { + dispatch_lazy!(self.lazy_neon::(handle_sequence)) + }, + // simd128 is a compile-time feature on wasm; no runtime gate. + #[cfg(all( + target_arch = "wasm32", + target_feature = "simd128", + feature = "kernel-simd128" + ))] + FastpathKernel::Simd128 => unsafe { + dispatch_lazy!(self.lazy_simd128::(handle_sequence)) + }, + // SAFETY: the scalar kernel has no `#[target_feature]`; the + // fn is `unsafe` only for macro uniformity. + FastpathKernel::Scalar => unsafe { + dispatch_lazy!(self.lazy_scalar::(handle_sequence)) + }, } } From c9f2f7366604532472f27de0c3cfec1740520281 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sun, 25 Oct 2026 00:02:06 +0300 Subject: [PATCH 08/15] fix(encode): detect the driver's tier at construction - The match generator's `Matcher` entry points, which wrappers such as the dictionary finalizer's recorder and the sequence capture drive, asked the CPU detector on every block. The driver now detects its tier once when it is built and those entry points run under that value. - The tier-identity test of the block encoder adds sequences whose extra bits take both conditional flushes of the sequence loop, which the narrow fixture never reached. --- zstd/src/encoding/blocks/compressed/tests.rs | 14 ++++++++++++- zstd/src/encoding/match_generator/mod.rs | 21 +++++++++++++------- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/zstd/src/encoding/blocks/compressed/tests.rs b/zstd/src/encoding/blocks/compressed/tests.rs index 64426599..f6e742c0 100644 --- a/zstd/src/encoding/blocks/compressed/tests.rs +++ b/zstd/src/encoding/blocks/compressed/tests.rs @@ -1338,7 +1338,7 @@ fn every_tier_writes_the_block_the_scalar_tier_writes() { .collect(); // Lengths and offsets across their extra-bit ranges, so the field widths // the loop shifts by vary from one sequence to the next. - let sequences: Vec = (0..4_000u32) + let mut sequences: Vec = (0..4_000u32) .map(|i| { let mix = i.wrapping_mul(0x9E37_79B9); RawSequence { @@ -1348,6 +1348,18 @@ fn every_tier_writes_the_block_the_scalar_tier_writes() { } }) .collect(); + // Wide extras for both conditional flushes of the loop: 16 + 16 + 24 = + // 56 extra bits takes the one before the extras (31 or more) alone, and + // 16 + 16 + 30 = 62 takes the one before the offset (more than 56) too. + for (k, seq) in sequences.iter_mut().step_by(97).enumerate() { + seq.ll = 40_000 + k as u32; + seq.ml = 60_000 + k as u32; + seq.off_base = if k % 2 == 0 { + (1u32 << 24) + k as u32 + } else { + (1u32 << 30) + k as u32 + }; + } let encode = |kernel| { let mut state = CompressState:: { matcher: EntropyOnlyMatcher, diff --git a/zstd/src/encoding/match_generator/mod.rs b/zstd/src/encoding/match_generator/mod.rs index 95d347b5..18593663 100644 --- a/zstd/src/encoding/match_generator/mod.rs +++ b/zstd/src/encoding/match_generator/mod.rs @@ -496,6 +496,10 @@ pub struct MatchGeneratorDriver { /// A parser asked for each block in place of the backends, when one is /// installed (C `ZSTD_registerSequenceProducer`). producer: Option, + /// This CPU's kernel tier, detected once at construction: the tier the + /// [`Matcher`] entry points run under when a caller drives the driver + /// through the trait rather than through the encoder's own tier. + tier: super::fastpath::FastpathKernel, } /// A [`SequenceProducer`] and what to do when it gives no parse. @@ -800,6 +804,7 @@ impl MatchGeneratorDriver { #[cfg(feature = "ldm")] ldm_split_block: None, producer: None, + tier: super::fastpath::select_kernel(), } } @@ -2112,7 +2117,7 @@ impl Matcher for MatchGeneratorDriver { // For a caller driving the matcher through the trait; the encoder // calls `prime_with_dictionary_with` under its frame's tier. crate::encoding::fastpath::with_encode_tier!( - crate::encoding::fastpath::select_kernel(), + self.tier, K => self.prime_with_dictionary_with::(dict_content, offset_hist) ) } @@ -2176,7 +2181,7 @@ impl Matcher for MatchGeneratorDriver { // For a caller driving the matcher through the trait; the encoder // calls `settle_block_cut_with` under its frame's tier. crate::encoding::fastpath::with_encode_tier!( - crate::encoding::fastpath::select_kernel(), + self.tier, K => self.settle_block_cut_with::(window, cut) ) } @@ -2269,11 +2274,13 @@ impl Matcher for MatchGeneratorDriver { } fn start_matching(&mut self, handle_sequence: impl FnMut(Sequence)) { - // For a caller driving the matcher through the trait itself. The - // encoder does not come here: it reaches the driver through - // `builtin_driver` and searches under the tier it chose for the frame. + // For a caller driving the matcher through the trait itself, such as + // a wrapper that records what it emits. The encoder does not come + // here: it reaches the driver through `builtin_driver` and searches + // under the tier it chose for the frame. Either way the tier was + // detected once, at construction. crate::encoding::fastpath::with_encode_tier!( - crate::encoding::fastpath::select_kernel(), + self.tier, K => self.start_matching_with::(handle_sequence) ) } @@ -2291,7 +2298,7 @@ impl Matcher for MatchGeneratorDriver { // For a caller driving the matcher through the trait; the encoder // calls `skip_matching_with_hint_with` under its frame's tier. crate::encoding::fastpath::with_encode_tier!( - crate::encoding::fastpath::select_kernel(), + self.tier, K => self.skip_matching_with_hint_with::(incompressible_hint) ) } From 4ce1d62810be8ce2f61dcdb8a9fe34dec66aeba3 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sun, 25 Oct 2026 02:09:31 +0300 Subject: [PATCH 09/15] build(encode): instantiate only the chosen tier's kernel - A `match` on `K::TIER` folds in an optimised build, but an unoptimised one still instantiates every arm's callee for every tier, so each tier's monomorph of a backend carried the kernels of all the others. The test build of the library emitted about three times the LLVM IR of `main` and its compiler peaked near 16 GB, past what a hosted CI runner holds. - `tier_match!` writes the same arms as `if const` tests, whose untaken branches are dropped before code generation, and keeps the `match`'s exhaustiveness check in an inline `const`. Every per-tier kernel choice and the test-only dictionary-loop door use it. The machine code of an optimised build does not change. --- zstd/src/encoding/blocks/compressed.rs | 4 +- .../blocks/compressed/literal_runs.rs | 6 +-- zstd/src/encoding/dfast/mod.rs | 16 +++++--- zstd/src/encoding/fastpath/mod.rs | 37 +++++++++++++++++-- zstd/src/encoding/hc/mod.rs | 4 +- zstd/src/encoding/hc/optimal.rs | 12 +++--- zstd/src/encoding/match_table/storage.rs | 12 +++--- zstd/src/encoding/row/mod.rs | 10 ++--- .../src/encoding/simple/fast_kernel/kernel.rs | 18 ++++----- zstd/src/huff0/huf_cstream.rs | 4 +- 10 files changed, 79 insertions(+), 44 deletions(-) diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index 684bc30b..15ec90b1 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -3028,7 +3028,7 @@ fn encode_sequences( // the caller is holding its slots. defaults: [&FSETable; 3], ) { - match K::TIER { + crate::encoding::fastpath::tier_match!(K { #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-avx2" @@ -3038,7 +3038,7 @@ fn encode_sequences( encode_sequences_bmi2(sequences, codes, writer, modes, defaults) }, _ => encode_sequences_scalar(sequences, codes, writer, modes, defaults), - } + }) } #[cfg(all( diff --git a/zstd/src/encoding/blocks/compressed/literal_runs.rs b/zstd/src/encoding/blocks/compressed/literal_runs.rs index 94a5ed1d..aacd0d85 100644 --- a/zstd/src/encoding/blocks/compressed/literal_runs.rs +++ b/zstd/src/encoding/blocks/compressed/literal_runs.rs @@ -35,7 +35,7 @@ pub(super) fn gather_literals( if dst.capacity() - dst.len() < block.len() { crate::encoding::workspace::capacity_exhausted(); } - match K::TIER { + crate::encoding::fastpath::tier_match!(K { #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-avx2" @@ -66,9 +66,9 @@ pub(super) fn gather_literals( ))] crate::encoding::fastpath::FastpathKernel::Simd128 => { gather_simd128(block, sequences, tail, dst) - } + }, _ => gather_scalar(block, sequences, tail, dst), - } + }) } /// Bytes of the block that must remain past a run for it to be copied in whole diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs index 4dccc7fb..916ad77c 100644 --- a/zstd/src/encoding/dfast/mod.rs +++ b/zstd/src/encoding/dfast/mod.rs @@ -3792,8 +3792,8 @@ impl DfastMatchGenerator { /// The dfast loop of the encoder tier `K`: its /// `start_matching_fast_loop_` wrapper, so every per-position /// `common_prefix_len_ptr` inlines under one `#[target_feature]` umbrella - /// (mirrors `build_optimal_plan_impl`). The match on `K::TIER` folds to - /// that one call. + /// (mirrors `build_optimal_plan_impl`). The tier match keeps only that one + /// call. fn start_matching_fast_loop( &mut self, current_abs_start: usize, @@ -3867,7 +3867,9 @@ impl DfastMatchGenerator { // target can check the two against each other — the branch is // `cfg(test)` and does not exist in a shipped build. #[cfg(test)] - if use_dict && K::TIER == crate::encoding::fastpath::FastpathKernel::Scalar { + if use_dict + && const { matches!(K::TIER, crate::encoding::fastpath::FastpathKernel::Scalar) } + { return self.start_matching_dict_loop_scalar( current_abs_start, current_len, @@ -3879,7 +3881,7 @@ impl DfastMatchGenerator { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match K::TIER { + crate::encoding::fastpath::tier_match!(K { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { dispatch_dict!( @@ -3895,7 +3897,7 @@ impl DfastMatchGenerator { start_matching_fast_loop_scalar, start_matching_dict_loop_scalar ), - } + }) } #[cfg(all( target_arch = "wasm32", @@ -3907,7 +3909,9 @@ impl DfastMatchGenerator { // compile time here, so without it this target could never run the // scalar dictionary loop to check the SIMD one against. #[cfg(test)] - if use_dict && K::TIER == crate::encoding::fastpath::FastpathKernel::Scalar { + if use_dict + && const { matches!(K::TIER, crate::encoding::fastpath::FastpathKernel::Scalar) } + { return self.start_matching_dict_loop_scalar( current_abs_start, current_len, diff --git a/zstd/src/encoding/fastpath/mod.rs b/zstd/src/encoding/fastpath/mod.rs index 76d847ea..4cd0e039 100644 --- a/zstd/src/encoding/fastpath/mod.rs +++ b/zstd/src/encoding/fastpath/mod.rs @@ -310,9 +310,9 @@ fn detect_kernel_uncached() -> FastpathKernel { /// A [`FastpathKernel`] tier as a type. The encoder chooses its tier once, in /// [`with_encode_tier`], and everything below that choice is compiled per tier: -/// a leaf that runs a different kernel per tier matches on `K::TIER`, which -/// folds to the one kernel at monomorphisation, so no tier is asked at run time -/// below the choice. +/// a leaf that runs a different kernel per tier picks it with [`tier_match`], +/// which keeps the one kernel at monomorphisation, so no tier is asked at run +/// time below the choice. pub(crate) trait EncodeTier { /// The tier this type stands for. const TIER: FastpathKernel; @@ -455,6 +455,37 @@ macro_rules! with_encode_tier { } pub(crate) use with_encode_tier; +/// A `match` on `$k`'s [`EncodeTier::TIER`] that instantiates only the arm of +/// that tier. +/// +/// A plain `match` on the constant folds in an optimised build, but an +/// unoptimised one still instantiates every arm's callee for every tier, so +/// each tier's monomorph would carry the kernels of all the others. Each arm +/// here is an `if` on an inline `const`, whose untaken branch the compiler +/// drops before code generation. The arms are checked for exhaustiveness as the +/// `match` they stand for. +macro_rules! tier_match { + ($k:ty { $( $(#[$attr:meta])* $pat:pat => $arm:expr ),+ $(,)? }) => {{ + const { + match <$k as $crate::encoding::fastpath::EncodeTier>::TIER { + $( $(#[$attr])* $pat => (), )+ + } + }; + 'tier: { + $( + $(#[$attr])* + if const { + matches!(<$k as $crate::encoding::fastpath::EncodeTier>::TIER, $pat) + } { + break 'tier ($arm); + } + )+ + unreachable!() + } + }}; +} +pub(crate) use tier_match; + /// [`with_encode_tier!`] on this CPU's tier, the one the encoder picks: unit /// tests drive a backend directly through the kernels it would ship with. #[cfg(test)] diff --git a/zstd/src/encoding/hc/mod.rs b/zstd/src/encoding/hc/mod.rs index 38429c0c..8376cde1 100644 --- a/zstd/src/encoding/hc/mod.rs +++ b/zstd/src/encoding/hc/mod.rs @@ -747,7 +747,7 @@ impl HcMatcher { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match K::TIER { + crate::encoding::fastpath::tier_match!(K { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { self.for_each_repcode_candidate_with_reps_avx2_bmi2( @@ -781,7 +781,7 @@ impl HcMatcher { min_match_len, f, ), - } + }) } #[cfg(not(any( all( diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index bbb12665..a522c7dc 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -1504,7 +1504,7 @@ impl HcMatchGenerator { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match K::TIER { + crate::encoding::fastpath::tier_match!(K { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => run_main_loop!(run_optimal_block_avx2_bmi2), #[cfg(feature = "kernel-sse")] @@ -1512,7 +1512,7 @@ impl HcMatchGenerator { #[cfg(feature = "kernel-sse")] FastpathKernel::Sse42 => run_main_loop!(run_optimal_block_sse42), FastpathKernel::Scalar => run_main_loop!(run_optimal_block_scalar), - } + }) } #[cfg(all( target_arch = "wasm32", @@ -1705,7 +1705,7 @@ impl HcMatchGenerator { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match K::TIER { + crate::encoding::fastpath::tier_match!(K { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => run_seed_loop!(run_optimal_block_avx2_bmi2), #[cfg(feature = "kernel-sse")] @@ -1713,7 +1713,7 @@ impl HcMatchGenerator { #[cfg(feature = "kernel-sse")] FastpathKernel::Sse42 => run_seed_loop!(run_optimal_block_sse42), FastpathKernel::Scalar => run_seed_loop!(run_optimal_block_scalar), - } + }) } #[cfg(all( target_arch = "wasm32", @@ -2231,7 +2231,7 @@ impl HcMatchGenerator { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match K::TIER { + crate::encoding::fastpath::tier_match!(K { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { self.collect_optimal_candidates_initialized_avx2_bmi2::( @@ -2274,7 +2274,7 @@ impl HcMatchGenerator { ll0, out, ), - } + }) } // wasm resolves `simd128` at compile time (no runtime detection), so // the tier comes from `cfg`, not from `self.table.kernel`. diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index 61ed24f1..f472e138 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -1752,7 +1752,7 @@ impl MatchTable { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match K::TIER { + crate::encoding::fastpath::tier_match!(K { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { self.bt_insert_range_avx2_bmi2(from, stop, current_abs_end, target_abs) @@ -1763,8 +1763,8 @@ impl MatchTable { }, FastpathKernel::Scalar => { self.bt_insert_range_scalar(from, stop, current_abs_end, target_abs) - } - } + }, + }) } // wasm has no runtime CPU detection: `simd128` is resolved at compile // time, so the tier is picked by `cfg` rather than by `K`. @@ -2546,7 +2546,7 @@ impl MatchTable { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; - match K::TIER { + crate::encoding::fastpath::tier_match!(K { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { self.hash3_candidate_avx2_bmi2(abs_pos, current_abs_end, min_match_len) @@ -2557,8 +2557,8 @@ impl MatchTable { }, FastpathKernel::Scalar => { self.hash3_candidate_scalar(abs_pos, current_abs_end, min_match_len) - } - } + }, + }) } #[cfg(all( target_arch = "wasm32", diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 79b91121..1284adb4 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -507,7 +507,7 @@ macro_rules! dispatch_tag_kernel { feature = "kernel-simd128" )))] { - match <$tier as EncodeTier>::TIER { + crate::encoding::fastpath::tier_match!($tier { #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-avx2" @@ -519,7 +519,7 @@ macro_rules! dispatch_tag_kernel { ))] FastpathKernel::Sse2 | FastpathKernel::Sse42 => { $self.$k_method::($($arg),*) - } + }, #[cfg(all( target_arch = "aarch64", target_endian = "little", @@ -527,7 +527,7 @@ macro_rules! dispatch_tag_kernel { ))] FastpathKernel::Neon => $self.$k_method::($($arg),*), FastpathKernel::Scalar => $self.$k_method::($($arg),*), - } + }) } }}; } @@ -4009,7 +4009,7 @@ impl RowMatchGenerator { pub(crate) fn start_matching(&mut self, handle_sequence: impl FnMut(Sequence)) { // SAFETY: same per-tier umbrella contract as `start_matching_greedy`: // `K` names a tier the encoder confirmed this CPU runs. - match K::TIER { + crate::encoding::fastpath::tier_match!(K { #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-avx2" @@ -4046,7 +4046,7 @@ impl RowMatchGenerator { FastpathKernel::Scalar => unsafe { dispatch_lazy!(self.lazy_scalar::(handle_sequence)) }, - } + }) } gen_lazy_monolith!( diff --git a/zstd/src/encoding/simple/fast_kernel/kernel.rs b/zstd/src/encoding/simple/fast_kernel/kernel.rs index fdab6d97..573c5106 100644 --- a/zstd/src/encoding/simple/fast_kernel/kernel.rs +++ b/zstd/src/encoding/simple/fast_kernel/kernel.rs @@ -2101,7 +2101,7 @@ pub(crate) fn compress_block_fast_dict_borrowed< }; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { - match K::TIER { + crate::encoding::fastpath::tier_match!(K { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { cbfd_borrowed_avx2_bmi2::( @@ -2135,8 +2135,8 @@ pub(crate) fn compress_block_fast_dict_borrowed< FastpathKernel::Scalar => { let mut hs = handle_sequence; scalar(inp, dict, main_table, dict_table, &mut hs) - } - } + }, + }) } #[cfg(all( target_arch = "aarch64", @@ -2144,7 +2144,7 @@ pub(crate) fn compress_block_fast_dict_borrowed< feature = "kernel-neon" ))] { - match K::TIER { + crate::encoding::fastpath::tier_match!(K { FastpathKernel::Neon => unsafe { cbfd_borrowed_neon::( inp, @@ -2162,8 +2162,8 @@ pub(crate) fn compress_block_fast_dict_borrowed< FastpathKernel::Scalar => { let mut hs = handle_sequence; scalar(inp, dict, main_table, dict_table, &mut hs) - } - } + }, + }) } #[cfg(all( target_arch = "wasm32", @@ -2171,7 +2171,7 @@ pub(crate) fn compress_block_fast_dict_borrowed< feature = "kernel-simd128" ))] { - match K::TIER { + crate::encoding::fastpath::tier_match!(K { FastpathKernel::Simd128 => unsafe { cbfd_borrowed_simd128::( inp, @@ -2189,8 +2189,8 @@ pub(crate) fn compress_block_fast_dict_borrowed< FastpathKernel::Scalar => { let mut hs = handle_sequence; scalar(inp, dict, main_table, dict_table, &mut hs) - } - } + }, + }) } #[cfg(not(any( target_arch = "x86", diff --git a/zstd/src/huff0/huf_cstream.rs b/zstd/src/huff0/huf_cstream.rs index 31585449..a8a6acf7 100644 --- a/zstd/src/huff0/huf_cstream.rs +++ b/zstd/src/huff0/huf_cstream.rs @@ -366,7 +366,7 @@ impl<'a> HufCStream<'a> { table: &[u64], data: &[u8], ) { - match K::TIER { + crate::encoding::fastpath::tier_match!(K { #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "kernel-avx2" @@ -378,7 +378,7 @@ impl<'a> HufCStream<'a> { self.encode_unrolled_bmi2::(table, data) }, _ => self.encode_unrolled_scalar::(table, data), - } + }) } /// Portable burst-encode kernel (no target feature). Bit-identical output to From 389500e9cafc4b61df1c001e1ba18d0ca6c02fcc Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sun, 25 Oct 2026 02:24:24 +0300 Subject: [PATCH 10/15] build(encode): one sink type behind the matcher trait - A caller that drives the match generator through `Matcher` (a wrapper that records what it emits, and the unit tests) got a copy of the whole search, every backend and kernel, per closure it passed. The trait entry now hands the search a `dyn` sink, so those callers share one copy per tier. The encoder reaches the driver directly and keeps its sink's concrete type, so its path is unchanged. --- zstd/src/encoding/match_generator/mod.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/zstd/src/encoding/match_generator/mod.rs b/zstd/src/encoding/match_generator/mod.rs index 18593663..de296c74 100644 --- a/zstd/src/encoding/match_generator/mod.rs +++ b/zstd/src/encoding/match_generator/mod.rs @@ -2273,15 +2273,20 @@ impl Matcher for MatchGeneratorDriver { } } - fn start_matching(&mut self, handle_sequence: impl FnMut(Sequence)) { + fn start_matching(&mut self, mut handle_sequence: impl FnMut(Sequence)) { // For a caller driving the matcher through the trait itself, such as // a wrapper that records what it emits. The encoder does not come // here: it reaches the driver through `builtin_driver` and searches // under the tier it chose for the frame. Either way the tier was // detected once, at construction. + // + // The sink is passed on as one `dyn` type, so the search is compiled + // once per tier for every such caller rather than once per caller's + // closure; the encoder's own path keeps its sink's concrete type. + let sink: &mut dyn FnMut(Sequence) = &mut handle_sequence; crate::encoding::fastpath::with_encode_tier!( self.tier, - K => self.start_matching_with::(handle_sequence) + K => self.start_matching_with::(&mut *sink) ) } From 338a02cdf1fb6739349ed9c6b69c731d89d958d7 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sun, 25 Oct 2026 02:31:53 +0300 Subject: [PATCH 11/15] test(encode): hand backend searches one sink type - Unit tests that drive a backend's search directly passed it their own closure, so each test compiled the search, every row log and kernel variant, again per closure and per tier. They now pass it through `dyn_sink`, and all of them share one copy per tier. - A lint-job gate rejects the two shapes that multiply the debug build: a plain `match` (or comparison) on `K::TIER` below the tier choice, and a closure handed straight to `start_matching::`. --- .github/workflows/ci.yml | 21 ++++++++++ zstd/src/encoding/fastpath/mod.rs | 10 +++++ zstd/src/encoding/hc/generator/tests.rs | 10 ++--- zstd/src/encoding/match_generator/tests.rs | 48 +++++++++++----------- 4 files changed, 60 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25c7b5e8..45580923 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -211,6 +211,27 @@ jobs: echo "::error::bench-only instrumentation symbols leaked into zstd/src/" exit 1 fi + - name: Gate — one search per tier in the debug build + # Below the encoder's one tier choice, code is compiled per tier. A + # plain `match` on `K::TIER` instantiates every arm's kernel for every + # tier in an unoptimised build (`tier_match!` does not), and a test + # closure handed straight to a search compiles that search again per + # closure (`dyn_sink` does not). Either one multiplies the test build's + # IR until the compiler no longer fits on a hosted runner. + # `fastpath/mod.rs` defines `tier_match!` itself, whose exhaustiveness + # check is the one `match` on the constant that belongs there. + run: | + found=$(rg -n --no-heading -g '!zstd/src/encoding/fastpath/mod.rs' \ + -e 'match\s+[A-Za-z_]+::TIER\b' \ + -e 'match\s+<[^>]+>::TIER\b' \ + -e '::TIER\s*[=!]=' \ + -e 'start_matching(_borrowed)?::\(\s*\|' \ + zstd/src/ || true) + if [ -n "$found" ]; then + echo "$found" + echo "::error::per-tier code must pick its kernel with tier_match! and take test sinks through dyn_sink" + exit 1 + fi c-abi: # Builds the libzstd-compatible C ABI front end and verifies it is a real diff --git a/zstd/src/encoding/fastpath/mod.rs b/zstd/src/encoding/fastpath/mod.rs index 4cd0e039..c6ce39fd 100644 --- a/zstd/src/encoding/fastpath/mod.rs +++ b/zstd/src/encoding/fastpath/mod.rs @@ -500,5 +500,15 @@ macro_rules! with_host_tier { #[cfg(test)] pub(crate) use with_host_tier; +/// A test's sequence closure as the one sink type tests hand a backend, so +/// each search is compiled once per tier for all of them rather than once per +/// closure and tier. +#[cfg(test)] +pub(crate) fn dyn_sink( + sink: &mut impl FnMut(crate::encoding::Sequence), +) -> &mut dyn FnMut(crate::encoding::Sequence) { + sink +} + #[cfg(test)] mod tests; diff --git a/zstd/src/encoding/hc/generator/tests.rs b/zstd/src/encoding/hc/generator/tests.rs index 532e0597..1f893af9 100644 --- a/zstd/src/encoding/hc/generator/tests.rs +++ b/zstd/src/encoding/hc/generator/tests.rs @@ -6,7 +6,7 @@ use crate::encoding::CompressionLevel; use crate::encoding::Matcher; #[cfg(test)] use crate::encoding::dfast::DfastMatchGenerator; -use crate::encoding::fastpath::with_host_tier; +use crate::encoding::fastpath::{dyn_sink, with_host_tier}; #[cfg(test)] use crate::encoding::match_generator::MatchGeneratorDriver; use crate::encoding::test_support::BlockReplay; @@ -23,13 +23,13 @@ fn dfast_matches_roundtrip_multi_block_pattern() { matcher.commit_input(&first_block); let mut history = Vec::new(); let mut replay = BlockReplay::new(&first_block); - with_host_tier!(K => matcher.start_matching::(|seq| replay.apply(&mut history, seq))); + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| replay.apply(&mut history, seq)))); assert_eq!(history, first_block); matcher.commit_input(&second_block); let prefix_len = history.len(); let mut replay = BlockReplay::new(&second_block); - with_host_tier!(K => matcher.start_matching::(|seq| replay.apply(&mut history, seq))); + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| replay.apply(&mut history, seq)))); assert_eq!(&history[prefix_len..], second_block.as_slice()); } @@ -83,7 +83,7 @@ fn dfast_accepts_exact_five_byte_match() { let mut saw_five_byte_match = false; let mut saw_longer_match = false; - with_host_tier!(K => matcher.start_matching::(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if let Sequence::Triple { offset, match_len, .. } = seq @@ -94,7 +94,7 @@ fn dfast_accepts_exact_five_byte_match() { saw_longer_match = true; } } - })); + }))); assert!( saw_five_byte_match, diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index cbc8fa1f..dd3823e4 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -3,7 +3,7 @@ //! optimal round-trips, and dictionary-priming behaviour. use super::*; -use crate::encoding::fastpath::with_host_tier; +use crate::encoding::fastpath::{dyn_sink, with_host_tier}; use crate::encoding::test_support::BlockReplay; use alloc::vec::Vec; @@ -231,12 +231,12 @@ fn row_mls_knob_gates_matches_and_roundtrips() { let mut out: Vec = Vec::with_capacity(data.len()); let mut shortest_match = usize::MAX; let mut replay = BlockReplay::new(&data); - with_host_tier!(K => matcher.start_matching::(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if let Sequence::Triple { match_len, .. } = seq { shortest_match = shortest_match.min(match_len); } replay.apply(&mut out, seq); - })); + }))); assert_eq!(out, data, "mls={mls} round-trip diverged"); if shortest_match != usize::MAX { @@ -2646,13 +2646,13 @@ fn row_matches_roundtrip_multi_block_pattern() { matcher.commit_input(&first_block); let mut history = Vec::new(); let mut replay = BlockReplay::new(&first_block); - with_host_tier!(K => matcher.start_matching::(|seq| replay.apply(&mut history, seq))); + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| replay.apply(&mut history, seq)))); assert_eq!(history, first_block); matcher.commit_input(&second_block); let prefix_len = history.len(); let mut replay = BlockReplay::new(&second_block); - with_host_tier!(K => matcher.start_matching::(|seq| replay.apply(&mut history, seq))); + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| replay.apply(&mut history, seq)))); assert_eq!(&history[prefix_len..], second_block.as_slice()); @@ -2661,7 +2661,7 @@ fn row_matches_roundtrip_multi_block_pattern() { matcher.commit_input(&third_block); let third_prefix = history.len(); let mut replay = BlockReplay::new(&third_block); - with_host_tier!(K => matcher.start_matching::(|seq| replay.apply(&mut history, seq))); + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| replay.apply(&mut history, seq)))); assert_eq!(&history[third_prefix..], third_block.as_slice()); } @@ -2675,10 +2675,10 @@ fn row_short_block_emits_literals_only() { let mut saw_triple = false; let mut reconstructed = Vec::new(); let mut replay = BlockReplay::new(b"abcde"); - with_host_tier!(K => matcher.start_matching::(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { saw_triple |= matches!(seq, Sequence::Triple { .. }); replay.apply(&mut reconstructed, seq); - })); + }))); assert!( !saw_triple, @@ -2691,11 +2691,11 @@ fn row_short_block_emits_literals_only() { // searches (upstream zstd `lazy_generic` `ilimit`). saw_triple = false; matcher.commit_input(b"abcdeabcdeabcdeabcde-padding-past-ilimit"); - with_host_tier!(K => matcher.start_matching::(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if let Sequence::Triple { .. } = seq { saw_triple = true; } - })); + }))); assert!( saw_triple, "row backend should emit triples on repeated data" @@ -2742,14 +2742,14 @@ fn row_backfills_previous_block_tail_for_cross_boundary_match() { matcher.commit_input(&first_block); let mut reconstructed = Vec::new(); let mut replay = BlockReplay::new(&first_block); - with_host_tier!(K => matcher.start_matching::(|seq| replay.apply(&mut reconstructed, seq))); + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| replay.apply(&mut reconstructed, seq)))); assert_eq!(reconstructed, first_block); matcher.commit_input(&second_block); let mut saw_cross_boundary = false; let prefix_len = reconstructed.len(); let mut replay = BlockReplay::new(&second_block); - with_host_tier!(K => matcher.start_matching::(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if let Sequence::Triple { literal_len: 0, offset: 3, @@ -2760,7 +2760,7 @@ fn row_backfills_previous_block_tail_for_cross_boundary_match() { saw_cross_boundary = true; } replay.apply(&mut reconstructed, seq); - })); + }))); assert!( saw_cross_boundary, @@ -4570,7 +4570,7 @@ fn dfast_skip_matching_handles_window_eviction() { let mut reconstructed = alloc::vec![7, 8, 9, 10, 11, 12]; let mut replay = BlockReplay::new(&[7, 8, 9, 10, 11, 12]); - with_host_tier!(K => matcher.start_matching::(|seq| replay.apply(&mut reconstructed, seq))); + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| replay.apply(&mut reconstructed, seq)))); assert_eq!(reconstructed, [7, 8, 9, 10, 11, 12, 7, 8, 9, 10, 11, 12]); } @@ -4690,18 +4690,18 @@ fn dfast_inserts_tail_positions_for_next_block_matching() { matcher.commit_input(b"012345bcdea"); let mut history = Vec::new(); let mut replay = BlockReplay::new(b"012345bcdea"); - with_host_tier!(K => matcher.start_matching::(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { assert!( matches!(seq, Sequence::Literals { .. }), "first block should not match history" ); replay.apply(&mut history, seq); - })); + }))); assert_eq!(history, b"012345bcdea"); matcher.commit_input(b"bcdeabcdeab"); let mut saw_first_sequence = false; - with_host_tier!(K => matcher.start_matching::(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { assert!(!saw_first_sequence, "expected a single cross-block match"); saw_first_sequence = true; match seq { @@ -4723,7 +4723,7 @@ fn dfast_inserts_tail_positions_for_next_block_matching() { } } } - })); + }))); assert!( saw_first_sequence, @@ -4836,7 +4836,7 @@ fn dfast_dense_skip_matching_backfills_previous_tail_for_next_block() { matcher.commit_input(&second); let mut first_sequence = None; - with_host_tier!(K => matcher.start_matching::(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if first_sequence.is_some() { return; } @@ -4848,7 +4848,7 @@ fn dfast_dense_skip_matching_backfills_previous_tail_for_next_block() { match_len, } => (literal_len, offset, match_len), }); - })); + }))); let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); assert_eq!( @@ -4882,7 +4882,7 @@ fn dfast_sparse_skip_matching_preserves_tail_cross_block_match() { matcher.commit_input(&second); let mut first_sequence = None; - with_host_tier!(K => matcher.start_matching::(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if first_sequence.is_some() { return; } @@ -4894,7 +4894,7 @@ fn dfast_sparse_skip_matching_preserves_tail_cross_block_match() { match_len, } => (literal_len, offset, match_len), }); - })); + }))); let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); assert_eq!( @@ -5107,7 +5107,7 @@ fn dfast_sparse_skip_matching_backfills_previous_tail_for_consecutive_sparse_blo matcher.commit_input(&third); let mut first_sequence = None; - with_host_tier!(K => matcher.start_matching::(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if first_sequence.is_some() { return; } @@ -5119,7 +5119,7 @@ fn dfast_sparse_skip_matching_backfills_previous_tail_for_consecutive_sparse_blo match_len, } => (literal_len, offset, match_len), }); - })); + }))); let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); assert_eq!( From 6e8e9f5739564b2b5c5c84b8e3d8b9434f97d5c2 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sun, 25 Oct 2026 02:40:17 +0300 Subject: [PATCH 12/15] build(encode): compile the built-in search once per tier - A closure's type carries every generic parameter of the function it is written in. The block's record closure was written in a function generic over the matcher type as well as the tier, so the built-in driver's whole search was compiled again for each matcher type that wraps it. The built-in branch now runs in a function generic over the tier alone. - The tier-identity tests that call a search directly pass their sink through `dyn_sink`, and the lint gate covers every `start_matching*` entry in test files. --- .github/workflows/ci.yml | 4 +++- zstd/src/encoding/blocks/compressed.rs | 26 +++++++++++++++++++--- zstd/src/encoding/match_generator/tests.rs | 8 +++---- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45580923..0716b83d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,7 +225,9 @@ jobs: -e 'match\s+[A-Za-z_]+::TIER\b' \ -e 'match\s+<[^>]+>::TIER\b' \ -e '::TIER\s*[=!]=' \ - -e 'start_matching(_borrowed)?::\(\s*\|' \ + zstd/src/ || true) + found+=$(rg -n --no-heading -g '*tests*.rs' \ + -e 'start_matching\w*::\(\s*\|' \ zstd/src/ || true) if [ -n "$found" ]; then echo "$found" diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index 15ec90b1..7a4d76af 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -683,13 +683,16 @@ fn collect_block_parts( let code_offsets = uses_fast_offset_codes(state.strategy_tag); let block_start_hist = state.offset_hist; let mut hist = state.offset_hist; - let record = |seq| record_sequence(seq, &mut tail, sequences, code_offsets, &mut hist); // The built-in matcher searches compiled for the frame's tier; a matcher // from outside the crate knows nothing of tiers. The branch folds per // matcher type. match state.matcher.builtin_driver() { - Some(driver) => driver.start_matching_with::(record), - None => state.matcher.start_matching(record), + Some(driver) => { + builtin_block_sequences::(driver, &mut tail, sequences, code_offsets, &mut hist) + } + None => state.matcher.start_matching(|seq| { + record_sequence(seq, &mut tail, sequences, code_offsets, &mut hist) + }), } if code_offsets { state.offset_hist = hist; @@ -715,6 +718,23 @@ fn collect_block_parts( ); } +/// The built-in matcher's search of the block under tier `K`, recorded as in +/// [`collect_block_parts`]. Generic over the tier alone: the record closure's +/// type carries every parameter of the function it is written in, so written +/// in `collect_block_parts` it would compile the whole search again for each +/// matcher type that wraps the driver. +#[inline] +fn builtin_block_sequences( + driver: &mut crate::encoding::match_generator::MatchGeneratorDriver, + tail: &mut usize, + sequences: &mut RegionVec, + code_offsets: bool, + hist: &mut [u32; 3], +) { + driver + .start_matching_with::(|seq| record_sequence(seq, tail, sequences, code_offsets, hist)); +} + /// Append a block's parse to `capture` as external sequences with plain /// offsets and their repeat-offset numbers, against the `history` the block /// started from, closed by a delimiter carrying the `tail` literals. With diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index dd3823e4..9ae2449d 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -779,14 +779,14 @@ fn bt_optimal_all_kernel_tiers_emit_identical_sequences() { let mut seqs = Vec::new(); crate::encoding::fastpath::with_encode_tier!( tier, - K => hc.start_matching_with::(|seq| match seq { + K => hc.start_matching_with::(dyn_sink(&mut |seq| match seq { Sequence::Triple { literal_len, offset, match_len, } => seqs.push((literal_len, offset, match_len)), Sequence::Literals { len } => seqs.push((len, 0, 0)), - }) + })) ); seqs }; @@ -880,14 +880,14 @@ fn dfast_dictionary_all_kernel_tiers_emit_identical_sequences() { "level 3 with a dictionary must run the dfast backend", ); driver.commit_input(&block); - driver.start_matching_with::(|seq| match seq { + driver.start_matching_with::(dyn_sink(&mut |seq| match seq { Sequence::Triple { literal_len, offset, match_len, } => seqs.push((literal_len, offset, match_len)), Sequence::Literals { len } => seqs.push((len, 0, 0)), - }); + })); }); seqs }; From 14b3509188e9b259ace4ce77cf38155a4c86903b Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sun, 25 Oct 2026 11:07:02 +0200 Subject: [PATCH 13/15] test(encode): check every kernel tier end to end A tier is chosen once and the code below it is compiled per tier, so on any one machine the detection runs a single tier and the scalar fallback (what a CPU without the extended instructions runs) is never exercised. - Frames and streams are written under every tier the CPU can run and compared byte for byte: the compressor's reader, one-shot and known-size entry points, the streaming write, flush, finish and single-block calls, and a matcher reaching the built-in driver only through the trait, with and without a dictionary, at a level of each backend. Every frame is decoded back. - The row probe test helpers and the dictionary trainer run under every tier and require one answer; the trainer takes its tier as a parameter for that. - Tests for the binary-tree rebase (the replay rebuilds the walked tree; a walk past the index range rebases and matches a plain walk), a burst flushing a full accumulator, and a skip with no hint. - The hash3 candidate dispatcher, its per-kernel variants and its body macro had no caller and are removed. - The streaming encoder's last-block pre-split loop could not run: a write emits a block the moment it fills. It is replaced by an assertion, and the test of that cut now builds a stream where the pre-splitter does cut, which fails if it does not. - The wide-extra sequences in the block tier test use lengths with 16 extra bits, as their comment states. --- zstd/src/bit_io/bit_writer/tests.rs | 22 +++ zstd/src/dictionary/legacy.rs | 23 ++- zstd/src/dictionary/legacy/tests.rs | 12 ++ zstd/src/encoding/blocks/compressed/tests.rs | 6 +- zstd/src/encoding/bt/mod.rs | 4 +- zstd/src/encoding/fastpath/mod.rs | 61 ++++++ zstd/src/encoding/frame_compressor/tests.rs | 92 +++++++++ zstd/src/encoding/hc/generator.rs | 59 ------ zstd/src/encoding/match_generator/mod.rs | 4 - zstd/src/encoding/match_generator/tests.rs | 114 ++++------- zstd/src/encoding/match_table/storage.rs | 186 +----------------- .../match_table/storage/storage_tests.rs | 67 +++++++ zstd/src/encoding/opt/types.rs | 1 + zstd/src/encoding/row/mod.rs | 34 ++-- zstd/src/encoding/streaming_encoder.rs | 30 +-- zstd/src/encoding/streaming_encoder/tests.rs | 127 +++++++++++- zstd/src/encoding/test_support.rs | 119 ++++++++++- 17 files changed, 593 insertions(+), 368 deletions(-) diff --git a/zstd/src/bit_io/bit_writer/tests.rs b/zstd/src/bit_io/bit_writer/tests.rs index 5f1faeb5..1472cc73 100644 --- a/zstd/src/bit_io/bit_writer/tests.rs +++ b/zstd/src/bit_io/bit_writer/tests.rs @@ -160,6 +160,28 @@ fn add_multiple_aligned() { assert_eq!(vec![0xFF, 0xF0, 0x0F, 0x00], bw.dump()); } +/// A burst flushing a full accumulator writes all eight bytes and holds +/// nothing after, so the next bits land on a clean byte rather than on bits +/// of the flushed value. +#[test] +fn a_burst_flushes_a_full_accumulator_clean() { + let mut bw = BitWriter::new(); + let mut burst = bw.burst(16); + // SAFETY: 64 bits into an empty accumulator, then 8 into an empty one, + // within the 16 bytes reserved. + unsafe { + burst.add(0x0123_4567_89AB_CDEF, 64); + burst.flush(); + burst.add(0xA5, 8); + burst.flush(); + } + burst.finish(); + assert_eq!( + bw.dump(), + vec![0xEF, 0xCD, 0xAB, 0x89, 0x67, 0x45, 0x23, 0x01, 0xA5] + ); +} + // #[test] // fn catches_more_than_in_buf() { // todo!(); diff --git a/zstd/src/dictionary/legacy.rs b/zstd/src/dictionary/legacy.rs index f3b6b917..30fc0f69 100644 --- a/zstd/src/dictionary/legacy.rs +++ b/zstd/src/dictionary/legacy.rs @@ -271,6 +271,24 @@ pub(crate) fn train_legacy_raw( sample_sizes: &[usize], dict_size: usize, selectivity: u32, +) -> Result, TooSmall> { + train_legacy_raw_under( + select_kernel(), + samples, + sample_sizes, + dict_size, + selectivity, + ) +} + +/// [`train_legacy_raw`] comparing through the tier `tier`, the run's one +/// choice of compare kernel. +fn train_legacy_raw_under( + tier: crate::encoding::fastpath::FastpathKernel, + samples: &[u8], + sample_sizes: &[usize], + dict_size: usize, + selectivity: u32, ) -> Result, TooSmall> { let total: usize = sample_sizes.iter().sum(); debug_assert_eq!(total, samples.len(), "the sizes describe the corpus"); @@ -310,10 +328,9 @@ pub(crate) fn train_legacy_raw( kept -= 1; used -= sample_sizes[kept]; } - // The run's one choice of compare kernel: the analysis below is compiled - // for it. + // The analysis below is compiled for the run's tier. crate::encoding::fastpath::with_encode_tier!( - select_kernel(), + tier, K => select_content::(&mut list, &samples[..used], min_rep, dict_size) ) } diff --git a/zstd/src/dictionary/legacy/tests.rs b/zstd/src/dictionary/legacy/tests.rs index 8de709a1..78c91a4a 100644 --- a/zstd/src/dictionary/legacy/tests.rs +++ b/zstd/src/dictionary/legacy/tests.rs @@ -85,6 +85,18 @@ fn the_content_is_made_of_repeated_corpus_runs() { assert!(!text.contains(&format!("ts={:08}", 3 * 7919))); } +/// Every tier the CPU can run trains the same content: the run compares +/// through one tier, and the scalar fallback must find what the SIMD compares +/// find. +#[test] +fn every_tier_trains_the_same_content() { + let (samples, sizes) = log_samples(400); + let content = crate::encoding::fastpath::same_under_every_tier(|tier| { + train_legacy_raw_under(tier, &samples, &sizes, 4096, 0) + }); + assert!(content.is_ok_and(|content| !content.is_empty())); +} + /// A size below the reference's minimum, a corpus under 512 bytes, and a corpus /// with nothing repeated are refused, as `ZDICT_trainFromBuffer_legacy` refuses /// them. diff --git a/zstd/src/encoding/blocks/compressed/tests.rs b/zstd/src/encoding/blocks/compressed/tests.rs index f6e742c0..aa83bb16 100644 --- a/zstd/src/encoding/blocks/compressed/tests.rs +++ b/zstd/src/encoding/blocks/compressed/tests.rs @@ -1352,8 +1352,10 @@ fn every_tier_writes_the_block_the_scalar_tier_writes() { // 56 extra bits takes the one before the extras (31 or more) alone, and // 16 + 16 + 30 = 62 takes the one before the offset (more than 56) too. for (k, seq) in sequences.iter_mut().step_by(97).enumerate() { - seq.ll = 40_000 + k as u32; - seq.ml = 60_000 + k as u32; + // The first values with 16 extra bits: literal length 65536 and match + // length 65539 (`ml` counts the 3 the match-length code leaves out). + seq.ll = 65_536 + k as u32; + seq.ml = 65_539 + k as u32; seq.off_base = if k % 2 == 0 { (1u32 << 24) + k as u32 } else { diff --git a/zstd/src/encoding/bt/mod.rs b/zstd/src/encoding/bt/mod.rs index 6774f628..8250fea0 100644 --- a/zstd/src/encoding/bt/mod.rs +++ b/zstd/src/encoding/bt/mod.rs @@ -27,9 +27,7 @@ use super::opt::types::{HcOptimalNode, HcOptimalPlanBuffers, HcOptimalSequence, /// Maximum offset reachable by the HC3 short-match probe. Upstream zstd /// parity: keeps the 3-byte side table from emitting offsets that -/// the main BT/HC paths would address more efficiently. Used inside -/// the `hash3_candidate_body!` macro that the kernel-specific -/// variants below expand. +/// the main BT/HC paths would address more efficiently. pub(crate) const HC3_MAX_OFFSET: usize = 1 << 18; /// Binary-tree matcher state used by the `BtOpt` / `BtUltra` / diff --git a/zstd/src/encoding/fastpath/mod.rs b/zstd/src/encoding/fastpath/mod.rs index c6ce39fd..a42bc25b 100644 --- a/zstd/src/encoding/fastpath/mod.rs +++ b/zstd/src/encoding/fastpath/mod.rs @@ -500,6 +500,67 @@ macro_rules! with_host_tier { #[cfg(test)] pub(crate) use with_host_tier; +/// Every tier this CPU may execute, scalar first: each SIMD arm assumes its +/// target feature is present, so a test forcing a tier takes it from here. A +/// test comparing tiers checks the scalar fallback against every SIMD kernel +/// the machine has, including the ones the detection leaves cold. +#[cfg(test)] +pub(crate) fn runnable_tiers() -> alloc::vec::Vec { + #[allow(unused_mut)] + let mut tiers = alloc::vec![FastpathKernel::Scalar]; + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-sse" + ))] + { + if std::is_x86_feature_detected!("sse2") { + tiers.push(FastpathKernel::Sse2); + } + if std::is_x86_feature_detected!("sse4.2") { + tiers.push(FastpathKernel::Sse42); + } + } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + feature = "kernel-avx2" + ))] + if std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("bmi2") { + tiers.push(FastpathKernel::Avx2Bmi2); + } + #[cfg(all( + target_arch = "aarch64", + target_endian = "little", + feature = "kernel-neon" + ))] + tiers.push(FastpathKernel::Neon); + #[cfg(all( + target_arch = "wasm32", + target_feature = "simd128", + feature = "kernel-simd128" + ))] + tiers.push(FastpathKernel::Simd128); + tiers +} + +/// `run` under every [`runnable_tiers`] tier, asserting they all agree, and +/// the answer they agree on. +#[cfg(test)] +pub(crate) fn same_under_every_tier( + run: impl Fn(FastpathKernel) -> R, +) -> R { + let tiers = runnable_tiers(); + let answer = run(tiers[0]); + for &tier in &tiers[1..] { + assert_eq!( + run(tier), + answer, + "tier {tier:?} disagrees with {:?}", + tiers[0] + ); + } + answer +} + /// A test's sequence closure as the one sink type tests hand a backend, so /// each search is compiled once per tier for all of them rather than once per /// closure and tier. diff --git a/zstd/src/encoding/frame_compressor/tests.rs b/zstd/src/encoding/frame_compressor/tests.rs index c464af08..9205c59a 100644 --- a/zstd/src/encoding/frame_compressor/tests.rs +++ b/zstd/src/encoding/frame_compressor/tests.rs @@ -4800,3 +4800,95 @@ fn estimated_heap_size_bounds_every_prepared_form() { ); } } + +/// The frames one compressor writes for `input` under `tier` through each of +/// its entry points, and through a matcher reaching the built-in driver only +/// by the trait. +fn frames_under_tier( + tier: crate::encoding::fastpath::FastpathKernel, + level: super::CompressionLevel, + input: &[u8], + dict: bool, +) -> Vec> { + use crate::encoding::test_support::{TraitDriver, tier_dictionary}; + let builtin = || { + let mut compressor: FrameCompressor<&[u8], &mut Vec> = FrameCompressor::new(level); + compressor.state.copy_kernel = tier; + if dict { + compressor.set_dictionary(tier_dictionary()).unwrap(); + } + compressor + }; + let mut frames = Vec::new(); + + let mut read = Vec::new(); + let mut compressor = builtin(); + compressor.set_source(input); + compressor.set_drain(&mut read); + compressor.compress(); + drop(compressor); + frames.push(read); + + let mut one_shot = Vec::new(); + builtin().compress_independent_frame_into(input, &mut one_shot); + frames.push(one_shot); + + #[cfg(feature = "dict-builder")] + { + let mut known = Vec::new(); + let mut compressor = builtin(); + compressor.set_source(input); + compressor.compress_known_into(input.len() as u64, &mut known); + frames.push(known); + } + + let mut through_trait = Vec::new(); + let mut compressor = FrameCompressor::new_with_matcher(TraitDriver::new(tier), level); + compressor.state.copy_kernel = tier; + if dict { + compressor.set_dictionary(tier_dictionary()).unwrap(); + } + compressor.set_source(input); + compressor.set_drain(&mut through_trait); + compressor.compress(); + drop(compressor); + frames.push(through_trait); + frames +} + +/// Every tier the CPU can run writes the same frames, with and without a +/// dictionary, at a level of each backend, and every frame decodes to its +/// input. The scalar tier is what a CPU without the extended instructions +/// runs, and the detection never picks it on one that has them, so this is +/// where the fallback is checked end to end against the SIMD kernels. +#[test] +fn every_tier_writes_the_same_frames() { + use crate::encoding::test_support::{TIER_LEVELS, tier_dictionary, tier_fixture}; + let input = tier_fixture(); + let tiers = crate::encoding::fastpath::runnable_tiers(); + for level in TIER_LEVELS { + let level = super::CompressionLevel::Level(level); + for dict in [false, true] { + let reference = frames_under_tier(tiers[0], level, &input, dict); + for frame in &reference { + let mut decoder = FrameDecoder::new(); + if dict { + decoder.add_dict(tier_dictionary()).unwrap(); + } + let mut decoded = Vec::with_capacity(input.len()); + decoder.decode_all_to_vec(frame, &mut decoded).unwrap(); + assert!( + decoded == input, + "{level:?} dict {dict}: a frame does not decode to its input" + ); + } + for &tier in &tiers[1..] { + assert!( + frames_under_tier(tier, level, &input, dict) == reference, + "{level:?} dict {dict}: tier {tier:?} wrote other frames than {:?}", + tiers[0], + ); + } + } + } +} diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs index 2474356c..c09f661d 100644 --- a/zstd/src/encoding/hc/generator.rs +++ b/zstd/src/encoding/hc/generator.rs @@ -345,65 +345,6 @@ macro_rules! bt_insert_range_body { } pub(crate) use bt_insert_range_body; -/// `hash3_candidate` body parameterized over the per-CPU -/// `common_prefix_len_ptr` symbol. The hash3 probe checks one candidate per -/// position when invoked, so the per-call ABI savings compound across the -/// segment. Crate-private (see `bt_insert_range_body!`). -macro_rules! hash3_candidate_body { - ( - $table:expr, - $abs_pos:ident, - $current_abs_end:ident, - $min_match_len:ident, - $cpl:path $(,)? - ) => {{ - if $table.hash3_log == 0 { - return None; - } - let idx = $abs_pos.checked_sub($table.history_abs_start)?; - let concat = $table.live_history(); - if idx + 4 > concat.len() { - return None; - } - let hash3 = $crate::encoding::match_table::storage::MatchTable::hash_position_at( - concat, - idx, - $table.hash3_log, - 3, - ); - let entry = $table - .hash3_table() - .get(hash3) - .copied() - .unwrap_or($crate::encoding::match_table::storage::HC_EMPTY); - let candidate_abs = - $crate::encoding::match_table::storage::MatchTable::stored_abs_position_fast( - entry, - $table.position_base, - $table.index_shift, - )?; - if candidate_abs < $table.history_abs_start || candidate_abs >= $abs_pos { - return None; - } - let offset = $abs_pos - candidate_abs; - if offset >= $crate::encoding::bt::HC3_MAX_OFFSET { - return None; - } - let candidate_idx = candidate_abs - $table.history_abs_start; - let tail_limit = $current_abs_end.saturating_sub($abs_pos); - let base = concat.as_ptr(); - // SAFETY: candidate/idx are within history range; tail_limit - // bounds the scan within `concat`. - let match_len = unsafe { $cpl(base.add(candidate_idx), base.add(idx), tail_limit) }; - (match_len >= $min_match_len).then_some($crate::encoding::opt::types::MatchCandidate { - start: $abs_pos, - offset, - match_len, - }) - }}; -} -pub(crate) use hash3_candidate_body; - /// `for_each_repcode_candidate_with_reps` body parameterized over the per-CPU /// `common_prefix_len_ptr` symbol so the per-rep prefix probe inlines under /// the wrapper's `target_feature` umbrella instead of crossing the ABI diff --git a/zstd/src/encoding/match_generator/mod.rs b/zstd/src/encoding/match_generator/mod.rs index de296c74..722814d2 100644 --- a/zstd/src/encoding/match_generator/mod.rs +++ b/zstd/src/encoding/match_generator/mod.rs @@ -139,10 +139,6 @@ use super::match_table::storage::{HC_PRIME3BYTES, HC_PRIME4BYTES}; // macros / configs / tests keep their unqualified names. #[cfg(test)] use super::match_table::storage::HC_EMPTY; -// HC3_MAX_OFFSET moved to encoding::bt alongside the hash3 candidate -// probe macro that consumes it; the macro references it via the -// fully-qualified `$crate::encoding::bt::HC3_MAX_OFFSET` path so this -// module no longer needs a local import. pub(crate) const HC_SEARCH_DEPTH: usize = 16; // HC_MIN_MATCH_LEN moved to encoding::hc; re-imported here so // existing references compile unchanged. diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index 9ae2449d..1110da1f 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -57,6 +57,12 @@ impl MatchGeneratorDriver { ); self.reset(level); } + + /// Test-only: the tier the [`Matcher`] entry points run under, in place + /// of the one detected at construction. + pub(crate) fn set_tier(&mut self, tier: crate::encoding::fastpath::FastpathKernel) { + self.tier = tier; + } } /// Drive a full compress parse for `data` at `level` (optionally with a @@ -733,34 +739,12 @@ fn btultra2_seed_pass_initializes_opt_state() { /// the search under each tier the CPU supports executes each monomorphized /// BT-collect / DP wrapper on one machine (only the detected tier would /// otherwise run), which pins the scalar-vs-SIMD bit-identity invariant. -/// x86-only: aarch64 has a single NEON tier. -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] #[test] fn bt_optimal_all_kernel_tiers_emit_identical_sequences() { use crate::encoding::Sequence; use crate::encoding::fastpath::FastpathKernel; - // Tiers the running CPU can legally execute: each dispatch arm is `unsafe` - // and assumes the tier's target_feature is present. Scalar is always safe; - // the SIMD tiers gate on runtime detection so the test stays valid on any - // x86 box (including a CI runner without AVX2). - // Seeded with the always-present scalar tier; the SIMD entries below are - // feature-gated, so `mut` goes unused in a build with every kernel - // compiled out. - #[allow(unused_mut)] - let mut tiers = alloc::vec![FastpathKernel::Scalar]; - #[cfg(feature = "kernel-sse")] - if std::is_x86_feature_detected!("sse2") { - tiers.push(FastpathKernel::Sse2); - } - #[cfg(feature = "kernel-sse")] - if std::is_x86_feature_detected!("sse4.2") { - tiers.push(FastpathKernel::Sse42); - } - #[cfg(feature = "kernel-avx2")] - if std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("bmi2") { - tiers.push(FastpathKernel::Avx2Bmi2); - } + let tiers = crate::encoding::fastpath::runnable_tiers(); // Mixed-redundancy input so the BT walk, the rep-code probe, and the hash3 // short-match probe all fire (a pure ramp would never exercise the rep path). @@ -822,36 +806,7 @@ fn bt_optimal_all_kernel_tiers_emit_identical_sequences() { fn dfast_dictionary_all_kernel_tiers_emit_identical_sequences() { use crate::encoding::fastpath::FastpathKernel; - // Only tiers the running CPU may legally execute: each dispatch arm is - // `unsafe` and assumes its target feature is present. - #[allow(unused_mut)] - let mut tiers = alloc::vec![FastpathKernel::Scalar]; - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" - ))] - if std::is_x86_feature_detected!("sse2") { - tiers.push(FastpathKernel::Sse2); - } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-avx2" - ))] - if std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("bmi2") { - tiers.push(FastpathKernel::Avx2Bmi2); - } - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - tiers.push(FastpathKernel::Neon); - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ))] - tiers.push(FastpathKernel::Simd128); + let tiers = crate::encoding::fastpath::runnable_tiers(); let dict: Vec = (0..20 * 1024u32) .map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8) @@ -1557,30 +1512,13 @@ fn hc_collect_optimal_candidates_dispatches_every_bt_strategy() { /// CPU can run finds the same candidates at every position of a block: the /// scalar fallback and the SIMD kernels agree on the search itself, not only on /// the sequences the parser builds from it. Forcing the tier runs the arms the -/// runtime detection leaves cold on this machine. x86 only, where the tier is a -/// runtime choice. -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +/// runtime detection leaves cold on this machine. #[test] fn hc_collect_optimal_candidates_agree_across_kernel_tiers() { use crate::encoding::fastpath::FastpathKernel; use crate::encoding::strategy::StrategyTag; - // Only tiers the running CPU may execute: each dispatch arm assumes its - // target feature is present. - #[allow(unused_mut)] - let mut tiers = alloc::vec![FastpathKernel::Scalar]; - #[cfg(feature = "kernel-sse")] - if std::is_x86_feature_detected!("sse2") { - tiers.push(FastpathKernel::Sse2); - } - #[cfg(feature = "kernel-sse")] - if std::is_x86_feature_detected!("sse4.2") { - tiers.push(FastpathKernel::Sse42); - } - #[cfg(feature = "kernel-avx2")] - if std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("bmi2") { - tiers.push(FastpathKernel::Avx2Bmi2); - } + let tiers = crate::encoding::fastpath::runnable_tiers(); // Mixed redundancy, so the repeat, hash3 and tree probes all find matches. let data: Vec = (0..4096usize) @@ -2415,6 +2353,38 @@ fn driver_rows_frame_releases_the_tree_buffer_capacity() { ); } +/// A block skipped through the plain [`Matcher::skip_matching`] is indexed as +/// one skipped with no hint: the next block finds the same matches into it. +#[test] +fn a_plain_skip_indexes_the_block_as_a_skip_without_a_hint() { + let first = b"tenant=demo table=orders region=eu-west payload=".repeat(20); + let run = |plain: bool| { + let mut driver = MatchGeneratorDriver::new(1 << 17, 1); + driver.set_source_size_hint(4 * first.len() as u64); + driver.reset(CompressionLevel::Level(3)); + driver.commit_input(&first); + if plain { + driver.skip_matching(); + } else { + driver.skip_matching_with_hint(None); + } + driver.commit_input(&first); + let mut seqs = Vec::new(); + driver.start_matching(|seq| seqs.push(seq)); + seqs + }; + let reference = run(false); + // From its first byte the block can only match into the skipped one. + assert!( + matches!( + reference.first(), + Some(Sequence::Triple { literal_len: 0, .. }) + ), + "fixture precondition: the second block must match into the skipped one", + ); + assert_eq!(run(true), reference); +} + #[test] fn driver_rows_reset_releases_the_tree_tables() { let mut driver = MatchGeneratorDriver::new(32, 2); diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index f472e138..242bd6b8 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -24,7 +24,7 @@ use super::super::cost_model::HC_OPT_NUM; use super::super::dict_attach::DictAttach; use super::super::fastpath::EncodeTier; use super::super::hc::HC_MIN_MATCH_LEN; -use super::super::opt::types::{HcOptimalSequence, MatchCandidate}; +use super::super::opt::types::HcOptimalSequence; use super::super::workspace::{HistoryBuf, Table, Workspace, region_bytes}; use super::helpers::INCOMPRESSIBLE_SKIP_STEP; @@ -2523,190 +2523,6 @@ impl MatchTable { } } - /// Stage D: HC3 short-match probe. Cross-platform dispatcher. - /// External / test callers only — the on-encode hot path bypasses - /// this via the per-kernel variant from inside the surrounding - /// `collect_optimal_candidates_initialized_` umbrella. - #[allow(dead_code)] - #[inline(always)] - pub(crate) fn hash3_candidate( - &self, - abs_pos: usize, - current_abs_end: usize, - min_match_len: usize, - ) -> Option { - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - unsafe { - self.hash3_candidate_neon(abs_pos, current_abs_end, min_match_len) - } - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - use crate::encoding::fastpath::FastpathKernel; - crate::encoding::fastpath::tier_match!(K { - #[cfg(feature = "kernel-avx2")] - FastpathKernel::Avx2Bmi2 => unsafe { - self.hash3_candidate_avx2_bmi2(abs_pos, current_abs_end, min_match_len) - }, - #[cfg(feature = "kernel-sse")] - FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { - self.hash3_candidate_sse2(abs_pos, current_abs_end, min_match_len) - }, - FastpathKernel::Scalar => { - self.hash3_candidate_scalar(abs_pos, current_abs_end, min_match_len) - }, - }) - } - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ))] - unsafe { - self.hash3_candidate_simd128(abs_pos, current_abs_end, min_match_len) - } - #[cfg(not(any( - all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ), - all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ), - target_arch = "x86", - target_arch = "x86_64" - )))] - { - self.hash3_candidate_scalar(abs_pos, current_abs_end, min_match_len) - } - } - - /// WebAssembly `simd128` umbrella HC3 probe. - /// - /// # Safety - /// wasm32 with `simd128` enabled at compile time. Body inlines via macro. - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ))] - #[target_feature(enable = "simd128")] - pub(crate) unsafe fn hash3_candidate_simd128( - &self, - abs_pos: usize, - current_abs_end: usize, - min_match_len: usize, - ) -> Option { - super::super::hc::generator::hash3_candidate_body!( - self, - abs_pos, - current_abs_end, - min_match_len, - crate::encoding::fastpath::simd128::common_prefix_len_ptr, - ) - } - - /// NEON umbrella HC3 probe. - /// - /// # Safety - /// AArch64 with NEON (baseline). Body inlines via macro. - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - #[target_feature(enable = "neon")] - pub(crate) unsafe fn hash3_candidate_neon( - &self, - abs_pos: usize, - current_abs_end: usize, - min_match_len: usize, - ) -> Option { - super::super::hc::generator::hash3_candidate_body!( - self, - abs_pos, - current_abs_end, - min_match_len, - crate::encoding::fastpath::neon::common_prefix_len_ptr, - ) - } - - /// SSE2 umbrella HC3 probe. - /// - /// # Safety - /// x86/x86_64 with SSE2. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" - ))] - #[target_feature(enable = "sse2")] - pub(crate) unsafe fn hash3_candidate_sse2( - &self, - abs_pos: usize, - current_abs_end: usize, - min_match_len: usize, - ) -> Option { - super::super::hc::generator::hash3_candidate_body!( - self, - abs_pos, - current_abs_end, - min_match_len, - crate::encoding::fastpath::sse2::common_prefix_len_ptr, - ) - } - - /// AVX2+BMI2 umbrella HC3 probe. - /// - /// # Safety - /// x86/x86_64 with AVX2 + BMI2. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-avx2" - ))] - #[target_feature(enable = "avx2,bmi2")] - pub(crate) unsafe fn hash3_candidate_avx2_bmi2( - &self, - abs_pos: usize, - current_abs_end: usize, - min_match_len: usize, - ) -> Option { - super::super::hc::generator::hash3_candidate_body!( - self, - abs_pos, - current_abs_end, - min_match_len, - crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr, - ) - } - - /// Scalar fallback HC3 probe, compiled unless the NEON tier covers this - /// target. - #[cfg(not(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - )))] - pub(crate) fn hash3_candidate_scalar( - &self, - abs_pos: usize, - current_abs_end: usize, - min_match_len: usize, - ) -> Option { - super::super::hc::generator::hash3_candidate_body!( - self, - abs_pos, - current_abs_end, - min_match_len, - crate::encoding::fastpath::scalar::common_prefix_len_ptr, - ) - } - /// Reset the rebase-derived bookkeeping (rolling `position_base` / /// `index_shift`) so every stored position re-encodes from /// `history_abs_start`, then clear the three index tables. Hot diff --git a/zstd/src/encoding/match_table/storage/storage_tests.rs b/zstd/src/encoding/match_table/storage/storage_tests.rs index 8324e97c..8117211d 100644 --- a/zstd/src/encoding/match_table/storage/storage_tests.rs +++ b/zstd/src/encoding/match_table/storage/storage_tests.rs @@ -333,6 +333,73 @@ fn rebase_positions_cold_rebuilds_hash3_for_btultra2() { ); } +/// A binary-tree table over 64 bytes of period-17 input, every position +/// representable from the origin. +fn bt_table() -> MatchTable { + let mut t = new_table(64); + t.history = (0..64u8) + .map(|i| i % 17) + .collect::>() + .into(); + t.history_start = 0; + t.history_abs_start = 0; + t.window_size = 64; + t.position_base = 0; + t.search_depth = 4; + t.uses_bt = true; + t.ensure_tables(); + t +} + +/// The tree the rebase replays is the tree the walk had built: the replay +/// runs on the scalar compare whatever tier built the original, so the two +/// agree only if every tier reports the same lengths, and a rebase that lost +/// or added a node would change what the optimal parser sees afterwards. +#[test] +fn rebase_positions_cold_rebuilds_the_binary_tree_the_walk_built() { + let mut t = bt_table(); + crate::encoding::fastpath::with_host_tier!(K => t.bt_update_tree_until::(32, 64)); + let walked = (t.hash_table().to_vec(), t.chain_table().to_vec()); + assert!( + walked.0.iter().any(|&v| v != HC_EMPTY), + "fixture precondition: the walk must populate the tree", + ); + + t.rebase_positions_cold(32); + + assert_eq!(t.position_base, t.history_abs_start); + assert_eq!( + (t.hash_table().to_vec(), t.chain_table().to_vec()), + walked, + "the replayed tree differs from the walked one", + ); +} + +/// A tree walk whose target is past the index range goes one position at a +/// time, rebasing before the first insert, and ends with the tree a walk from +/// a representable start builds. +#[test] +fn a_tree_walk_past_the_index_range_rebases_then_matches_a_plain_walk() { + let mut plain = bt_table(); + crate::encoding::fastpath::with_host_tier!(K => plain.bt_update_tree_until::(40, 64)); + + let mut t = bt_table(); + t.index_shift = u32::MAX as usize - 10; + assert!( + !t.can_skip_rebase_check(40), + "fixture precondition: the target must not be representable", + ); + crate::encoding::fastpath::with_host_tier!(K => t.bt_update_tree_until::(40, 64)); + + assert_eq!(t.index_shift, 0, "the positions were re-encoded"); + assert_eq!(t.skip_insert_until_abs, 40); + assert_eq!( + (t.hash_table(), t.chain_table()), + (plain.hash_table(), plain.chain_table()), + "the position-at-a-time walk built a different tree", + ); +} + #[test] fn insert_positions_with_step_zero_step_is_noop() { let mut t = new_table(32); diff --git a/zstd/src/encoding/opt/types.rs b/zstd/src/encoding/opt/types.rs index 243bf38e..a8361e14 100644 --- a/zstd/src/encoding/opt/types.rs +++ b/zstd/src/encoding/opt/types.rs @@ -18,6 +18,7 @@ use super::super::cost_model::HcOptimalCostProfile; /// One candidate match produced by a match-finder probe. Carries the /// absolute starting position and the upstream zstd-style offset/length pair. #[derive(Copy, Clone, Debug)] +#[cfg_attr(test, derive(PartialEq))] pub(crate) struct MatchCandidate { pub(crate) start: usize, pub(crate) offset: usize, diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 1284adb4..d686dcc1 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -4192,18 +4192,24 @@ impl RowMatchGenerator { self.skip_matching_with_hint(incompressible_hint); } + /// The best match at `abs_pos`, which every tier the CPU runs must find + /// alike. #[cfg(test)] pub(crate) fn best_match(&self, abs_pos: usize, lit_len: usize) -> Option { - crate::encoding::fastpath::with_encode_tier!( - crate::encoding::fastpath::select_kernel(), - T => dispatch_tag_kernel!(T, self.best_match_k(abs_pos, lit_len)) - ) + crate::encoding::fastpath::same_under_every_tier(|tier| { + crate::encoding::fastpath::with_encode_tier!( + tier, + T => dispatch_tag_kernel!(T, self.best_match_k(abs_pos, lit_len)) + ) + }) } #[cfg(test)] fn best_match_k(&self, abs_pos: usize, lit_len: usize) -> Option { dispatch_row_log!(self.best_match_rl::(abs_pos, lit_len)) } + /// The lazy pick at `abs_pos`, which every tier the CPU runs must make + /// alike. #[cfg(test)] pub(crate) fn pick_lazy_match( &self, @@ -4211,10 +4217,12 @@ impl RowMatchGenerator { lit_len: usize, best: Option, ) -> Option { - crate::encoding::fastpath::with_encode_tier!( - crate::encoding::fastpath::select_kernel(), - T => dispatch_tag_kernel!(T, self.pick_lazy_match_k(abs_pos, lit_len, best)) - ) + crate::encoding::fastpath::same_under_every_tier(|tier| { + crate::encoding::fastpath::with_encode_tier!( + tier, + T => dispatch_tag_kernel!(T, self.pick_lazy_match_k(abs_pos, lit_len, best)) + ) + }) } #[cfg(test)] fn pick_lazy_match_k( @@ -4232,10 +4240,12 @@ impl RowMatchGenerator { // its `#[target_feature]` umbrella (no dispatcher branch inside the loop). #[cfg(test)] pub(crate) fn row_candidate(&self, abs_pos: usize, lit_len: usize) -> Option { - crate::encoding::fastpath::with_encode_tier!( - crate::encoding::fastpath::select_kernel(), - T => dispatch_tag_kernel!(T, self.row_candidate_k(abs_pos, lit_len)) - ) + crate::encoding::fastpath::same_under_every_tier(|tier| { + crate::encoding::fastpath::with_encode_tier!( + tier, + T => dispatch_tag_kernel!(T, self.row_candidate_k(abs_pos, lit_len)) + ) + }) } #[cfg(test)] fn row_candidate_k( diff --git a/zstd/src/encoding/streaming_encoder.rs b/zstd/src/encoding/streaming_encoder.rs index 6604928f..ba0fe025 100644 --- a/zstd/src/encoding/streaming_encoder.rs +++ b/zstd/src/encoding/streaming_encoder.rs @@ -1302,11 +1302,9 @@ impl CompressionContext { /// Where the full pending block is cut (upstream `ZSTD_compress_frameChunk` /// sizing every block with `ZSTD_optimalBlockSize`): the pre-splitter's - /// boundary once the frame has saved enough, else the whole block. - /// `remaining` is the input still to come as far as the splitter knows: - /// a full block again while writes continue, the buffered bytes at the - /// end of the frame. - fn pre_split_len(&mut self, block_capacity: usize, remaining: usize) -> usize { + /// boundary once the frame has saved enough, else the whole block. Writes + /// may continue, so the splitter is told a full block is still to come. + fn pre_split_len(&mut self, block_capacity: usize) -> usize { if matches!(self.compression_level, CompressionLevel::Uncompressed) { return self.pending_len(); } @@ -1315,7 +1313,7 @@ impl CompressionContext { let cut = crate::encoding::frame_compressor::optimal_block_size_with( self.state.pre_split.map(usize::from), pending, - remaining, + block_capacity, block_capacity, self.savings, ) @@ -1343,7 +1341,7 @@ impl CompressionContext { if self.pending_len() != block_capacity { return Ok(()); } - let block_len = self.pre_split_len::(block_capacity, block_capacity); + let block_len = self.pre_split_len::(block_capacity); self.encode_block::(drain, block_len, false) .map_err(|err| self.fail(err)) } @@ -1353,21 +1351,11 @@ impl CompressionContext { drain: &mut D, last_block: bool, ) -> Result<(), Error> { - let block_capacity = self.block_capacity(); - if last_block { - // A full final buffer is cut like any other block (the reader - // path splits it with `remaining = len`); each cut prefix goes - // out as a non-last block and the suffix is re-examined. - while self.pending_len() == block_capacity { - let block_len = self.pre_split_len::(block_capacity, block_capacity); - if block_len == block_capacity { - break; - } - self.encode_block::(drain, block_len, false) - .map_err(|err| self.fail(err))?; - } - } + // `write` emits a block the moment it fills, so what is left here is + // always short of one and goes out whole; a write that failed on the + // full block left the context failed, and `ensure_open` stops here. let block_len = self.pending_len(); + debug_assert!(block_len < self.block_capacity()); self.encode_block::(drain, block_len, last_block) .map_err(|err| self.fail(err)) } diff --git a/zstd/src/encoding/streaming_encoder/tests.rs b/zstd/src/encoding/streaming_encoder/tests.rs index 4ff37d1b..365f5733 100644 --- a/zstd/src/encoding/streaming_encoder/tests.rs +++ b/zstd/src/encoding/streaming_encoder/tests.rs @@ -1960,13 +1960,15 @@ fn a_streamed_raw_frame_carries_a_valid_checksum() { assert_eq!(decoded, payload); } -/// A full final buffer is cut like any other block: where the content changes -/// inside it, the pre-split sends the prefix out as a block of its own before -/// the last one, and the frame still decodes to its input. +/// The buffer the last write fills is cut where its content changes, once the +/// frame has saved enough for the pre-splitter to run: the prefix goes out +/// when the buffer fills and the suffix waits to close the frame. Uncut, the +/// whole buffer would go out with the write and the frame would close on an +/// empty last block. #[test] -fn a_full_last_buffer_is_pre_split_where_its_content_changes() { - let mut payload = b"log line with a steady shape, ".repeat(64 * 1024 / 30); - payload.truncate(64 * 1024); +fn a_buffer_filled_by_the_last_write_is_pre_split_where_its_content_changes() { + let mut payload = b"log line with a steady shape, ".repeat(320 * 1024 / 30 + 1); + payload.truncate(320 * 1024); let mut state = 0x2545_F491_u32; payload.extend((0..64 * 1024).map(|_| { state ^= state << 13; @@ -1977,8 +1979,121 @@ fn a_full_last_buffer_is_pre_split_where_its_content_changes() { let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Level(19)); encoder.write_all(&payload).unwrap(); let compressed = encoder.finish().unwrap(); + // RFC 8878 3.1.1.2: a 3-byte header (Last_Block, Block_Type, Block_Size) + // then the content: Block_Size bytes, or one byte for an RLE block. + let (_, header_len) = crate::decoding::frame::read_frame_header(&compressed[..]).unwrap(); + let mut at = usize::from(header_len); + let last = loop { + let header = + u32::from_le_bytes([compressed[at], compressed[at + 1], compressed[at + 2], 0]); + let (block_type, size) = ((header >> 1) & 3, (header >> 3) as usize); + at += 3 + if block_type == 1 { 1 } else { size }; + if header & 1 == 1 { + break (block_type, size); + } + }; + assert_ne!( + last, + (0, 0), + "the full buffer went out uncut: the frame closed on an empty block" + ); let mut decoder = StreamingDecoder::new(compressed.as_slice()); let mut decoded = Vec::new(); decoder.read_to_end(&mut decoded).unwrap(); assert_eq!(decoded, payload); } + +/// Write `input` in pieces that end off block boundaries, flushing once +/// partway, and finish the frame. +fn stream_in_pieces( + mut encoder: StreamingEncoder, M>, + input: &[u8], +) -> Vec { + for (i, piece) in input.chunks(100_000).enumerate() { + encoder.write_all(piece).unwrap(); + if i == 0 { + encoder.flush().unwrap(); + } + } + encoder.finish().unwrap() +} + +/// What streaming `input` under `tier` writes: frames from the built-in +/// matcher and from one reached only by the trait, and a frame start with one +/// block compressed on its own. +fn streams_under_tier( + tier: crate::encoding::fastpath::FastpathKernel, + level: CompressionLevel, + input: &[u8], + dict: bool, +) -> Vec> { + use crate::encoding::test_support::{TraitDriver, tier_dictionary}; + let dictionary = || EncoderDictionary::from_dictionary(tier_dictionary()); + let mut streams = Vec::new(); + + let mut encoder = StreamingEncoder::new(Vec::new(), level); + encoder.context.state.copy_kernel = tier; + if dict { + encoder.set_encoder_dictionary(dictionary()).unwrap(); + } + streams.push(stream_in_pieces(encoder, input)); + + let mut encoder = StreamingEncoder::new_with_matcher(TraitDriver::new(tier), Vec::new(), level); + encoder.context.state.copy_kernel = tier; + if dict { + encoder.set_encoder_dictionary(dictionary()).unwrap(); + } + streams.push(stream_in_pieces(encoder, input)); + + let mut context = CompressionContext::new(level); + context.state.copy_kernel = tier; + if dict { + context.set_encoder_dictionary(dictionary()).unwrap(); + } + let mut block = Vec::new(); + context.start_frame(&mut block).unwrap(); + assert!( + context + .compress_block(&input[..64 * 1024], &mut block) + .unwrap() + ); + streams.push(block); + streams +} + +/// Every tier the CPU can run streams the same frames, with and without a +/// dictionary, at a level of each backend, and every finished frame decodes +/// to its input: the streaming counterpart of the frame compressor's check, +/// over the write, flush, finish and single-block entry points. +#[test] +fn every_tier_streams_the_same_frames() { + use crate::encoding::test_support::{TIER_LEVELS, tier_dictionary, tier_fixture}; + let input = tier_fixture(); + let tiers = crate::encoding::fastpath::runnable_tiers(); + for level in TIER_LEVELS { + let level = CompressionLevel::Level(level); + for dict in [false, true] { + let reference = streams_under_tier(tiers[0], level, &input, dict); + // The last entry is one block's content, not a frame. + for frame in &reference[..reference.len() - 1] { + let mut decoder = crate::decoding::FrameDecoder::new(); + if dict { + decoder.add_dict(tier_dictionary()).unwrap(); + } + let mut decoded = Vec::with_capacity(input.len()); + decoder.decode_all_to_vec(frame, &mut decoded).unwrap(); + assert!( + decoded == input, + "{level:?} dict {dict}: a frame does not decode to its input" + ); + } + for &tier in &tiers[1..] { + assert!( + streams_under_tier(tier, level, &input, dict) == reference, + "{level:?} dict {dict}: tier {tier:?} streamed other bytes than {:?}", + tiers[0], + ); + } + } + } +} diff --git a/zstd/src/encoding/test_support.rs b/zstd/src/encoding/test_support.rs index 9f800faa..5a85957d 100644 --- a/zstd/src/encoding/test_support.rs +++ b/zstd/src/encoding/test_support.rs @@ -2,7 +2,124 @@ use alloc::vec::Vec; -use super::Sequence; +use super::{CompressionLevel, HistoryBuf, MatchGeneratorDriver, Matcher, Sequence}; + +/// The built-in driver reached only through [`Matcher`], as a caller wrapping +/// it does: the encoder sees no built-in driver, so every block runs through +/// the trait entry points under the driver's own tier. +pub(crate) struct TraitDriver(pub(crate) MatchGeneratorDriver); + +impl TraitDriver { + /// A driver whose trait entry points run under `tier`. + pub(crate) fn new(tier: super::fastpath::FastpathKernel) -> Self { + let mut driver = MatchGeneratorDriver::new(crate::common::MAX_BLOCK_SIZE as usize, 1); + driver.set_tier(tier); + Self(driver) + } +} + +impl Matcher for TraitDriver { + fn get_last_space(&mut self) -> &[u8] { + self.0.get_last_space() + } + + fn fill_in_place( + &mut self, + capacity: usize, + fill: &mut dyn FnMut(&mut HistoryBuf) -> (usize, bool), + ) -> (usize, bool) { + self.0.fill_in_place(capacity, fill) + } + + fn uncommitted_input(&self) -> &[u8] { + self.0.uncommitted_input() + } + + fn commit_filled(&mut self, len: usize) { + self.0.commit_filled(len); + } + + fn skip_matching(&mut self) { + self.0.skip_matching(); + } + + fn skip_matching_with_hint(&mut self, incompressible_hint: Option) { + self.0.skip_matching_with_hint(incompressible_hint); + } + + fn start_matching(&mut self, handle_sequence: impl FnMut(Sequence)) { + self.0.start_matching(handle_sequence); + } + + fn reset(&mut self, level: CompressionLevel) { + self.0.reset(level); + } + + fn set_source_size_hint(&mut self, size: u64) { + self.0.set_source_size_hint(size); + } + + fn set_dictionary_size_hint(&mut self, sizes: super::DictionarySizes) { + self.0.set_dictionary_size_hint(sizes); + } + + fn prime_with_dictionary(&mut self, dict_content: &[u8], offset_hist: [u32; 3]) { + self.0.prime_with_dictionary(dict_content, offset_hist); + } + + fn seed_dictionary_entropy( + &mut self, + huff: Option<&crate::huff0::huff0_encoder::HuffmanTable>, + ll: Option<&crate::fse::fse_encoder::FSETable>, + ml: Option<&crate::fse::fse_encoder::FSETable>, + of: Option<&crate::fse::fse_encoder::FSETable>, + ) { + self.0.seed_dictionary_entropy(huff, ll, ml, of); + } + + fn supports_dictionary_priming(&self) -> bool { + self.0.supports_dictionary_priming() + } + + fn settle_block_cut(&mut self, window: usize, cut: usize) -> usize { + self.0.settle_block_cut(window, cut) + } + + fn window_size(&self) -> u64 { + self.0.window_size() + } +} + +/// Levels across every backend: the negative and positive Fast band, dfast, +/// the row lazy band, btlazy2 and the optimal parsers. +pub(crate) const TIER_LEVELS: [i32; 7] = [-5, 1, 3, 6, 13, 16, 19]; + +/// Input that takes each block path: full blocks of log lines, so the frame +/// saves enough for the pre-splitter, then a block that turns to noise halfway +/// (a pre-split cut), a block of one byte repeated (a block written without +/// a search) and a short tail of text. +pub(crate) fn tier_fixture() -> Vec { + const BLOCK: usize = crate::common::MAX_BLOCK_SIZE as usize; + let line = b"ts=2026-03-26T21:39:28Z level=INFO msg=\"flush memtable\" table=orders\n"; + let mut data = line.repeat((2 * BLOCK + BLOCK / 2) / line.len() + 1); + data.truncate(2 * BLOCK + BLOCK / 2); + let mut state = 0x2545_F491_u32; + data.extend((0..BLOCK / 2).map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + })); + data.extend(core::iter::repeat_n(b'z', BLOCK)); + data.extend_from_slice(&line.repeat(40)); + data +} + +/// A raw-content dictionary of the fixture's log lines. +pub(crate) fn tier_dictionary() -> crate::decoding::Dictionary { + let content = b"level=INFO msg=\"rotate segment\" table=orders region=eu-west\n".repeat(64); + crate::decoding::Dictionary::from_raw_content(0xD1C7_7135, content).unwrap() +} /// Rebuilds a block from its sequences the way a decoder does: each literal /// run is taken from the block at the position the sequences before it reach, From c248fba91c0d513f124bd63ca6aea58b7bdc8fd6 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sun, 25 Oct 2026 11:14:04 +0200 Subject: [PATCH 14/15] test(encode): cover refused blocks and every optimal strategy - The optimal-search tier test covers btopt and btultra as well as btultra2. - A drain refusing the block a flush or a finish emits fails that call and leaves the context failed. --- zstd/src/encoding/match_generator/tests.rs | 69 ++++++++++---------- zstd/src/encoding/streaming_encoder/tests.rs | 26 ++++++++ 2 files changed, 62 insertions(+), 33 deletions(-) diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index 1110da1f..c1a44ddf 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -752,41 +752,44 @@ fn bt_optimal_all_kernel_tiers_emit_identical_sequences() { .map(|i| ((i * 7 + i / 13) % 67) as u8) .collect(); - let run = |tier: FastpathKernel| -> Vec<(usize, usize, usize)> { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure( - BTULTRA2_HC_CONFIG, - super::super::strategy::StrategyTag::BtUltra2, - 26, - ); - hc.table.commit_input(&data); - let mut seqs = Vec::new(); - crate::encoding::fastpath::with_encode_tier!( - tier, - K => hc.start_matching_with::(dyn_sink(&mut |seq| match seq { - Sequence::Triple { - literal_len, - offset, - match_len, - } => seqs.push((literal_len, offset, match_len)), - Sequence::Literals { len } => seqs.push((len, 0, 0)), - })) - ); - seqs - }; + use super::super::strategy::StrategyTag; + for strategy in [ + StrategyTag::BtOpt, + StrategyTag::BtUltra, + StrategyTag::BtUltra2, + ] { + let run = |tier: FastpathKernel| -> Vec<(usize, usize, usize)> { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure(BTULTRA2_HC_CONFIG, strategy, 26); + hc.table.commit_input(&data); + let mut seqs = Vec::new(); + crate::encoding::fastpath::with_encode_tier!( + tier, + K => hc.start_matching_with::(dyn_sink(&mut |seq| match seq { + Sequence::Triple { + literal_len, + offset, + match_len, + } => seqs.push((literal_len, offset, match_len)), + Sequence::Literals { len } => seqs.push((len, 0, 0)), + })) + ); + seqs + }; - let reference = run(tiers[0]); - assert!( - !reference.is_empty(), - "btultra2 should emit sequences on mixed-redundancy input" - ); - for &tier in &tiers[1..] { - assert_eq!( - run(tier), - reference, - "kernel tier {tier:?} diverged from {:?}: scalar/SIMD bit-identity broken", - tiers[0], + let reference = run(tiers[0]); + assert!( + !reference.is_empty(), + "{strategy:?} should emit sequences on mixed-redundancy input" ); + for &tier in &tiers[1..] { + assert_eq!( + run(tier), + reference, + "{strategy:?}: kernel tier {tier:?} diverged from {:?}: scalar/SIMD bit-identity broken", + tiers[0], + ); + } } } diff --git a/zstd/src/encoding/streaming_encoder/tests.rs b/zstd/src/encoding/streaming_encoder/tests.rs index 365f5733..48b035ae 100644 --- a/zstd/src/encoding/streaming_encoder/tests.rs +++ b/zstd/src/encoding/streaming_encoder/tests.rs @@ -2003,6 +2003,32 @@ fn a_buffer_filled_by_the_last_write_is_pre_split_where_its_content_changes() { assert_eq!(decoded, payload); } +/// A drain refusing the block a flush or a finish emits fails that call and +/// leaves the context failed, so a later write reports the failure rather than +/// continuing a frame with a block missing. +#[test] +fn a_drain_refusing_the_pending_block_fails_flush_and_finish() { + for finish in [false, true] { + let mut context = CompressionContext::new(CompressionLevel::Fastest); + // The header goes out with the first write; the block is the second. + let mut drain = FailingWriteOnce::new(2); + context.write(&mut drain, b"ABCD").unwrap(); + let emitted = if finish { + context.finish_frame(&mut drain) + } else { + context.flush(&mut drain) + }; + assert!( + emitted.is_err(), + "finish {finish}: the refused block must fail the call" + ); + assert!( + context.write(&mut drain, b"EFGH").is_err(), + "finish {finish}: the context must stay failed", + ); + } +} + /// Write `input` in pieces that end off block boundaries, flushing once /// partway, and finish the frame. fn stream_in_pieces( From 37862f30c9cdefa5ce7ad40ae4ad57339c081277 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sun, 25 Oct 2026 14:16:52 +0200 Subject: [PATCH 15/15] feat(encode): pick the lazy band's match finder The greedy, lazy and lazy2 strategies search rows or a hash chain. The choice followed the window alone; it can now be set, as upstream's ZSTD_c_useRowMatchFinder sets it (zstd_compress.c:246). - RowMatchFinderMode (auto, enable, disable) on the parameter builder. A dictionary frame keeps its dictionary's choice, as upstream's CDict does. The C ABI takes ZSTD_c_useRowMatchFinder and the command takes --[no-]row-match-finder; both were accepted and ignored before. - The hash-chain backend's own lazy parser is removed. The lazy band runs one parse in the row backend, which already searches the chain for small windows, so nothing in a build reached the second parser except tests. With it go its chain layout, chain dictionary state, borrowed scan and the HcBackend discriminator: the backend now holds only the optimal parsers' tree. - Tests that drove the removed parser now drive the path that runs: the chain finder in the row backend, or the tree. The long-distance test on the chain uses the new mode. Frames are byte-identical to the previous commit at every level from -5 to 22 on five fixtures, with and without a dictionary. --- README.md | 10 +- c-api/src/cctx_params.rs | 25 +- c-api/src/tests/cctx_params.rs | 55 + zstd/src/bin/structured-zstd/main.rs | 44 +- zstd/src/bin/structured-zstd/tests.rs | 60 +- zstd/src/encoding/cparams.rs | 14 +- zstd/src/encoding/frame_compressor.rs | 3 +- zstd/src/encoding/hc/generator.rs | 545 ++-------- zstd/src/encoding/hc/hc_tests.rs | 154 --- zstd/src/encoding/hc/mod.rs | 938 +----------------- zstd/src/encoding/hc/optimal.rs | 67 +- zstd/src/encoding/lazy_parse.rs | 18 +- zstd/src/encoding/levels/config.rs | 42 +- .../encoding/match_generator/dict_prime.rs | 60 +- .../encoding/match_generator/long_distance.rs | 2 +- zstd/src/encoding/match_generator/mod.rs | 175 +--- zstd/src/encoding/match_generator/tests.rs | 477 ++++----- zstd/src/encoding/match_table/storage.rs | 551 ++-------- .../match_table/storage/storage_tests.rs | 94 +- zstd/src/encoding/mod.rs | 3 +- zstd/src/encoding/parameters.rs | 67 ++ zstd/src/encoding/row/mod.rs | 11 +- zstd/src/encoding/strategy.rs | 23 +- 23 files changed, 740 insertions(+), 2698 deletions(-) delete mode 100644 zstd/src/encoding/hc/hc_tests.rs diff --git a/README.md b/README.md index 51e2fbeb..f9ae3419 100644 --- a/README.md +++ b/README.md @@ -389,13 +389,13 @@ benefits. |-------------|----------|---------| | 1-2 | `Fast` | `Simple` matcher | | 3-4 | `Dfast` | `Dfast` two-tier hash | -| 5-12 | `Greedy` / `Lazy` / `Lazy2` | `Row` lazy parse (`lazy_depth=0/1/2`): row match-finder above a 2^14 window, hash chain at or below it | +| 5-12 | `Greedy` / `Lazy` / `Lazy2` | `Row` lazy parse (`lazy_depth=0/1/2`): row match-finder above a 2^14 window, hash chain at or below it, unless `RowMatchFinderMode` forces one | | 13-15 | `Btlazy2` | `Row` lazy parse over the lazily-sorted binary tree | -| 16-17 | `BtOpt` | `HashChain` candidates + `btopt` price parser | -| 18 | `BtUltra` | `HashChain` candidates + `btultra` price parser | -| 19-22 | `BtUltra2` | `HashChain` candidates + `btultra2` dual-profile parse | +| 16-17 | `BtOpt` | binary-tree candidates + `btopt` price parser | +| 18 | `BtUltra` | binary-tree candidates + `btultra` price parser | +| 19-22 | `BtUltra2` | binary-tree candidates + `btultra2` dual-profile parse | -The level → strategy column matches upstream zstd `ZSTD_defaultCParameters[0]` at `zstd/lib/compress/clevels.h:25-50` (srcSize > 256 KiB tier); smaller sources shift the row per upstream's size tiers. The whole greedy..btlazy2 band runs upstream's `ZSTD_compressBlock_lazy_generic` parse on the `Row` backend over the three upstream match finders (rows / hash chain per `ZSTD_resolveRowMatchFinderMode`, lazily-sorted binary tree for `btlazy2`). +The level → strategy column matches upstream zstd `ZSTD_defaultCParameters[0]` at `zstd/lib/compress/clevels.h:25-50` (srcSize > 256 KiB tier); smaller sources shift the row per upstream's size tiers. The whole greedy..btlazy2 band runs upstream's `ZSTD_compressBlock_lazy_generic` parse on the `Row` backend over the three upstream match finders (rows / hash chain per `ZSTD_resolveRowMatchFinderMode`, which `CompressionParametersBuilder::row_match_finder` and `ZSTD_c_useRowMatchFinder` override; lazily-sorted binary tree for `btlazy2`). diff --git a/c-api/src/cctx_params.rs b/c-api/src/cctx_params.rs index 9bec9b89..c14184ff 100644 --- a/c-api/src/cctx_params.rs +++ b/c-api/src/cctx_params.rs @@ -4,7 +4,9 @@ use core::ffi::c_int; -use codec::encoding::{CompressionLevel, CompressionParameters, LiteralCompressionMode, Strategy}; +use codec::encoding::{ + CompressionLevel, CompressionParameters, LiteralCompressionMode, RowMatchFinderMode, Strategy, +}; use crate::cdict::ZSTD_compressionParameters; use crate::context::{ZSTD_CCtx, free_boxed, try_box}; @@ -46,7 +48,7 @@ impl ParamSwitch { /// searches, schedules or checks its work, with no counterpart in this /// encoder: stored as set so they read back, without effect on the frames. /// The sequence-format ones are read by the sequence entry points. -const ADVISORY: [c_int; 18] = [ +const ADVISORY: [c_int; 17] = [ ZSTD_C_NB_WORKERS, ZSTD_C_JOB_SIZE, ZSTD_C_OVERLAP_LOG, @@ -59,7 +61,6 @@ const ADVISORY: [c_int; 18] = [ ZSTD_C_BLOCK_DELIMITERS, ZSTD_C_VALIDATE_SEQUENCES, ZSTD_C_SPLIT_AFTER_SEQUENCES, - ZSTD_C_USE_ROW_MATCH_FINDER, ZSTD_C_DETERMINISTIC_REF_PREFIX, ZSTD_C_PREFETCH_CDICT_TABLES, ZSTD_C_ENABLE_SEQ_PRODUCER_FALLBACK, @@ -105,6 +106,8 @@ pub(crate) struct CCtxParams { pub(crate) magicless: bool, /// `ZSTD_c_literalCompressionMode`. pub(crate) literal_compression: LiteralCompressionMode, + /// `ZSTD_c_useRowMatchFinder`. + pub(crate) row_match_finder: RowMatchFinderMode, /// `ZSTD_c_srcSizeHint`; 0 = none. pub(crate) src_size_hint: c_int, /// The [`ADVISORY`] parameters, in that order. @@ -147,6 +150,7 @@ impl CCtxParams { max_block_size: 0, magicless: false, literal_compression: LiteralCompressionMode::Auto, + row_match_finder: RowMatchFinderMode::Auto, src_size_hint: 0, advisory: [0; ADVISORY.len()], pledged_src_size: CONTENTSIZE_UNKNOWN, @@ -179,7 +183,8 @@ impl CCtxParams { /// (should be unreachable: every knob was bounds-checked on entry). pub(crate) fn resolve(&self) -> Option { let mut builder = CompressionParameters::builder(CompressionLevel::from_level(self.level)) - .literal_compression(self.literal_compression); + .literal_compression(self.literal_compression) + .row_match_finder(self.row_match_finder); if let Some(v) = self.window_log { builder = builder.window_log(v); } @@ -301,6 +306,13 @@ impl CCtxParams { _ => LiteralCompressionMode::Auto, }; } + ZSTD_C_USE_ROW_MATCH_FINDER => { + self.row_match_finder = match ParamSwitch::from_c(within(value)?) { + Some(ParamSwitch::Enable) => RowMatchFinderMode::Enable, + Some(ParamSwitch::Disable) => RowMatchFinderMode::Disable, + _ => RowMatchFinderMode::Auto, + }; + } // Clamped into range as upstream's multi-threaded build stores them // (zstd_compress.c:866-908); frames are still built on the calling // thread, which a worker count only permits, never requires. @@ -371,6 +383,11 @@ impl CCtxParams { LiteralCompressionMode::Enable => ParamSwitch::Enable.to_c(), LiteralCompressionMode::Disable => ParamSwitch::Disable.to_c(), }, + ZSTD_C_USE_ROW_MATCH_FINDER => match self.row_match_finder { + RowMatchFinderMode::Auto => ParamSwitch::Auto.to_c(), + RowMatchFinderMode::Enable => ParamSwitch::Enable.to_c(), + RowMatchFinderMode::Disable => ParamSwitch::Disable.to_c(), + }, _ => match advisory_slot(param) { Some(slot) => self.advisory[slot], None => return Err(ZSTD_ErrorCode::ZSTD_error_parameter_unsupported), diff --git a/c-api/src/tests/cctx_params.rs b/c-api/src/tests/cctx_params.rs index 358cf656..3bd614b1 100644 --- a/c-api/src/tests/cctx_params.rs +++ b/c-api/src/tests/cctx_params.rs @@ -45,6 +45,7 @@ const FORCE_MAX_WINDOW: c_int = 1000; const FORCE_ATTACH_DICT: c_int = 1001; const LITERAL_COMPRESSION_MODE: c_int = 1002; const SRC_SIZE_HINT: c_int = 1004; +const USE_ROW_MATCH_FINDER: c_int = 1011; const MAX_BLOCK_SIZE: c_int = 1015; const BLOCK_SPLITTER_LEVEL: c_int = 1017; @@ -421,6 +422,60 @@ fn literal_compression_mode_reaches_the_encoder() { assert_eq!(decompress(&raw, text.len()), text); } +/// `ZSTD_c_useRowMatchFinder` picks the greedy / lazy band's finder: `enable` +/// writes what `auto` writes on a source wide enough for rows and `disable` +/// what `auto` writes on one of 16 KiB or less (upstream +/// `ZSTD_resolveRowMatchFinderMode`), the forced choice differs from the +/// window's, and every frame decodes. A value past the switch is refused. +#[test] +fn use_row_match_finder_reaches_the_encoder() { + let text = |lines: u32| -> Vec { + let mut state = 0x9E37_79B9u32; + let mut out = Vec::new(); + for i in 0..lines { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + out.extend_from_slice( + format!( + "{i:06} level=info shard={} took={}ms\n", + state % 13, + state >> 22 + ) + .as_bytes(), + ); + } + out + }; + let frame = |input: &[u8], mode: c_int| { + let cctx = ZSTD_createCCtx(); + unsafe { + assert_eq!(ZSTD_CCtx_setParameter(cctx, LEVEL, 7), 0); + assert_eq!(ZSTD_CCtx_setParameter(cctx, USE_ROW_MATCH_FINDER, mode), 0); + assert_eq!(get_cctx(cctx, USE_ROW_MATCH_FINDER), mode); + } + let out = compress2(cctx, input); + unsafe { ZSTD_freeCCtx(cctx) }; + assert_eq!(decompress(&out, input.len()), input, "mode {mode}"); + out + }; + let (auto, enable, disable) = (0, 1, 2); + let wide = text(6000); + assert_eq!(frame(&wide, enable), frame(&wide, auto)); + assert_ne!(frame(&wide, disable), frame(&wide, auto)); + let narrow = text(200); + assert!(narrow.len() <= 1 << 14); + assert_eq!(frame(&narrow, disable), frame(&narrow, auto)); + assert_ne!(frame(&narrow, enable), frame(&narrow, auto)); + + let cctx = ZSTD_createCCtx(); + unsafe { + assert_eq!( + code(ZSTD_CCtx_setParameter(cctx, USE_ROW_MATCH_FINDER, 3)), + ZSTD_ErrorCode::ZSTD_error_parameter_outOfBound + ); + ZSTD_freeCCtx(cctx); + } +} + /// A size hint stays for every streamed frame and can be cleared again. #[test] fn src_size_hint_round_trips_streamed_frames() { diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index 321f865f..853db47e 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -17,7 +17,7 @@ use std::path::{Path, PathBuf}; use structured_zstd::encoding::{ CompressionContext, CompressionLevel, CompressionParameters, LevelParameters, - LiteralCompressionMode, Strategy, StreamingEncoder, + LiteralCompressionMode, RowMatchFinderMode, Strategy, StreamingEncoder, }; /// Error type for the tool: a boxed message, which is all a command-line @@ -341,6 +341,8 @@ struct Options { advanced: AdvancedParams, /// Whether literals are entropy-coded (`--[no-]compress-literals`). literals: LiteralCompressionMode, + /// The greedy / lazy band's finder (`--[no-]row-match-finder`). + row_match_finder: RowMatchFinderMode, /// Reference file for `--patch-from`: raw-content dictionary compression /// with the window sized to the input, so the whole reference is /// reachable. @@ -1498,6 +1500,7 @@ fn parse_args_into( progress: Progress::Auto, advanced: AdvancedParams::default(), literals: LiteralCompressionMode::Auto, + row_match_finder: RowMatchFinderMode::Auto, patch_from: None, show_default_cparams: false, trainer: Trainer::FastCover, @@ -1612,7 +1615,7 @@ fn parse_args_into( return Ok(Parsed::Handled { pause: *pause }); } // Flags that steer HOW the work is done, not what comes out: - // thread counts, IO strategy, matcher hints. We are + // thread counts, IO strategy. We are // single-threaded and pick our own limits, so accepting them // yields the same valid stream. Upstream takes them, so a // script that passes them must not fail here: that is the @@ -1620,14 +1623,8 @@ fn parse_args_into( "single-thread" => opts.threads = Some(0), "adapt" => opts.adapt = true, "rsyncable" => opts.rsyncable = true, - "sparse" - | "no-sparse" - | "asyncio" - | "no-asyncio" - | "mmap-dict" - | "no-mmap-dict" - | "row-match-finder" - | "no-row-match-finder" => {} + "sparse" | "no-sparse" | "asyncio" | "no-asyncio" | "mmap-dict" + | "no-mmap-dict" => {} // The reference prints a trace of its own file-system helpers // by their C names (`UTIL_stat`, `UTIL_isLink`, ...). There is // nothing of ours that output would describe, and taking the @@ -1643,6 +1640,8 @@ fn parse_args_into( "fake-stderr-is-console" => opts.console.stderr = true, "compress-literals" => opts.literals = LiteralCompressionMode::Enable, "no-compress-literals" => opts.literals = LiteralCompressionMode::Disable, + "row-match-finder" => opts.row_match_finder = RowMatchFinderMode::Enable, + "no-row-match-finder" => opts.row_match_finder = RowMatchFinderMode::Disable, _ => { if long == "fast" { // `--fast` is the level -1 alias. @@ -2310,6 +2309,8 @@ Advanced compression options: --no-dictID Don't write `dictID` into the header (dictionary compression only). --[no-]content-size Write the input size into the frame header when it is known. [Default: Write] --[no-]compress-literals Force (un)compressed literals. + --[no-]row-match-finder Explicitly enable/disable the fast, row-based matchfinder for + the 'greedy', 'lazy', and 'lazy2' strategies. --format=zstd Compress files to the `.zst` format. [Default] @@ -2347,8 +2348,7 @@ Environment: ZSTD_CLEVEL sets the default compression level; ZSTD_NBTHREADS is r Accepted for compatibility, with no effect here: -T#/--threads=#, --single-thread, --auto-threads, -B# and --block-size=# when compressing (the job size of a multi-threaded run), --adapt, --rsyncable, --zstd=ovlog=#, --[no-]sparse, ---[no-]asyncio, --[no-]mmap-dict, --[no-]row-match-finder (compression runs -single-threaded). --adapt and --rsyncable are refused beside --single-thread or +--[no-]asyncio, --[no-]mmap-dict (compression runs single-threaded). --adapt and --rsyncable are refused beside --single-thread or -T0 when compressing. Rejected rather than ignored, because they would change the result: --format= @@ -6291,6 +6291,8 @@ struct FrameSettings { advanced: AdvancedParams, /// Whether literals are entropy-coded (`--[no-]compress-literals`). literals: LiteralCompressionMode, + /// The greedy / lazy band's finder (`--[no-]row-match-finder`). + row_match_finder: RowMatchFinderMode, } impl Default for FrameSettings { @@ -6310,6 +6312,7 @@ impl Default for FrameSettings { dict_id_flag: true, advanced: AdvancedParams::default(), literals: LiteralCompressionMode::Auto, + row_match_finder: RowMatchFinderMode::Auto, } } } @@ -6330,21 +6333,26 @@ impl FrameSettings { dict_id_flag: opts.dict_id_flag, advanced: opts.advanced, literals: opts.literals, + row_match_finder: opts.row_match_finder, } } } /// The fine-grained parameters a frame runs under, or `None` when nothing /// overrides the level: `--zstd=` knobs, `--long` (whose window yields to a -/// `--zstd=wlog=`, as the reference command has it), and the literal mode. -/// The long-distance knobs from `--zstd=` reach the encoder only with -/// `--long`, since there they tune a matcher that otherwise does not run. +/// `--zstd=wlog=`, as the reference command has it), and the literal and +/// match-finder modes. The long-distance knobs from `--zstd=` reach the +/// encoder only with `--long`, since there they tune a matcher that otherwise +/// does not run. fn frame_parameters( level: CompressionLevel, settings: &FrameSettings, ) -> Result> { let advanced = settings.advanced; - if !settings.long && advanced.is_default() && settings.literals == LiteralCompressionMode::Auto + if !settings.long + && advanced.is_default() + && settings.literals == LiteralCompressionMode::Auto + && settings.row_match_finder == RowMatchFinderMode::Auto { return Ok(None); } @@ -6388,7 +6396,9 @@ fn frame_parameters( builder = builder.ldm_hash_rate_log(log); } } - builder = builder.literal_compression(settings.literals); + builder = builder + .literal_compression(settings.literals) + .row_match_finder(settings.row_match_finder); builder .build() .map(Some) diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index 236a7e28..7c499d94 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -4491,7 +4491,6 @@ fn performance_only_flags_are_accepted() { &["--sparse", "f"][..], &["--no-asyncio", "f"][..], &["--no-mmap-dict", "f"][..], - &["--no-row-match-finder", "f"][..], ] { let parsed = parse(args); assert!( @@ -7472,6 +7471,65 @@ fn literal_compression_flags_reach_the_frame() { assert_eq!(decoded(&fast_coded).unwrap(), payload); } +/// `--[no-]row-match-finder` picks the greedy / lazy band's finder: forcing the +/// hash chain on a source wide enough for rows writes another frame than the +/// level's own, forcing rows writes the level's own, and both decode. +#[test] +fn row_match_finder_flags_reach_the_frame() { + assert_eq!( + parse(&["--no-row-match-finder", "f"]) + .unwrap() + .row_match_finder, + RowMatchFinderMode::Disable + ); + assert_eq!( + parse(&["--row-match-finder", "f"]) + .unwrap() + .row_match_finder, + RowMatchFinderMode::Enable + ); + assert_eq!( + parse(&["f"]).unwrap().row_match_finder, + RowMatchFinderMode::Auto + ); + + let mut payload = Vec::new(); + let mut state = 0x9E37_79B9u32; + for i in 0..6000u32 { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + payload.extend_from_slice( + format!( + "{i:06} level=info shard={} took={}ms\n", + state % 13, + state >> 22 + ) + .as_bytes(), + ); + } + let frame_with = |row_match_finder: RowMatchFinderMode| { + let mut frame = Vec::new(); + compress_stream( + payload.as_slice(), + &mut frame, + &FrameSettings { + level: 7, + pledged_size: Some(payload.len() as u64), + row_match_finder, + ..FrameSettings::default() + }, + &mut no_dict(), + ) + .unwrap(); + frame + }; + let auto = frame_with(RowMatchFinderMode::Auto); + let rows = frame_with(RowMatchFinderMode::Enable); + let chain = frame_with(RowMatchFinderMode::Disable); + assert_eq!(rows, auto); + assert_ne!(chain, auto); + assert_eq!(decoded(&chain).unwrap(), payload); +} + /// The trainer flags take their tuning the way the reference command reads /// it, and the options built from it follow the reference's rules: a value /// the trainer cannot take is refused, `shrink` defaults its bound to 1%, and diff --git a/zstd/src/encoding/cparams.rs b/zstd/src/encoding/cparams.rs index 40c52c4a..5e11f746 100644 --- a/zstd/src/encoding/cparams.rs +++ b/zstd/src/encoding/cparams.rs @@ -328,11 +328,15 @@ pub(crate) fn copy_cparams(cdict: CParams, window_log: u32) -> CParams { cp } -/// `ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, cp)`: the greedy / lazy / -/// lazy2 strategies search rows only above a 2^14 window. A CDict resolves -/// this from its own cParams and the frame inherits the answer. -pub(crate) fn uses_row_match_finder(cp: &CParams) -> bool { - (3..=5).contains(&cp.strategy) && cp.window_log > 14 +/// `ZSTD_resolveRowMatchFinderMode(mode, cp)` + `ZSTD_rowMatchFinderUsed` +/// (zstd_compress.c:240-261): whether the greedy / lazy / lazy2 strategies +/// search rows. A CDict resolves this from its own cParams and mode, and the +/// frame inherits the answer. +pub(crate) fn uses_row_match_finder( + cp: &CParams, + mode: crate::encoding::parameters::RowMatchFinderMode, +) -> bool { + (3..=5).contains(&cp.strategy) && mode.searches_rows(1usize << cp.window_log) } /// `ZSTD_shouldAttachDict`: the CDict tables are referenced in place diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index d0e8397f..c65159e8 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -2015,8 +2015,7 @@ impl FrameCompressor { return attached; } // The borrowed (no-copy, in-place over-window) scan exists for the - // Simple (Fast), Dfast, and Row backends, and for the HashChain - // backend's lazy CHAIN parser; BT/optimal (BinaryTree search) stay on + // Simple (Fast), Dfast, and Row backends; the optimal parsers stay on // the owned path. Every borrowed scan applies the per-position // `window_low = abs_ip - advertised_window` offset cap so over-window // inputs are matched in place (no input->history copy), matching C's diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs index c09f661d..3c746df8 100644 --- a/zstd/src/encoding/hc/generator.rs +++ b/zstd/src/encoding/hc/generator.rs @@ -1,62 +1,29 @@ -//! HashChain / binary-tree match generator (`HcMatchGenerator`). +//! Binary-tree match generator (`HcMatchGenerator`) of the optimal parsers. //! -//! The HC chain matcher plus the BT-backed optimal-parse machinery the lazy / -//! btopt / btultra strategies run on: the `HcBackend` discriminator, the -//! `HcMatchGenerator` storage + methods, the `btlazy2` cost helpers, the -//! optimal-plan body macros, and the `build_optimal_plan` driver. Moved -//! verbatim from `match_generator.rs` (no behaviour change); encoding-level -//! paths are absolute (`crate::encoding::…`) for the deeper module. +//! The BT-backed optimal-parse machinery the btopt / btultra / btultra2 +//! strategies run on: the `HcMatchGenerator` storage + methods, the +//! optimal-plan body macros, and the `build_optimal_plan` driver; +//! encoding-level paths are absolute (`crate::encoding::…`) for the deeper +//! module. +#[cfg(test)] use crate::encoding::Sequence; -use crate::encoding::blocks::encode_offset_with_history; use crate::encoding::cost_model::HC_PREDEF_THRESHOLD; use crate::encoding::fastpath::EncodeTier; -use crate::encoding::hc::{HC_MIN_MATCH_LEN, HcMatch, MAX_HC_SEARCH_DEPTH}; use crate::encoding::levels::config::HcConfig; -use crate::encoding::match_generator::{HC_SEARCH_DEPTH, HC_TARGET_LEN, HcBackend}; +use crate::encoding::match_generator::{HC_SEARCH_DEPTH, HC_TARGET_LEN}; use crate::encoding::match_table::storage::HC3_HASH_LOG; -impl HcBackend { - /// Heap bytes held by the backend. `Hc` is zero-sized; `Bt` boxes a - /// `BtMatcher`, so count the boxed payload plus its own scratch heap. - pub(crate) fn heap_size(&self) -> usize { - match self { - Self::Hc => 0, - Self::Bt(bt) => core::mem::size_of::() + bt.heap_size(), - } - } - - /// Mutable accessor on the BT matcher; panics if the active - /// backend is `Hc`. The HC-or-Bt branches in orchestrator code use - /// `let HcBackend::Bt(bt) = &self.backend` directly for readonly - /// access — this helper exists so macro bodies that already drive - /// a mutable BT update through the optimal parser can write - /// `$self.backend.bt_mut().X` without an outer `match` ladder. - #[inline(always)] - pub(crate) fn bt_mut(&mut self) -> &mut crate::encoding::bt::BtMatcher { - match self { - Self::Bt(bt) => bt, - Self::Hc => unreachable!("BT-only accessor called in HC mode"), - } - } -} - #[derive(Clone)] pub(crate) struct HcMatchGenerator { - /// Shared match-finder storage (window, history, hash / chain / - /// hash3 tables, dictionary-priming flags). Used identically by HC - /// and BT modes; backend-specific table interpretation lives in the - /// matcher methods on this struct. + /// Shared match-finder storage (window, history, hash / tree / + /// hash3 tables, dictionary-priming flags). pub(crate) table: crate::encoding::match_table::storage::MatchTable, - /// HC runtime knobs (lazy_depth, search_depth, target_len). Always - /// present — BT modes still consult `hc.search_depth` for repcode - /// probing and chain candidate enumeration. + /// Search knobs (search_depth, target_len). pub(crate) hc: crate::encoding::hc::HcMatcher, - /// Backend discriminator. [`HcBackend::Hc`] is zero-sized for the - /// lazy / lazy2 path so HC-only generators don't carry the BT - /// optimal-parser scratch buffers. [`HcBackend::Bt`] holds the - /// `BtMatcher` when an optimal mode is configured. - pub(crate) backend: HcBackend, + /// The optimal parser's per-frame state: cost model, scratch arenas and + /// long-distance candidates. Boxed so the generator stays small. + pub(crate) bt: alloc::boxed::Box, /// Compile-time strategy tag mirrored from /// [`MatchGeneratorDriver::strategy_tag`](crate::encoding::match_generator::MatchGeneratorDriver::strategy_tag) /// during `configure()`. @@ -67,8 +34,7 @@ pub(crate) struct HcMatchGenerator { /// const-generic optimal parser (BtOpt vs BtUltra vs BtUltra2). /// Without this field the test path would have to collapse /// `BtOpt` and `BtUltra` onto the same monomorphisation since - /// `table.uses_bt` / `table.is_btultra2` alone can't tell them - /// apart. + /// `table.is_btultra2` alone can't tell them apart. pub(crate) strategy_tag: crate::encoding::strategy::StrategyTag, } @@ -96,10 +62,9 @@ pub(crate) struct HcMatchGenerator { /// re-exported only inside this crate via `pub(crate) use` below. macro_rules! bt_insert_range_body { ($table:expr, $search_depth:expr, $from:ident, $stop:ident, $current_abs_end:ident, $target_abs:ident, $cmf:path) => {{ - // Borrowed-aware live region (owned: `history[history_start..]`; - // borrowed: the in-place input `[0, block_end)`). Reborrow-then-raw-ptr - // so the slice holds NO borrow and coexists with the `&mut $table` - // binary-tree writes below. Owned is byte-identical (same bytes). + // The live region `history[history_start..]`, reborrowed as a raw + // slice so it holds NO borrow and coexists with the `&mut $table` + // binary-tree writes below. let concat: &[u8] = unsafe { let lh = $table.live_history(); core::slice::from_raw_parts(lh.as_ptr(), lh.len()) @@ -345,102 +310,6 @@ macro_rules! bt_insert_range_body { } pub(crate) use bt_insert_range_body; -/// `for_each_repcode_candidate_with_reps` body parameterized over the per-CPU -/// `common_prefix_len_ptr` symbol so the per-rep prefix probe inlines under -/// the wrapper's `target_feature` umbrella instead of crossing the ABI -/// boundary through the dispatcher. Three rep probes per encoded position → -/// thousands per segment, so the per-call barrier was non-trivial. -/// -/// The callback `f` runs in the wrapper's umbrella context too, so closures -/// that capture mutable state still work (FnMut). Crate-private -/// (see `bt_insert_range_body!`). -macro_rules! for_each_repcode_candidate_body { - ( - $table:expr, - $abs_pos:ident, - $lit_len:ident, - $reps:ident, - $current_abs_end:ident, - $min_match_len:ident, - $f:ident, - $cpl:path $(,)? - ) => {{ - let rep_offsets: [Option; 3] = if $lit_len == 0 { - [ - Some($reps[1] as usize), - Some($reps[2] as usize), - ($reps[0] > 1).then_some(($reps[0] - 1) as usize), - ] - } else { - [ - Some($reps[0] as usize), - Some($reps[1] as usize), - Some($reps[2] as usize), - ] - }; - let concat = $table.live_history(); - let current_idx = $abs_pos - $table.history_abs_start; - if current_idx + 4 > concat.len() { - return; - } - let tail_limit = $current_abs_end.saturating_sub($abs_pos); - let base = concat.as_ptr(); - let concat_len = concat.len(); - for rep in rep_offsets.into_iter().flatten() { - if rep == 0 || rep > $abs_pos { - continue; - } - let candidate_pos = $abs_pos - rep; - if candidate_pos < $table.history_abs_start { - continue; - } - let candidate_idx = candidate_pos - $table.history_abs_start; - // Upstream zstd `ZSTD_readMINMATCH` gate (zstd_opt.c:657-674): a - // 4-byte (3-byte when min_match_len == 3) equality probe - // before the full prefix scan. Equivalent filtering — a - // mismatch here means `match_len < min_match_len`, which - // the post-scan check rejects anyway — but it skips the - // prefix-kernel call for the common no-match case (rep - // offsets rarely hit on low-redundancy input). - // - // SAFETY: `current_idx + 4 <= concat_len` (early return - // above) and `candidate_idx < current_idx` (rep >= 1), so - // both 4-byte reads stay inside `concat`. - let gate_matches = unsafe { - let cand = base.add(candidate_idx).cast::().read_unaligned(); - let cur = base.add(current_idx).cast::().read_unaligned(); - if $min_match_len == 3 { - // Compare the low-address 3 bytes regardless of - // endianness: byte-shift on LE, mask via to_le. - (cand.to_le() & 0x00FF_FFFF) == (cur.to_le() & 0x00FF_FFFF) - } else { - cand == cur - } - }; - if !gate_matches { - continue; - } - // SAFETY: `candidate_idx ≤ current_idx < concat_len` (since - // candidate_pos ≤ abs_pos and we early-returned on - // `current_idx + 4 > concat_len`). `max` clamps to the shorter - // remaining run so neither pointer overruns `concat`. - let max = (concat_len - candidate_idx) - .min(concat_len - current_idx) - .min(tail_limit); - let match_len = unsafe { $cpl(base.add(candidate_idx), base.add(current_idx), max) }; - if match_len < $min_match_len { - continue; - } - $f(MatchCandidate { - start: $abs_pos, - offset: rep, - match_len, - }); - } - }}; -} -pub(crate) use for_each_repcode_candidate_body; - /// Common prefix of `$a` and `$b`, capped at `$limit`, with the first eight /// bytes settled in place: `$cpl` is called only when they all match. Most /// probed candidates part within one word, and upstream zstd's inlined @@ -1300,10 +1169,12 @@ macro_rules! bt_insert_and_collect_matches_body { pub(crate) use bt_insert_and_collect_matches_body; impl HcMatchGenerator { - /// Heap bytes this generator owns: the shared match table plus the BT - /// backend's optimal-parser / LDM scratch (the HC knobs are inline). + /// Heap bytes this generator owns: the shared match table plus the boxed + /// optimal-parser / LDM state and its scratch (the knobs are inline). pub(crate) fn heap_size(&self) -> usize { - self.table.heap_size() + self.backend.heap_size() + self.table.heap_size() + + core::mem::size_of::() + + self.bt.heap_size() } pub(crate) fn should_run_btultra2_seed_pass( @@ -1318,9 +1189,7 @@ impl HcMatchGenerator { if !S::TWO_PASS_SEED { return false; } - let HcBackend::Bt(bt) = &self.backend else { - return false; - }; + let bt = &self.bt; // The frame's first block, whatever absolute position the frame starts // at (upstream `curr == ms->window.dictLimit`): the window holding // only this block says so. A reused compressor starts each frame at @@ -1343,17 +1212,11 @@ impl HcMatchGenerator { pub(crate) fn new(max_window_size: usize) -> Self { Self { table: crate::encoding::match_table::storage::MatchTable::new(max_window_size), - hc: crate::encoding::hc::HcMatcher::new(2, HC_SEARCH_DEPTH, HC_TARGET_LEN), - // Default to the zero-sized HC backend; `configure()` swaps - // in a `BtMatcher` only when an optimal strategy lands. - backend: HcBackend::Hc, - // Lazy is the per-construct default — every production - // caller calls `configure()` before the first encode and - // overwrites this. Tests that drive `HcMatchGenerator` - // without calling `configure()` end up in the - // `start_matching_lazy` arm of the test dispatcher, which - // matches the previous default behaviour. - strategy_tag: crate::encoding::strategy::StrategyTag::Lazy, + hc: crate::encoding::hc::HcMatcher::new(HC_SEARCH_DEPTH, HC_TARGET_LEN), + bt: alloc::boxed::Box::new(crate::encoding::bt::BtMatcher::new()), + // Every caller runs `configure()` before the first encode, which + // overwrites this. + strategy_tag: crate::encoding::strategy::StrategyTag::BtOpt, } } @@ -1368,11 +1231,14 @@ impl HcMatchGenerator { // `#[cfg(test)] start_matching` dispatcher can route // BtOpt / BtUltra / BtUltra2 to distinct monomorphisations. self.strategy_tag = tag; - let is_btultra2 = tag == StrategyTag::BtUltra2; - let uses_bt = matches!( - tag, - StrategyTag::BtOpt | StrategyTag::BtUltra | StrategyTag::BtUltra2 + debug_assert!( + matches!( + tag, + StrategyTag::BtOpt | StrategyTag::BtUltra | StrategyTag::BtUltra2 + ), + "only the optimal strategies run on this backend, got {tag:?}" ); + let is_btultra2 = tag == StrategyTag::BtUltra2; // btultra and btultra2 both run the mls=3 hash3 short-match probe // (clevels.h minMatch 3). The `is_btultra2` flag below stays // exclusive to btultra2 because it tweaks the BT rebase boundary, @@ -1386,60 +1252,38 @@ impl HcMatchGenerator { let resize = self.table.hash_log != config.hash_log || self.table.chain_log != config.chain_log || self.table.hash3_log != next_hash3_log; - // Capture the layout flip BEFORE `uses_bt` is overwritten below — it - // feeds the dms invalidation (the dms is keyed by layout too). - let uses_bt_changed = self.table.uses_bt != uses_bt; self.table.hash_log = config.hash_log; self.table.chain_log = config.chain_log; self.table.hash3_log = next_hash3_log; - self.hc.search_depth = if uses_bt { - config.search_depth - } else { - config.search_depth.min(MAX_HC_SEARCH_DEPTH) - }; + self.hc.search_depth = config.search_depth; self.hc.target_len = config.target_len; - // Mirror strategy-derived flags + HC search depth onto MatchTable - // so the BT walker and rebase machinery can read them directly - // without dispatching back through HcMatchGenerator. + // Mirror strategy-derived flags + search depth onto MatchTable so the + // BT walker and rebase machinery can read them directly without + // dispatching back through HcMatchGenerator. self.table.search_depth = self.hc.search_depth; self.table.is_btultra2 = is_btultra2; - self.table.uses_bt = uses_bt; // BT finder hash width, upstream zstd `mls = BOUNDED(4, cParams.minMatch, 6)`, // carried explicitly in the level config so a `target_length` override - // cannot silently flip the finder between 5- and 4-byte hashing. Only - // the BT body reads it; HC/lazy levels leave it at 4. clevels.h - // (srcSize > 256 KiB tier): btlazy2 L13-15 + btopt L16 are minMatch=5, - // btopt L17 is minMatch=4, btultra/btultra2 are minMatch=3 (4-byte main - // hash + the hash3 short-match probe). + // cannot silently flip the finder between 5- and 4-byte hashing. + // clevels.h (srcSize > 256 KiB tier): btopt L16 is minMatch=5, btopt + // L17 is minMatch=4, btultra/btultra2 are minMatch=3 (4-byte main hash + // + the hash3 short-match probe). // The cached dms is keyed by the full (region, layout, mls, hash_log) // shape that `build_dms!` validates on the normal prime path, but the // reborrow fast path in `MatchTable::reset` reuses it on `dms.is_primed()` - // ALONE. A reused-compressor level switch can change the search mls (e.g. - // btlazy2 -> lazy), the table geometry (hash_log / chain_log / hash3, - // captured in `resize`), OR the HC<->BT layout (`uses_bt_changed`) - // independently of each other, and any of them leaves the dms hashed for - // a different shape. Invalidate on ANY so the next dict frame re-primes at - // the new shape (configure runs before reset) instead of probing a - // mismatched dms and silently degrading match quality. Over-invalidation - // only costs a re-prime, which a real shape change needs anyway. + // ALONE. A reused-compressor level switch can change the search mls or + // the table geometry (hash_log / chain_log / hash3, captured in + // `resize`) independently of each other, and either leaves the dms + // hashed for a different shape. Invalidate on ANY so the next dict frame + // re-primes at the new shape (configure runs before reset) instead of + // probing a mismatched dms and silently degrading match quality. + // Over-invalidation only costs a re-prime, which a real shape change + // needs anyway. let mls_changed = self.table.search_mls != config.search_mls; - if resize || mls_changed || uses_bt_changed { + if resize || mls_changed { self.table.dms.invalidate(); } self.table.search_mls = config.search_mls; - // Stage D: promote the backend discriminator. HC modes drop the - // BT scratch buffers entirely; switching back into a BT mode - // allocates a fresh `BtMatcher` on demand. - match (&self.backend, self.table.uses_bt) { - (HcBackend::Hc, true) => { - self.backend = - HcBackend::Bt(alloc::boxed::Box::new(crate::encoding::bt::BtMatcher::new())); - } - (HcBackend::Bt(_), false) => { - self.backend = HcBackend::Hc; - } - _ => {} - } if resize && !self.table.tables.is_empty() { // Drop the tables so they are laid out (or allocated) again at the // new widths. The seams go with the buffer. @@ -1456,29 +1300,22 @@ impl HcMatchGenerator { ml: Option<&crate::fse::fse_encoder::FSETable>, of: Option<&crate::fse::fse_encoder::FSETable>, ) { - if let HcBackend::Bt(bt) = &mut self.backend { - bt.opt_state.seed_dictionary_entropy(huff, ll, ml, of); - } + self.bt.opt_state.seed_dictionary_entropy(huff, ll, ml, of); } pub(crate) fn reset(&mut self) { self.table.reset(); - if let HcBackend::Bt(bt) = &mut self.backend { - bt.reset(); - } + self.bt.reset(); } - /// Backfill positions from the tail of the previous slice that couldn't be - /// hashed at the time (insert_position needs 4 bytes of lookahead). + /// Index the last committed block without searching it. pub(crate) fn skip_matching(&mut self, incompressible_hint: Option) { self.table.skip_matching::(incompressible_hint); } /// Runtime-dispatched entry kept only for in-crate tests. Production - /// callers reach the inner loops through - /// [`Self::start_matching_strategy`] / [`MatchGeneratorDriver::compress_block`] - /// which pick the lazy / optimal arm from `S::USE_BT` at - /// monomorphisation time. + /// callers reach the optimal parser through + /// [`MatchGeneratorDriver::compress_block`], monomorphised per strategy. #[cfg(test)] pub(crate) fn start_matching(&mut self, handle_sequence: impl FnMut(Sequence)) { crate::encoding::fastpath::with_encode_tier!( @@ -1497,13 +1334,15 @@ impl HcMatchGenerator { // Dispatch on the mirrored `strategy_tag` so each test runs // under the same monomorphisation production would pick. // `BtOpt` / `BtUltra` / `BtUltra2` remain distinct here even - // though `table.uses_bt` / `is_btultra2` alone can't separate - // BtOpt from BtUltra. + // though `is_btultra2` alone can't separate BtOpt from BtUltra. match self.strategy_tag { - StrategyTag::Fast | StrategyTag::Dfast | StrategyTag::Greedy | StrategyTag::Lazy => { - self.start_matching_lazy::(&mut handle_sequence) + StrategyTag::Fast + | StrategyTag::Dfast + | StrategyTag::Greedy + | StrategyTag::Lazy + | StrategyTag::Btlazy2 => { + unreachable!("only the optimal strategies run on this backend") } - StrategyTag::Btlazy2 => unreachable!("btlazy2 runs on the lazy (Row) backend"), StrategyTag::BtOpt => { self.start_matching_optimal::(&mut handle_sequence) } @@ -1515,260 +1354,6 @@ impl HcMatchGenerator { } } } - - /// Strategy-aware entry point used by - /// [`MatchGeneratorDriver::compress_block`](crate::encoding::match_generator::MatchGeneratorDriver::compress_block). - /// Branches on `S::USE_BT` — a compile-time `const` — so each - /// monomorphisation keeps exactly one arm: `Lazy` / - /// `Fast` / `Dfast` / `Greedy` see only `start_matching_lazy`, - /// `BtOpt` / `BtUltra` / `BtUltra2` see only - /// `start_matching_optimal`. The inherent test-only - /// `HcMatchGenerator::start_matching` reaches the same arms by - /// runtime-matching on `self.strategy_tag` (the parse-mode field - /// has been removed); production never invokes that path. - pub(crate) fn start_matching_strategy( - &mut self, - handle_sequence: &mut impl FnMut(Sequence), - ) { - debug_assert_eq!( - self.table.uses_bt, - S::USE_BT, - "Strategy::USE_BT disagrees with runtime table.uses_bt at HC dispatch" - ); - if S::USE_BT { - self.start_matching_optimal::(handle_sequence) - } else { - self.start_matching_lazy::(handle_sequence) - } - } - - /// Dispatcher: pick the dict-aware monomorph when a separate dms is primed - /// (attach-mode dictionary), else the no-dict monomorph. Mirrors upstream's - /// compile-time `dictMode` split — the `DICT = false` body carries no dms - /// code at all, so the no-dict hot path is unaffected by the dict search. - pub(crate) fn start_matching_lazy( - &mut self, - handle_sequence: impl FnMut(Sequence), - ) { - if self.table.dms.is_primed() { - self.start_matching_lazy_impl::(handle_sequence); - } else { - self.start_matching_lazy_impl::(handle_sequence); - } - } - - pub(crate) fn start_matching_lazy_impl( - &mut self, - mut handle_sequence: impl FnMut(Sequence), - ) { - self.table.ensure_tables(); - - // `current_block_range()` is borrowed-aware: owned → last committed - // chunk; borrowed → the staged in-place block range. - let (current_abs_start, current_len) = self.table.current_block_range(); - if current_len == 0 { - return; - } - // Full live history (dict + committed blocks + current block), hoisted - // ONCE for the whole position scan and threaded into every - // `find_best_match` / `pick_lazy_match` call. `live_history()` is - // loop-invariant here (the scan mutates the hash/chain tables + - // `offset_hist` but never the history bytes or length), so re-fetching - // it per find — inside `hash_chain_candidate` + the rep probe, plus - // again for each lazy lookahead at pos+1 / pos+2 — was pure - // per-position overhead. Detached as a raw slice so the loop's - // `&mut self.table` inserts coexist with this `&[u8]`; the scan never - // reallocates the history. - let concat: &[u8] = { - let lh = self.table.live_history(); - unsafe { core::slice::from_raw_parts(lh.as_ptr(), lh.len()) } - }; - // Dict-match-state primed flag, hoisted ONCE for the scan: it is - // block-invariant (the dict is primed before the block) and lives on the - // cold `dms` cacheline, so the per-find `dms.is_primed()` load was a - // measurable hot-path cost (~8% of `hash_chain_candidate` on the - // dict-over-random fixture). The `DICT = false` monomorph ignores it. - let dms_primed = self.table.dms.is_primed(); - - let current_abs_end = current_abs_start + current_len; - self.table - .backfill_boundary_positions::(current_abs_start, current_abs_end); - - let mut pos = 0usize; - let mut literals_start = 0usize; - // Lookahead carry (upstream's lazy depth loop searches each position - // ONCE): when the lookahead defers to `abs_pos + 1`, its already-computed - // match is carried here and reused next iteration instead of re-searched. - // `None` means search this position fresh. - let mut carried: Option = None; - // Set when the previous position deferred WITHOUT a carry (depth-2: - // `abs_pos + 2` won but `abs_pos + 1` was cold). If this position then - // misses, the no-match skip below must advance by exactly ONE so it - // cannot hop over the already-proven `abs_pos + 2` winner. - let mut deferred_without_carry = false; - while pos + HC_MIN_MATCH_LEN <= current_len { - let abs_pos = current_abs_start + pos; - let lit_len = pos - literals_start; - let forced_single_step = core::mem::take(&mut deferred_without_carry); - - // Reuse the carried lookahead match (searched WITH the previous - // position already inserted, so its offset-1 candidate is visible) - // or search this position fresh. `find_best_match` returns the - // forward `(offset, length)` in registers (`HcMatch`, 16 bytes). - let best = match carried.take() { - Some(m) => m, - None => self.hc.find_best_match::( - concat, - dms_primed, - &self.table, - abs_pos, - lit_len, - ), - }; - if best.is_match() { - // Insert `abs_pos` BEFORE the lookahead so the `abs_pos + 1` - // probe sees it at offset 1 — upstream inserts the searched - // position during its own search, before the depth loop probes - // the next one. - self.table.insert_position(abs_pos); - if let Some(carry) = self.hc.lazy_decide_carry::( - concat, - dms_primed, - &self.table, - abs_pos, - lit_len, - best, - ) { - // DEFER: the lazy lookahead found a better match one (or two) - // bytes ahead. Advance exactly ONE byte. Reuse the lookahead - // match when it is real (so it is not re-searched — upstream - // searches each position once); the `NONE` sentinel is the - // depth-2 defer-without-carry case (`abs_pos + 2` won but - // `abs_pos + 1` had no match), so search the next fresh and - // pin the following advance to one byte (below) so the skip - // heuristic cannot hop over that `abs_pos + 2` winner. - carried = carry.is_match().then_some(carry); - deferred_without_carry = carried.is_none(); - pos += 1; - continue; - } - // COMMIT `best`. Backward-extend over the literal run (upstream - // `zstd_lazy.c` after rep-vs-chain selection) through the shared - // raw-pointer helper the Row / Dfast probes use — one - // back-extension source, no per-step slice bounds checks. - // `abs_pos` is already inserted above, so the forward span fill - // starts at `abs_pos + 1`. - let ext = crate::encoding::match_table::helpers::extend_backwards_shared( - concat, - self.table.history_abs_start, - abs_pos - best.offset, - abs_pos, - best.match_len, - lit_len, - ); - let match_len = ext.match_len; - self.table - .insert_match_span(abs_pos + 1, ext.start + match_len); - // The backward extension stops at the literal run's start. - let start = ext.start - current_abs_start; - debug_assert!(start >= literals_start); - let literal_len = start - literals_start; - handle_sequence(Sequence::Triple { - literal_len, - offset: best.offset, - match_len, - }); - let _ = encode_offset_with_history( - best.offset as u32, - literal_len as u32, - &mut self.table.offset_hist, - ); - pos = start + match_len; - literals_start = pos; - continue; - } - // No match found. - self.table.insert_position(abs_pos); - // Lazy skipping (upstream zstd `ZSTD_compressBlock_lazy_generic`, - // zstd_lazy.c:1614): advance faster over runs with no match. - // `step = ((ip - anchor) >> kSearchStrength) + 1` with - // kSearchStrength = 8, where `ip - anchor` is the current - // literal-run length. On compressible input the run stays short - // (step == 1, identical to a 1-byte advance); on incompressible - // / dict-over-random input the run grows so the parser skips - // ahead (one search per `step` positions) instead of searching - // every byte. Skipped positions are not inserted, mirroring - // upstream (it inserts only searched positions during a no-match - // run). Ratio follows upstream (not byte-identical). - // A miss one byte after a depth-2 defer-without-carry must advance by - // exactly one so the skip cannot pass the proven `abs_pos + 2` match. - let step = if forced_single_step { - 1 - } else { - ((pos - literals_start) >> 8) + 1 - }; - pos += step; - // No clamp needed before the tail loop: the search bound and the - // hashable bound are both `pos + HC_MIN_MATCH_LEN <= current_len` - // (HC_MIN_MATCH_LEN == 4 == the insert width), so there is no - // non-searchable-but-hashable anchor to miss. Positions the skip - // jumps over inside the searchable region are intentionally not - // inserted — same as upstream zstd, which advances past them via - // the identical `ip += step` and never hashes them either. - } - - // Insert remaining hashable positions in the tail (the matching loop - // stops at HC_MIN_MATCH_LEN but insert_position only needs 4 bytes). - while pos + 4 <= current_len { - self.table.insert_position(current_abs_start + pos); - pos += 1; - } - - if literals_start < current_len { - handle_sequence(Sequence::Literals { - len: current_len - literals_start, - }); - } - } - - /// Register the borrowed input window for the no-copy one-shot path. - /// # Safety - /// `buffer` must outlive the borrowed scans (see `MatchTable`). - pub(crate) unsafe fn set_borrowed_window(&mut self, buffer: &[u8]) { - // SAFETY: forwarded liveness contract. - unsafe { self.table.set_borrowed_window(buffer) }; - } - - pub(crate) fn clear_borrowed_window(&mut self) { - self.table.clear_borrowed_window(); - } - - /// Borrowed (no-copy) equivalent of [`Self::start_matching_lazy`]: stage - /// the in-place block range, then run the same lazy chain parse. The - /// parse reads its range via `current_block_range()` and its bytes via - /// `get_last_space()` / `live_history()`, all borrowed-aware, so the block - /// is scanned in place with the per-position window_low offset cap. - pub(crate) fn start_matching_lazy_borrowed( - &mut self, - block_start: usize, - block_end: usize, - handle_sequence: impl FnMut(Sequence), - ) { - self.table.stage_borrowed_block(block_start, block_end); - self.start_matching_lazy::(handle_sequence); - } - - /// Borrowed (no-copy) equivalent of the lazy `skip_matching`: stage the - /// in-place block, then seed positions without an owned-history append. - pub(crate) fn skip_matching_borrowed( - &mut self, - block_start: usize, - block_end: usize, - incompressible_hint: Option, - ) { - self.table.stage_borrowed_block(block_start, block_end); - self.table.skip_matching::(incompressible_hint); - } } #[cfg(test)] diff --git a/zstd/src/encoding/hc/hc_tests.rs b/zstd/src/encoding/hc/hc_tests.rs deleted file mode 100644 index f51139b6..00000000 --- a/zstd/src/encoding/hc/hc_tests.rs +++ /dev/null @@ -1,154 +0,0 @@ -//! Unit coverage for `HcMatcher` paths the encode-level suite -//! doesn't naturally hit: short-suffix early returns on probe -//! helpers, chain-walk self-loop branch, and the lazy-pick -//! "next match is better" decline paths. -use super::*; -use crate::encoding::fastpath::with_host_tier; -use crate::encoding::match_table::storage::MatchTable; - -fn table_with_history(buf: &[u8]) -> MatchTable { - let mut t = MatchTable::new(buf.len().max(8)); - t.history = buf.to_vec().into(); - t.history_start = 0; - t.history_abs_start = 0; - t.window_size = buf.len(); - t.position_base = 0; - t.hash_log = 8; - t.chain_log = 8; - t.hash3_log = 0; - t.ensure_tables(); - // `history` is set directly above; record one live chunk. - t.chunk_lens.push_back(buf.len()); - t -} - -#[test] -fn repcode_candidate_returns_none_when_suffix_too_short() { - let mut t = table_with_history(b"abc"); - t.offset_hist = [1, 2, 3]; - // current_idx + HC_MIN_MATCH_LEN > concat.len() → early no-match. - assert!(!with_host_tier!(K => HcMatcher::repcode_candidate::(&t, 0, 1)).is_match()); -} - -#[test] -fn repcode_candidate_skips_rep_at_history_boundary() { - // rep=5 but abs_pos=4, so candidate_pos would underflow into - // pre-history bytes; the `rep > abs_pos` guard must skip it. - let mut t = table_with_history(b"abcdefgh"); - t.offset_hist = [5, 6, 7]; - // No match possible at abs_pos=4 because every rep aims past - // history start. - let result = with_host_tier!(K => HcMatcher::repcode_candidate::(&t, 4, 1)); - assert!(!result.is_match(), "no rep can land in-range"); -} - -#[test] -fn find_best_match_returns_none_for_short_suffix() { - let hc = HcMatcher::new(2, 4, 32); - let t = table_with_history(b"abc"); - assert!( - !with_host_tier!(K => hc.find_best_match::(t.live_history(), false, &t, 0, 1)) - .is_match() - ); -} - -/// Forward-length selection (upstream `ZSTD_HcFindBestMatch`): the chain -/// walk keeps the longest FORWARD match (`currentMl > ml`) and applies the -/// single backward extension to THAT winner — a shorter-forward candidate -/// is NOT promoted just because it has more backward (`lit_len`) room. -/// Backward "catch up" happens once, on the forward winner, after the walk; -/// it never changes which candidate wins. -/// -/// Fixture (40 bytes): `"AAAabcdefZMQabcdefIJBAAAabcdefIJKKKKKKKK"`. -/// Probing `abs_pos = 24`: the 4-byte hash at `idx 24` ("abcd") collides -/// with `idx 3` and `idx 12`, so the walk visits `[12, 3]` (LIFO). -/// - `idx 12`: forward 8 (`"abcdefIJ"`), `concat[11] = 'Q'` != -/// `concat[23] = 'A'` so no backward room. Total 8, offset 12. -/// - `idx 3`: forward only 6 (`"abcdef"`), but `concat[0..3] = "AAA"` == -/// `concat[21..24]` so it could backward-extend 3 to a TOTAL of 9 — yet -/// its forward length (6) loses to candidate 12's (8), so it is never -/// selected. The forward winner (12) wins at both `lit_len`s. -#[test] -fn hash_chain_candidate_picks_longest_forward_over_shorter_with_backward_room() { - let mut t = MatchTable::new(64); - t.history = b"AAAabcdefZMQabcdefIJBAAAabcdefIJKKKKKKKK".to_vec().into(); - t.history_start = 0; - t.history_abs_start = 0; - t.window_size = t.history.len(); - t.position_base = 0; - t.hash_log = 8; - t.chain_log = 8; - t.hash3_log = 0; - t.ensure_tables(); - t.chunk_lens.push_back(t.history.len()); - t.insert_positions(0, 24); - - let hc = HcMatcher::new(2, 16, 64); - - // The forward winner (idx 12, forward 8) has no backward room. - let c0 = - with_host_tier!(K => hc.hash_chain_candidate::(t.live_history(), false, &t, 24)); - assert!(c0.is_match(), "forward match must be found"); - assert_eq!(c0.match_len, 8, "longest forward match is 8 (idx 12)"); - assert_eq!( - c0.offset, 12, - "winner is the forward-8 candidate at offset 12" - ); -} - -/// Forward-length ties keep the FIRST-visited candidate (upstream -/// `ZSTD_HcFindBestMatch` uses `currentMl > ml`, strictly-longer, so an -/// equal-length later candidate never displaces the earlier one). The walk -/// is newest-first, so in organic chains "first visited" is the closest -/// (smallest-offset) position anyway; this test hand-wires the chain into a -/// non-monotonic order to pin the tie-break rule itself. -/// -/// Fixture: four 8-byte `"abcdefgh"` chunks at `0 / 9 / 18 / 27`, each -/// followed by a unique terminator (`'A'/'B'/'C'/'D'`) capping cross-chunk -/// forward matches at exactly 8. Probing `abs_pos = 27` with the chain -/// hand-wired to visit pos 9 (offset 18) THEN pos 18 (offset 9): both have -/// forward length 8, so the first-visited (pos 9, offset 18) wins. The -/// self-tightening gate at `ml = 8` also rejects pos 18 (its tail byte is a -/// different chunk terminator), so the equal-length later candidate is -/// skipped before the count — consistent with ties-keep-first. -#[test] -fn hash_chain_candidate_forward_ties_keep_first_visited() { - let mut t = MatchTable::new(64); - t.history = b"abcdefghAabcdefghBabcdefghCabcdefghDZZZZ".to_vec().into(); - assert_eq!(t.history.len(), 40); - t.history_start = 0; - t.history_abs_start = 0; - t.window_size = t.history.len(); - t.position_base = 0; - t.hash_log = 8; - t.chain_log = 8; - t.hash3_log = 0; - t.ensure_tables(); - t.chunk_lens.push_back(t.history.len()); - - let abs_pos = 27usize; - let concat = t.live_history(); - let probe_hash = t.hash_position(&concat[abs_pos..]); - // Hand-wire the chain head + link so the walk surfaces pos 9 first - // (offset 18) then pos 18 (offset 9). `stored = pos + 1`. - t.hash_table_mut()[probe_hash] = 9 + 1; - let chain_mask = (1usize << t.chain_log) - 1; - t.chain_table_mut()[9 & chain_mask] = 18 + 1; - t.chain_table_mut()[18 & chain_mask] = HC_EMPTY; - - let hc = HcMatcher::new(2, 16, 64); - let cand = with_host_tier!( - K => hc.hash_chain_candidate::(t.live_history(), false, &t, abs_pos) - ); - assert!(cand.is_match(), "walk must still produce a match"); - assert_eq!( - cand.match_len, 8, - "both candidates have an 8-byte forward prefix" - ); - assert_eq!( - cand.offset, 18, - "forward-length ties keep the first-visited candidate (pos 9, \ - offset 18); a value of 9 would mean the equal-length later \ - candidate displaced it (non-upstream gain-based tie-break)" - ); -} diff --git a/zstd/src/encoding/hc/mod.rs b/zstd/src/encoding/hc/mod.rs index 8376cde1..086b947c 100644 --- a/zstd/src/encoding/hc/mod.rs +++ b/zstd/src/encoding/hc/mod.rs @@ -1,17 +1,11 @@ -//! Hash-chain match finder used by `Lazy2`. +//! Binary-tree match generator and optimal parser of the btopt / btultra / +//! btultra2 strategies. //! -//! Hosts the runtime knobs of the lazy parser — the lookahead depth -//! (`lazy_depth`), the chain-walk search budget (`search_depth`), and -//! the "sufficient match length" threshold (`target_len`). Method -//! bodies (chain walk, `insert_position`, `pick_lazy_match`, -//! `start_matching_lazy`) still live on `HcMatchGenerator` and will -//! move onto `impl HcMatcher` in Stage 2b alongside the -//! `&mut MatchTable` thread-through; this stage establishes the -//! ownership boundary so the BT-side extraction in Stage 3 has a -//! clean counterpart type to mirror. +//! [`HcMatcher`] carries the search knobs the tree walk and the optimal +//! parser read; the tables live on the shared +//! [`super::match_table::storage::MatchTable`]. //! -//! Upstream zstd parity reference: `lib/compress/zstd_lazy.c`, -//! `ZSTD_HcFindBestMatch` / `ZSTD_compressBlock_lazy2_generic`. +//! Upstream zstd parity reference: `lib/compress/zstd_opt.c`. #![allow(dead_code)] @@ -20,49 +14,12 @@ pub(crate) mod optimal; pub(crate) mod priceset; use super::cost_model::{HC_FORMAT_MINMATCH, HC_OPT_NUM, HcOptimalCostProfile}; -use super::fastpath::EncodeTier; -use super::match_table::helpers::common_prefix_len; -use super::match_table::storage::{HC_EMPTY, MatchTable}; -use super::opt::types::MatchCandidate; -/// Minimum match length emitted by the lazy / lazy2 chain walker. -/// Upstream zstd parity: `MIN_MATCH` in `lib/compress/zstd_lazy.c`. +/// Key width the match tables hash a position on. Upstream zstd parity: +/// `MIN_MATCH` in `lib/compress/zstd_lazy.c`. pub(crate) const HC_MIN_MATCH_LEN: usize = 4; -/// Forward-only match found by the lazy chain walk: the `(offset, length)` pair -/// upstream's `ZSTD_HcFindBestMatch` returns (`size_t matchLength` in `rax` + -/// the offset through `offsetPtr`). 16 bytes, so the find result + the lazy -/// lookahead arg travel in registers (`rax:rdx`) instead of the 24-byte -/// `MatchCandidate` (`start` included) the old find returned by value, which -/// the System V ABI spills to the stack and the lazy loop then copies. The -/// backward extension that produces `start` is deferred to the commit site -/// (upstream does it once in the lazy loop after rep-vs-chain selection), so it -/// never touches the per-position find/compare path. -#[derive(Clone, Copy)] -pub(crate) struct HcMatch { - /// Match offset (`abs_pos - candidate_pos`). Meaningful only when - /// [`HcMatch::is_match`] is true. - pub(crate) offset: usize, - /// Forward match length, or `< HC_MIN_MATCH_LEN` for "no match". - pub(crate) match_len: usize, -} - -impl HcMatch { - /// The "no match found" sentinel (`match_len = 0`). - pub(crate) const NONE: Self = Self { - offset: 0, - match_len: 0, - }; - - /// A real match reaches at least `HC_MIN_MATCH_LEN`; shorter is the seed / - /// sentinel, not an emittable match. - #[inline] - pub(crate) fn is_match(self) -> bool { - self.match_len >= HC_MIN_MATCH_LEN - } -} - -/// Hard cap on chain-walk depth. +/// Hard cap on tree-walk depth. pub(crate) const MAX_HC_SEARCH_DEPTH: usize = 512; /// Capacity the optimal parser reserves for its per-position candidate buffer: @@ -82,889 +39,27 @@ const _: () = { assert!(Btlazy2::MAX_CHAIN_DEPTH <= MAX_HC_SEARCH_DEPTH); }; -/// Hash-chain matcher state used by the `Lazy2` parse mode (and the -/// short-history fast path of the BT cascade's initial pass). -/// -/// Owns only the per-frame *configuration* — the actual chain / hash -/// tables live on the shared -/// [`super::match_table::storage::MatchTable`] that this matcher -/// borrows when it runs. +/// The per-frame search configuration of the tree walk. #[derive(Clone)] pub(crate) struct HcMatcher { - /// Lookahead depth (1 = lazy, 2 = lazy2). Upstream zstd parity: - /// `params->cParams.strategy >= ZSTD_lazy2`. - pub(crate) lazy_depth: u8, - /// Maximum number of chain entries inspected per `find_best_match` - /// call. Upstream zstd parity: `params->cParams.searchLog` (clamped to - /// [`MAX_HC_SEARCH_DEPTH`] for HC mode; BT modes use the unclamped value as their walk - /// budget). + /// Tree compares per position. Upstream zstd parity: + /// `1 << params->cParams.searchLog`. pub(crate) search_depth: usize, - /// "Sufficient" match length — once a candidate reaches this - /// length, the lazy decision short-circuits without checking the - /// next position. Upstream zstd parity: + /// "Sufficient" match length that ends a search. Upstream zstd parity: /// `params->cParams.targetLength`. pub(crate) target_len: usize, } impl HcMatcher { - pub(crate) fn new(lazy_depth: u8, search_depth: usize, target_len: usize) -> Self { + pub(crate) fn new(search_depth: usize, target_len: usize) -> Self { Self { - lazy_depth, search_depth, target_len, } } - /// Upstream zstd "match gain" heuristic: `match_len * 4 - offset_bits`. - /// The lazy lookahead uses this to compare a candidate at the - /// Pick the better of two candidate matches by LENGTH (ties to the smaller - /// offset), the same selector the chain walk and the shared Row/Dfast - /// repcode probe use. Upstream `ZSTD_compressBlock_lazy_generic` compares - /// the repcode against the searched match by length at depth 0 (`ml2 > - /// matchLength`, the repcode keeps ties); a longer or equal-length-closer - /// match wins. `None` arms pass the surviving `Some` through. - pub(crate) fn better_candidate(lhs: HcMatch, rhs: HcMatch) -> HcMatch { - if !lhs.is_match() { - return rhs; - } - if !rhs.is_match() { - return lhs; - } - if rhs.match_len > lhs.match_len - || (rhs.match_len == lhs.match_len && rhs.offset < lhs.offset) - { - rhs - } else { - lhs - } - } - - /// Probe the 3 rep-code offsets (with the upstream zstd `ll0 ↦ rep[0] − 1` - /// fallback) and return the best in-range match. Pure helper — - /// only reads from `MatchTable`, no HcMatcher state needed. - pub(crate) fn repcode_candidate( - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - ) -> HcMatch { - let reps = if lit_len == 0 { - [ - Some(table.offset_hist[1] as usize), - Some(table.offset_hist[2] as usize), - (table.offset_hist[0] > 1).then_some((table.offset_hist[0] - 1) as usize), - ] - } else { - [ - Some(table.offset_hist[0] as usize), - Some(table.offset_hist[1] as usize), - Some(table.offset_hist[2] as usize), - ] - }; - - let concat = table.live_history(); - let current_idx = abs_pos - table.history_abs_start; - if current_idx + HC_MIN_MATCH_LEN > concat.len() { - return HcMatch::NONE; - } - // Raw base pointer for the upstream zstd-style `MEM_read32` 4-byte rep - // gate below (single unaligned load each, no per-rep slice bounds check). - let base_ptr = concat.as_ptr(); - - let mut best = HcMatch::NONE; - for rep in reps.into_iter().flatten() { - if rep == 0 || rep > abs_pos { - continue; - } - let candidate_pos = abs_pos - rep; - if candidate_pos - < table - .history_abs_start - .max(abs_pos.saturating_sub(table.max_window_size)) - { - continue; - } - let candidate_idx = candidate_pos - table.history_abs_start; - // Cheap 4-byte equality gate before the wider SIMD count (upstream - // zstd `MEM_read32` repcode gate in `ZSTD_compressBlock_lazy`). - // `HC_MIN_MATCH_LEN == 4` and `current_idx + 4 <= len` (the early - // return above), so a first-4-byte mismatch can never reach the - // match floor — reject without the vector load+count. Falls through - // to the full count only when the candidate lacks a 4-byte lookahead - // (short tail), so the accepted set is byte-identical to the - // unconditional count. Mirrors the chain-walk gate. - if candidate_idx + 4 <= concat.len() - && unsafe { - MatchTable::read_le_u32_ptr(base_ptr.add(candidate_idx)) - != MatchTable::read_le_u32_ptr(base_ptr.add(current_idx)) - } - { - continue; - } - let match_len = - common_prefix_len::(&concat[candidate_idx..], &concat[current_idx..]); - if match_len >= HC_MIN_MATCH_LEN { - best = Self::better_candidate( - best, - HcMatch { - offset: rep, - match_len, - }, - ); - } - } - best - } - - /// Best hash-chain match at `abs_pos`, in upstream zstd's - /// `ZSTD_HcFindBestMatch` forward-length model: walk the live chain (and - /// the dms) tracking the longest FORWARD match (`currentMl > ml`, ties - /// keep the closest), gate each candidate on the self-tightening 4-byte - /// tail probe, and extend the single winner backwards over the literal - /// run ONCE at the end. No per-candidate `Option` / gain merge / backward - /// extension — that per-attempt overhead is what the old gain-based walk - /// paid over C. - /// - /// `#[inline(always)]`: folded INTO the out-of-line `find_best_match` so the - /// whole find (rep + chain + dms + selection) is ONE function — upstream's - /// `ZSTD_HcFindBestMatch` shape — reached by a single call per position, not - /// the find->chain->dms call chain that an out-of-line chain walk adds. - #[inline(always)] - pub(crate) fn hash_chain_candidate( - &self, - concat: &[u8], - dms_primed: bool, - table: &MatchTable, - abs_pos: usize, - ) -> HcMatch { - let current_idx = abs_pos - table.history_abs_start; - let history_tail = concat.len(); - if current_idx + HC_MIN_MATCH_LEN > history_tail { - return HcMatch::NONE; - } - - // Chain walk is inlined below — avoids the per-call 4 KiB - // `[usize; MAX_HC_SEARCH_DEPTH]` array that - // [`Self::chain_candidates`] materializes (zero-init on entry, - // memcpy on return). At lazy_depth=2 (L7+) `pick_lazy_match` - // triggers up to three chain walks per committed position, so - // the array form costs ~12 KiB of stack traffic per accepted - // match. Upstream zstd (`zstd_lazy.c` `ZSTD_HcFindBestMatch`) runs a - // single fused loop with no intermediate buffer; mirror that. - // - // `chain_candidates` itself is still alive — the chain-walk - // unit tests drive it directly, and the BT-optimal HC - // candidate collector in `match_generator.rs` consumes it - // through a macro pipeline that inherits the array form. - // Inlining the array out of that BT-optimal callsite is a - // separate, larger refactor; this commit only addresses the - // lazy hot path. - let hash = table.hash_position(&concat[current_idx..]); - let chain_mask = (1usize << table.chain_log) - 1; - let mut cur = table.hash_table()[hash]; - // Cap loop at MAX_HC_SEARCH_DEPTH so a misconfigured - // `search_depth > MAX_HC_SEARCH_DEPTH` (BT modes set it from the - // upstream zstd config, which can exceed our cap) cannot run forever. - let max_chain_steps = self.search_depth.min(MAX_HC_SEARCH_DEPTH); - let mut steps = 0usize; - let history_abs_start = table.history_abs_start; - - // Forward-length match state (upstream `ZSTD_HcFindBestMatch`): `ml` is - // the FORWARD match length, seeded at `MIN_MATCH - 1` so the first - // candidate's self-tightening gate (`match + ml - 3`) degenerates to a - // first-4-byte compare. `best_idx` is the winning candidate's concat - // index. - // - // `found` is tracked ONLY in the `DICT = false` monomorph. `ml >= - // MIN_MATCH` is always exactly equivalent to "a match was found" (ml - // only updates via `current_ml > ml`, whose first hit is >= MIN_MATCH), - // so the `DICT = true` path uses that equivalence and drops the bool — - // the dict monomorph is register-tight (it also runs the dms walk), and - // eliding `found` there frees a stack slot (measured: -5% cycles on the - // dict path). The `DICT = false` monomorph is NOT register-tight, and - // there the equivalent `ml < MIN_MATCH` final check changed codegen - // layout enough to cost ~+4% (same instruction count, worse IPC), so it - // keeps the original bool. `DICT` is const, so each monomorph compiles - // exactly one arm with no runtime branch; in the dict monomorph `found` - // is written/read only in dead `!DICT` arms and is fully eliminated. - let mut ml = HC_MIN_MATCH_LEN - 1; - let mut best_idx = 0usize; - let mut found = false; - // Raw base pointer for the upstream zstd-style `MEM_read32` gate - // (single unaligned load each, no per-candidate slice bounds check). - let base_ptr = concat.as_ptr(); - // Loop-invariant precompute, hoisted out of the chain walk (upstream zstd: - // `lowLimit` + base pointers are set once before the loop). Candidates - // are tracked in history-RELATIVE index space so each step is a single - // `add` + range check + 4-byte gate, mirroring `ZSTD_HcFindBestMatch`'s - // tight body (`matchIndex >= lowLimit`; `NEXT_IN_CHAIN`) instead of an - // `Option`-returning absolute-position decode plus a per-candidate - // `.max()/.saturating_sub()` window-floor recompute. - let floor_idx = current_idx.saturating_sub(table.max_window_size); - // `candidate_idx = position_base + (cur-1) - index_shift - - // history_abs_start`, folded into one wrapping bias so the per-step cost - // is `cur + idx_bias`. Stale / sub-floor chain entries wrap to a huge - // index and are rejected by the `< current_idx` upper bound — the - // accepted set is identical to the `stored_abs_position_fast` decode - // (which returned `None` for the same sub-floor entries). - let idx_bias = table - .position_base - .wrapping_sub(1) - .wrapping_sub(table.index_shift) - .wrapping_sub(history_abs_start); - // Hoist the chain-table base pointer ONCE: through `&MatchTable` the - // optimizer reloads the `Vec` (ptr, len) header — and re-checks bounds — - // on every `table.chain_table[..]`, a measured ~1.7x of upstream's - // per-find load count. `candidate_rel & chain_mask` is always - // `< chain_table.len()` (the table is `1 << chain_log` and - // `chain_mask == len - 1`), so the raw read is in-bounds. - let chain_ptr: *const u32 = table.chain_table().as_ptr(); - while steps < max_chain_steps { - if cur == HC_EMPTY { - break; - } - let candidate_rel = cur.wrapping_sub(1) as usize; - // SAFETY: `candidate_rel & chain_mask < chain_table.len()`. - let next = unsafe { *chain_ptr.add(candidate_rel & chain_mask) }; - steps += 1; - - let candidate_idx = (cur as usize).wrapping_add(idx_bias); - // A wrapped index (`>= current_idx`) means `abs < history_abs_start`: - // a stale entry from a previous frame. The chain is head-insert - // monotonic, so the rest of the tail is older and also stale -- stop. - // This is the early-exit the no-memset floor-advance reset needs; in - // the memset reset the chain has no wrapped entries, so it never - // fires. INVARIANT: the memset reset zeroes every slot to `HC_EMPTY` - // (caught by the `cur == HC_EMPTY` break above) and uses `idx_bias` - // that maps stored positions back to indices `< current_idx`, so no - // surviving entry can wrap -- only the floor-advance path retains - // cross-frame entries that decode past `current_idx`. If a future - // change let a memset-reset chain produce a wrap, this break would - // silently cut the walk short (a miss-rate regression, not a - // correctness bug). Below-floor (`< floor_idx`) entries are NOT a - // break: a chain-slot collision can put an in-window entry after one. - if candidate_idx >= current_idx { - break; - } - if candidate_idx >= floor_idx { - // Upstream zstd's single self-tightening gate (`zstd_lazy.c:714`): - // MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3) - // proves the candidate can possibly reach `ml + 1` before paying - // for the full `common_prefix_len`. `ml >= MIN_MATCH - 1 = 3` so - // `gate_off >= 0`. The iLimit break below keeps - // `current_idx + ml < history_tail` on entry, so - // `current_idx + gate_off + 4 = current_idx + ml + 1 <= history_tail`, - // and `candidate_idx < current_idx` keeps the match-side read in - // range too -- no per-iteration bounds check (matches C's - // branchless gate). - let gate_off = ml - 3; - // SAFETY: both reads are `<= history_tail` per the bound above. - let gate_ok = unsafe { - MatchTable::read_le_u32_ptr(base_ptr.add(candidate_idx + gate_off)) - == MatchTable::read_le_u32_ptr(base_ptr.add(current_idx + gate_off)) - }; - if gate_ok { - let current_ml = - common_prefix_len::(&concat[candidate_idx..], &concat[current_idx..]); - // `currentMl > ml`: strictly longer forward match wins; ties - // keep the first (closest, the walk is newest-first). - if current_ml > ml { - ml = current_ml; - best_idx = candidate_idx; - // Dict path reads `ml >= MIN_MATCH`; only the no-dict - // monomorph keeps the `found` flag (see seed comment). - if !DICT { - found = true; - } - // Upstream `if (ip + currentMl == iLimit) break`: reached - // the input end (best possible); also keeps the next gate - // read in bounds. - if current_idx + ml >= history_tail { - break; - } - } - } - } - - // Self-loop (two positions share `candidate_rel & chain_mask`): - // checked here where `next` / `cur` are already live, not carried as - // a bool across the body (a register in this saturated walk). - if next == cur { - break; - } - cur = next; - } - - // Separate dictionary match state (upstream `ZSTD_dictMatchState`). The - // walk is OUT-OF-LINE so the dict-only code never bloats this hot - // function on no-dict frames. It shares `ml` / `steps` (upstream's - // single `nbAttempts` across the live + dms loops). Skip it when the - // live walk already reached iLimit (`ml` maximal, and the dms - // self-tightening gate would read past `history_tail`). - // `dms_primed` is hoisted by the caller (block-invariant — the dict is - // primed before the block scan), replacing a per-find `dms.is_primed()` - // load from the cold `dms` cacheline. - if DICT && dms_primed && current_idx + ml < history_tail { - // dms runs only in the `DICT = true` monomorph, which uses the - // `ml >= MIN_MATCH` match test, so the walk tracks `(ml, best_idx)` - // and never the `found` flag. - let walk = self.dms_chain_walk::( - table, - concat, - current_idx, - history_tail, - base_ptr, - max_chain_steps, - steps, - ml, - best_idx, - ); - ml = walk.0; - best_idx = walk.1; - } - - // Dict monomorph: `ml >= MIN_MATCH` is the match test (drops `found`). - // No-dict monomorph: keep the original `found` flag (see seed comment). - let matched = if DICT { ml >= HC_MIN_MATCH_LEN } else { found }; - if !matched { - return HcMatch::NONE; - } - // Forward `(offset, length)` only — the backward extension that produces - // `start` runs once at the commit site (upstream's lazy loop), off the - // per-position find/compare path. `offset = abs_pos - (best_idx + - // history_abs_start) = current_idx - best_idx`. - HcMatch { - offset: current_idx - best_idx, - match_len: ml, - } - } - - /// Out-of-line dms HC4 walk (upstream `ZSTD_HcFindBestMatch` dms loop, - /// `zstd_lazy.c:751-769`). Split from [`Self::hash_chain_candidate`] so the - /// no-dict hot path keeps its small body / register budget. Shares the - /// caller's forward-length state (`ml` / `best_idx`) and `steps` - /// budget -- the live + dms loops are one bounded operation (upstream's - /// single `nbAttempts`). The dict sits at the front of `concat` - /// (`[0, region)`); a dms candidate is a concat index `< current_idx`, so - /// the offset / gate / count logic matches the live walk. Only the - /// `DICT = true` monomorph calls this, and it uses the `ml >= MIN_MATCH` - /// match test, so the walk returns `(ml, best_idx)` and tracks no `found` - /// flag. - /// - /// `#[inline]`: the `DICT = false` monomorph never references this (the - /// `if DICT &&` gate is a compile-time false), so it is dropped from the - /// no-dict path; the `DICT = true` monomorph inlines it into the dict hot - /// path. - #[inline] - #[allow(clippy::too_many_arguments)] - fn dms_chain_walk( - &self, - table: &MatchTable, - concat: &[u8], - current_idx: usize, - history_tail: usize, - base_ptr: *const u8, - max_chain_steps: usize, - mut steps: usize, - mut ml: usize, - mut best_idx: usize, - ) -> (usize, usize) { - let dms = match table.dms.table() { - Some(d) => d, - None => return (ml, best_idx), - }; - // Match upstream's effective dict search depth: the CDict path runs the - // dedicated dict search deeper than the bare level searchLog (measured: - // upstream needs nbAttempts >= 16 to surface the long dict match on the - // per-label-dict fixtures). Floor the shared budget at 16. - // - // NOTE on the per-find step budget: `steps` is SHARED with the live - // walk. The floor below is `max(max_chain_steps, 16)`, so the live and - // dms walks TOGETHER are bounded by it (= 16 at every standard HC level, - // where `max_chain_steps < 16`: L6 `searchLog = 3` -> 8); the dms - // consumes whatever the live walk left. Giving the dms a fresh 16 - // attempts on top of the live walk instead (a per-walk, non-shared - // budget) was measured +3% cycles on the small-10k-random dict path - // with zero ratio change -- the shared floor already surfaces every - // dict match that reaches the output, so sharing it is intentional: the - // floor (not `max_chain_steps`) is the binding dict-search constraint. - let dms_budget = max_chain_steps.max(16); - let dms_hash = MatchTable::hash_position_at(concat, current_idx, dms.hash_log, dms.mls); - let mut dcur = dms.hash_table[dms_hash]; - // Hoist the dms chain-table base pointer (same Vec-header-reload removal - // as the live walk above). - let dms_chain_ptr: *const u32 = dms.chain_table.as_ptr(); - // Reachability floor, hoisted out of the loop. A dict position is - // decoder-reachable iff its offset (`current_idx - dict_idx`) does not - // exceed `max_window_size`, i.e. `dict_idx >= current_idx - - // max_window_size`. Comparing `dict_idx` against this precomputed floor - // drops the per-iteration `current_idx - dict_idx` subtract from the hot - // path (the offset is only needed when a candidate WINS, which is rare on - // the walk) and frees the `max_window_size` register inside the loop. - // `saturating_sub` is the intended business logic, not an overflow guard: - // when `current_idx <= max_window_size` the whole live history fits the - // window, so every dict position is reachable and the floor is 0. Mirrors - // C's `ZSTD_HcFindBestMatch` dms loop, which bounds reachability by chain - // construction rather than re-checking each candidate. - let dict_reachable_floor = current_idx.saturating_sub(table.max_window_size); - while steps < dms_budget { - if dcur == 0 { - break; - } - // Dict position is a concat index in `[0, region)`; the dict is at - // the front so `dict_idx < current_idx` always (and - // `< dms.chain_table.len()`). - let dict_idx = (dcur - 1) as usize; - // SAFETY: `dict_idx` is a stored dict position `< chain_table.len()`. - let dnext = unsafe { *dms_chain_ptr.add(dict_idx) }; - steps += 1; - // Out-of-window dict positions are unreachable to the decoder. - if dict_idx >= dict_reachable_floor { - // Same self-tightening gate as the live walk; the caller's guard - // (`current_idx + ml < history_tail`) + the iLimit break keep - // `current_idx + (ml - 3) + 4 <= history_tail`, and - // `dict_idx < current_idx` keeps the dict-side read in range. - let gate_off = ml - 3; - // SAFETY: both reads are `<= history_tail` per the bound above. - let gate_ok = unsafe { - MatchTable::read_le_u32_ptr(base_ptr.add(dict_idx + gate_off)) - == MatchTable::read_le_u32_ptr(base_ptr.add(current_idx + gate_off)) - }; - if gate_ok { - let current_ml = - common_prefix_len::(&concat[dict_idx..], &concat[current_idx..]); - if current_ml > ml { - ml = current_ml; - best_idx = dict_idx; - if current_idx + ml >= history_tail { - break; - } - } - } - } - // Chain links are strictly decreasing (head insert); `dnext == dcur` - // (or the `dcur == 0` at the top) ends the walk. - if dnext == dcur { - break; - } - dcur = dnext; - } - (ml, best_idx) - } - - /// Single-rep (offset_1) probe — the lazy parser's inline rep check - /// (upstream `ZSTD_compressBlock_lazy_generic`: only `rep[0]` is tested per - /// position, the 3-rep rotation is the optimal parser's). Caller guarantees - /// `lit_len > 0`, so `rep[0]` is repcode index 0 with no ll0 rotation, and - /// extending it backwards over the literal run is sound. Strips - /// [`Self::repcode_candidate`] to its first rep: one window check, one - /// 4-byte gate, one count — a third of the per-position rep cost the 3-rep - /// probe paid (and which the lazy lookahead paid again at pos+1 / pos+2). - /// - /// `#[inline(always)]`: upstream checks the rep INLINE in the lazy loop; - /// folded into the (out-of-line) `find_best_match` monolith so the rep is - /// part of the single per-position find call, not a nested out-of-line probe. - #[inline(always)] - pub(crate) fn repcode_candidate_offset1( - concat: &[u8], - table: &MatchTable, - abs_pos: usize, - ) -> HcMatch { - let rep = table.offset_hist[0] as usize; - if rep == 0 || rep > abs_pos { - return HcMatch::NONE; - } - let current_idx = abs_pos - table.history_abs_start; - if current_idx + HC_MIN_MATCH_LEN > concat.len() { - return HcMatch::NONE; - } - let candidate_pos = abs_pos - rep; - if candidate_pos - < table - .history_abs_start - .max(abs_pos.saturating_sub(table.max_window_size)) - { - return HcMatch::NONE; - } - let candidate_idx = candidate_pos - table.history_abs_start; - // Cheap 4-byte equality gate before the SIMD count (upstream `MEM_read32` - // repcode gate); identical to the per-rep gate in `repcode_candidate`. - let base_ptr = concat.as_ptr(); - if candidate_idx + 4 <= concat.len() - && unsafe { - MatchTable::read_le_u32_ptr(base_ptr.add(candidate_idx)) - != MatchTable::read_le_u32_ptr(base_ptr.add(current_idx)) - } - { - return HcMatch::NONE; - } - let match_len = common_prefix_len::(&concat[candidate_idx..], &concat[current_idx..]); - if match_len >= HC_MIN_MATCH_LEN { - HcMatch { - offset: rep, - match_len, - } - } else { - HcMatch::NONE - } - } - - /// Combine the rep-code and chain-walk candidates and pick the - /// better of the two. Monomorphised over `DICT` (upstream's compile-time - /// `dictMode` template param): the `DICT = false` instance compiles WITHOUT - /// any dms code so the no-dict hot path keeps its tight body, while the - /// `DICT = true` instance dual-probes the separate dictMatchState. The - /// dispatcher in `start_matching_lazy` picks the instance from whether a dms - /// is primed. - /// - /// `#[inline(never)]`: kept out-of-line (upstream's `ZSTD_HcFindBestMatch` - /// is its own function, with the lazy loop thin) so the find's register - /// pressure is contained in its own frame instead of competing with the - /// per-position loop state; the 16-byte `HcMatch` return travels in - /// `rax:rdx`, so the find->loop boundary stays a register pair, not a stack - /// spill. - #[inline(never)] - pub(crate) fn find_best_match( - &self, - concat: &[u8], - dms_primed: bool, - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - ) -> HcMatch { - // `concat` is the full live history (dict + committed blocks + current - // block), hoisted ONCE per block by the caller — `live_history()` is - // loop-invariant for the position scan, so fetching it per find (in - // `hash_chain_candidate` + the rep probe) was pure per-position - // overhead. See `start_matching_lazy`'s block-level hoist. - // - // Upstream `ZSTD_compressBlock_lazy_generic` checks only offset_1 - // (`rep[0]`) inline per position; the 3-rep rotation lives in the - // optimal parser, not the lazy loop. For `lit_len > 0` that is exactly - // `rep[0]` (repcode index 0, no ll0 rotation), so probe the single rep. - // The `lit_len == 0` case keeps the full rotated probe for encode - // correctness (the ll0 `rep[0]-1` rule). - let rep = if lit_len == 0 { - Self::repcode_candidate::(table, abs_pos, lit_len) - } else { - Self::repcode_candidate_offset1::(concat, table, abs_pos) - }; - let hash = self.hash_chain_candidate::(concat, dms_primed, table, abs_pos); - Self::better_candidate(rep, hash) - } - - /// Upstream zstd `lazy` / `lazy2` lookahead: evaluate the match a byte - /// (and optionally two) ahead before committing the current one. - /// Returns `true` if the current match `best` wins (commit it), `false` - /// if the caller should defer to a later position. - /// - /// Takes the already-unwrapped `best` BY VALUE and returns the lazy - /// commit/defer decision: `None` = COMMIT `best`; `Some(carry)` = DEFER and - /// advance one byte. The caller reuses `carry` as the next `best` when it is - /// a real match (`carry.is_match()`), or searches the next position fresh - /// when it is the `NONE` sentinel — the depth-2 defer-WITHOUT-carry case - /// (the `abs_pos + 2` probe wins but `abs_pos + 1` had no match). That case - /// must DEFER, not commit, or lazy2 misses the two-ahead match. - /// - /// The caller inserts `abs_pos` into the hash tables BEFORE calling this, so - /// the `pos + 1` / `pos + 2` lookahead sees `abs_pos` at offset 1 — matching - /// upstream, where the searched position is inserted during its own search, - /// before the depth loop probes the next one. - pub(crate) fn lazy_decide_carry( - &self, - concat: &[u8], - dms_primed: bool, - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - best: HcMatch, - ) -> Option { - let history_end = table.history_abs_start + concat.len(); - // HC's finder is out-of-line by design (`#[inline(never)]`), so the - // shared decision macro expands here with no register cost. Map the - // macro's `Option>` to one `Option` level: `None` - // (commit) stays `None`; `Some(Some(carry))` (defer with a reusable - // lookahead) -> `Some(carry)`; `Some(None)` (depth-2 defer with no - // carryable match one byte ahead) -> `Some(NONE)` so the caller still - // DEFERS but searches the next position fresh. - match crate::encoding::lazy_parse::lazy_decide!( - best_len = best.match_len, - best_off = best.offset, - target_len = self.target_len, - lazy_depth = self.lazy_depth, - abs_pos = abs_pos, - lit_len = lit_len, - history_end = history_end, - min_match = HC_MIN_MATCH_LEN, - search = |pos, ll| { - let m = self.find_best_match::(concat, dms_primed, table, pos, ll); - m.is_match().then_some(m) - }, - ) { - ::core::option::Option::None => ::core::option::Option::None, - ::core::option::Option::Some(::core::option::Option::Some(carry)) => Some(carry), - ::core::option::Option::Some(::core::option::Option::None) => Some(HcMatch::NONE), - } - } - - /// Cross-platform dispatcher for the rep-code probe used by the - /// optimal-parser pipeline. Routes to the kernel-specific variant - /// so the per-rep `common_prefix_len_ptr` call inlines under the - /// callee's `target_feature` umbrella. Test / external callers - /// only — the on-encode hot path bypasses this dispatcher via the - /// kernel-specific variants invoked from inside - /// `collect_optimal_candidates_initialized_`. - #[allow(dead_code)] - #[inline(always)] - #[allow(clippy::too_many_arguments)] - pub(crate) fn for_each_repcode_candidate_with_reps( - &self, - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - reps: [u32; 3], - current_abs_end: usize, - min_match_len: usize, - f: impl FnMut(MatchCandidate), - ) { - // SAFETY: each branch verifies the target_feature requirement of - // the callee (same shape as the BT walk dispatchers). - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - unsafe { - self.for_each_repcode_candidate_with_reps_neon( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - ) - } - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - use crate::encoding::fastpath::FastpathKernel; - crate::encoding::fastpath::tier_match!(K { - #[cfg(feature = "kernel-avx2")] - FastpathKernel::Avx2Bmi2 => unsafe { - self.for_each_repcode_candidate_with_reps_avx2_bmi2( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - ) - }, - #[cfg(feature = "kernel-sse")] - FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { - self.for_each_repcode_candidate_with_reps_sse2( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - ) - }, - FastpathKernel::Scalar => self.for_each_repcode_candidate_with_reps_scalar( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - ), - }) - } - #[cfg(not(any( - all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ), - target_arch = "x86", - target_arch = "x86_64" - )))] - { - self.for_each_repcode_candidate_with_reps_scalar( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - ) - } - } - - /// NEON umbrella variant of the rep-code probe. - /// - /// # Safety - /// Caller must be running on an AArch64 target with NEON - /// available (baseline on AArch64). - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - #[target_feature(enable = "neon")] - #[allow(clippy::too_many_arguments)] - pub(crate) unsafe fn for_each_repcode_candidate_with_reps_neon( - &self, - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - reps: [u32; 3], - current_abs_end: usize, - min_match_len: usize, - mut f: impl FnMut(MatchCandidate), - ) { - let _ = self; - super::hc::generator::for_each_repcode_candidate_body!( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - crate::encoding::fastpath::neon::common_prefix_len_ptr, - ) - } - - /// SSE2 umbrella variant. - /// - /// # Safety - /// Caller must be running on x86/x86_64 with SSE2 available. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" - ))] - #[target_feature(enable = "sse2")] - #[allow(clippy::too_many_arguments)] - pub(crate) unsafe fn for_each_repcode_candidate_with_reps_sse2( - &self, - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - reps: [u32; 3], - current_abs_end: usize, - min_match_len: usize, - mut f: impl FnMut(MatchCandidate), - ) { - let _ = self; - super::hc::generator::for_each_repcode_candidate_body!( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - crate::encoding::fastpath::sse2::common_prefix_len_ptr, - ) - } - - /// AVX2+BMI2 umbrella variant. - /// - /// # Safety - /// Caller must be running on x86/x86_64 with AVX2 + BMI2 available. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-avx2" - ))] - #[target_feature(enable = "avx2,bmi2")] - #[allow(clippy::too_many_arguments)] - pub(crate) unsafe fn for_each_repcode_candidate_with_reps_avx2_bmi2( - &self, - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - reps: [u32; 3], - current_abs_end: usize, - min_match_len: usize, - mut f: impl FnMut(MatchCandidate), - ) { - let _ = self; - super::hc::generator::for_each_repcode_candidate_body!( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr, - ) - } - - /// Scalar fallback, compiled unless the NEON tier covers this target. - #[cfg(not(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - )))] - #[allow(clippy::too_many_arguments)] - pub(crate) fn for_each_repcode_candidate_with_reps_scalar( - &self, - table: &MatchTable, - abs_pos: usize, - lit_len: usize, - reps: [u32; 3], - current_abs_end: usize, - min_match_len: usize, - mut f: impl FnMut(MatchCandidate), - ) { - let _ = self; - super::hc::generator::for_each_repcode_candidate_body!( - table, - abs_pos, - lit_len, - reps, - current_abs_end, - min_match_len, - f, - crate::encoding::fastpath::scalar::common_prefix_len_ptr, - ) - } - - /// Walk a candidate match backwards over the literal run so the - /// matcher can absorb literal bytes that happen to match the byte - /// preceding the candidate. Upstream zstd parity: equivalent to the back - /// extend inside `ZSTD_HcFindBestMatch` before committing a - /// sequence. - /// - /// Takes `&MatchTable` because the only thing it needs is read - /// access to the contiguous history mirror — kept off `&self` so - /// callers don't have to hand it an HcMatcher reference they - /// don't otherwise use. - pub(crate) fn extend_backwards( - table: &MatchTable, - candidate_pos: usize, - abs_pos: usize, - match_len: usize, - lit_len: usize, - ) -> MatchCandidate { - crate::encoding::match_table::helpers::extend_backwards_shared( - table.live_history(), - table.history_abs_start, - candidate_pos, - abs_pos, - match_len, - lit_len, - ) - } - /// Upstream zstd parity: per-pass clamp of the "good enough — stop probing" - /// threshold that the optimal parser passes to the BT/HC walkers. + /// threshold that the optimal parser passes to the tree walk. /// Reflects upstream zstd `ZSTD_compressBlock_opt_generic` which caps the /// profile's `sufficient_match_len` by the user-configured /// `targetLength` and the `HC_OPT_NUM` ceiling. @@ -975,6 +70,3 @@ impl HcMatcher { .clamp(HC_FORMAT_MINMATCH, HC_OPT_NUM - 1) } } - -#[cfg(test)] -mod hc_tests; diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index a522c7dc..1f7026cb 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -39,7 +39,7 @@ use crate::encoding::{ }, hc::HC_MIN_MATCH_LEN, levels::config::HcConfig, - match_generator::{HC_OPT_MIN_MATCH_LEN, HC_SEARCH_DEPTH, HC_TARGET_LEN, HcBackend}, + match_generator::{HC_OPT_MIN_MATCH_LEN, HC_SEARCH_DEPTH, HC_TARGET_LEN}, match_table::storage::HC3_HASH_LOG, opt::ldm::{HcOptLdmState, HcRawSeqStore}, opt::types::{ @@ -279,7 +279,7 @@ macro_rules! build_optimal_plan_impl_body { seq_store: HcRawSeqStore { pos: 0, pos_in_sequence: 0, - size: $self.backend.bt_mut().ldm_sequences.len(), + size: $self.bt.ldm_sequences.len(), }, ..HcOptLdmState::default() } @@ -300,13 +300,11 @@ macro_rules! build_optimal_plan_impl_body { // matches that copy the wrong bytes (undecodable frame). if ldm_block_offset > 0 { $self - .backend - .bt_mut() + .bt .ldm_skip_raw_seq_store_bytes(&mut opt_ldm.seq_store, ldm_block_offset); } $self - .backend - .bt_mut() + .bt .ldm_get_next_match_and_update_seq_store(&mut opt_ldm, 0, $current_len); } @@ -314,7 +312,7 @@ macro_rules! build_optimal_plan_impl_body { // at current position before entering the generic forward DP loop. if $current_len >= min_match_len { let seed_ldm = if HAS_LDM { - $self.backend.bt_mut().ldm_process_match_candidate( + $self.bt.ldm_process_match_candidate( &mut opt_ldm, 0, $current_len, @@ -367,15 +365,15 @@ macro_rules! build_optimal_plan_impl_body { HC_OPT_PRICE_STRIDE, ) }; - $self.backend.bt_mut().opt_ll_price_stamp = - $self.backend.bt_mut().opt_ll_price_stamp.wrapping_add(1).max(1); - ll_price_stamp = $self.backend.bt_mut().opt_ll_price_stamp; - $self.backend.bt_mut().opt_lit_price_stamp = - $self.backend.bt_mut().opt_lit_price_stamp.wrapping_add(1).max(1); - lit_price_stamp = $self.backend.bt_mut().opt_lit_price_stamp; - $self.backend.bt_mut().opt_ml_price_stamp = - $self.backend.bt_mut().opt_ml_price_stamp.wrapping_add(1).max(1); - ml_price_stamp = $self.backend.bt_mut().opt_ml_price_stamp; + $self.bt.opt_ll_price_stamp = + $self.bt.opt_ll_price_stamp.wrapping_add(1).max(1); + ll_price_stamp = $self.bt.opt_ll_price_stamp; + $self.bt.opt_lit_price_stamp = + $self.bt.opt_lit_price_stamp.wrapping_add(1).max(1); + lit_price_stamp = $self.bt.opt_lit_price_stamp; + $self.bt.opt_ml_price_stamp = + $self.bt.opt_ml_price_stamp.wrapping_add(1).max(1); + ml_price_stamp = $self.bt.opt_ml_price_stamp; // Deferred base/seed prices: only reached on a matched seed (see // the declarations above). Assign before the forward DP / seed // paths below read them. @@ -543,7 +541,7 @@ macro_rules! build_optimal_plan_impl_body { let prev_node = unsafe { *nodes.add(pos - 1) }; let lit_len = prev_node.litlen as usize + 1; let lit_price = { - let bt = $self.backend.bt_mut(); + let bt = &mut *$self.bt; BtMatcher::cached_literal_price::( profile, $stats, @@ -594,7 +592,7 @@ macro_rules! build_optimal_plan_impl_body { { if ll1_price < ll0_price { let next_lit_price = { - let bt = $self.backend.bt_mut(); + let bt = &mut *$self.bt; BtMatcher::cached_literal_price::( profile, $stats, @@ -740,7 +738,7 @@ macro_rules! build_optimal_plan_impl_body { // Upstream `ZSTD_optLdm_processMatchCandidate`: the producer // advances past this position and its candidate joins after // the search. - if let Some(ldm) = $self.backend.bt_mut().ldm_process_match_candidate( + if let Some(ldm) = $self.bt.ldm_process_match_candidate( &mut opt_ldm, pos, $current_len - pos, @@ -1312,8 +1310,6 @@ impl HcMatchGenerator { mut handle_sequence: impl FnMut(Sequence), ) { self.table.ensure_tables(); - // Borrowed-aware: owned → last committed chunk; borrowed → staged - // in-place block range. let (current_abs_start, current_len) = self.table.current_block_range(); if current_len == 0 { return; @@ -1369,16 +1365,15 @@ impl HcMatchGenerator { // The DP bodies read the strategy's `FAVOR_SMALL_OFFSETS` const directly; // verify the runtime profile (built from the same strategy) agrees. debug_assert_eq!(profile.favor_small_offsets, S::FAVOR_SMALL_OFFSETS); - let mut opt_state = - core::mem::replace(&mut self.backend.bt_mut().opt_state, HcOptState::new()); + let mut opt_state = core::mem::replace(&mut self.bt.opt_state, HcOptState::new()); opt_state.rescale_freqs(current, S::ACCURATE_PRICE); - let mut best_plan = core::mem::take(&mut self.backend.bt_mut().opt_segment_plan_scratch); + let mut best_plan = core::mem::take(&mut self.bt.opt_segment_plan_scratch); best_plan.clear(); let plan_reps = self.table.offset_hist; let (cursor, plan_litlen) = self.table.opt_start_cursor_and_litlen(current_abs_start); // Frame-constant LDM presence, resolved once per block (not per segment // and not in the DP hot loop): drives the HAS_LDM const-generic dispatch. - let has_ldm = !self.backend.bt_mut().ldm_sequences.is_empty(); + let has_ldm = !self.bt.ldm_sequences.is_empty(); // Resolve the SIMD tier ONCE here: the whole block then runs under one // kernel-monomorphized pass (`run_optimal_block_`) instead of // hitting select_kernel()'s OnceLock atomic + a CPU-tier match per @@ -1543,12 +1538,9 @@ impl HcMatchGenerator { self.table .emit_optimal_plan(current_len, &best_plan, &mut handle_sequence); best_plan.clear(); - self.backend.bt_mut().opt_segment_plan_scratch = best_plan; - let _ = self - .backend - .bt_mut() - .finish_optimal_plan(plan_buffers, (0, [0; 3], 0, 0)); - self.backend.bt_mut().opt_state = opt_state; + self.bt.opt_segment_plan_scratch = best_plan; + let _ = self.bt.finish_optimal_plan(plan_buffers, (0, [0; 3], 0, 0)); + self.bt.opt_state = opt_state; } // The scalar arm of `run_seed_loop!` wraps a safe call in `unsafe`. @@ -1573,14 +1565,13 @@ impl HcMatchGenerator { // Same per-block clamp the main pass does; the DP body expects it done. seed_profile.sufficient_match_len = self.hc.sufficient_match_len_for_pass(seed_profile); debug_assert_eq!(seed_profile.favor_small_offsets, S::FAVOR_SMALL_OFFSETS); - let mut opt_state = - core::mem::replace(&mut self.backend.bt_mut().opt_state, HcOptState::new()); + let mut opt_state = core::mem::replace(&mut self.bt.opt_state, HcOptState::new()); opt_state.rescale_freqs(current, S::ACCURATE_PRICE); let seed_reps = self.table.offset_hist; let (cursor, seed_litlen) = self.table.opt_start_cursor_and_litlen(current_abs_start); - let mut seed_plan = core::mem::take(&mut self.backend.bt_mut().opt_seed_plan_scratch); + let mut seed_plan = core::mem::take(&mut self.bt.opt_seed_plan_scratch); seed_plan.clear(); - let has_ldm = !self.backend.bt_mut().ldm_sequences.is_empty(); + let has_ldm = !self.bt.ldm_sequences.is_empty(); // SIMD tier resolved ONCE (see start_matching_optimal): the seed pass // runs as one kernel-monomorphized block pass. It keeps the statistics // and drops the plan (`KEEP_PLAN = false`). @@ -1741,11 +1732,11 @@ impl HcMatchGenerator { run_seed_loop!(run_optimal_block_scalar); } seed_plan.clear(); - self.backend.bt_mut().opt_seed_plan_scratch = seed_plan; + self.bt.opt_seed_plan_scratch = seed_plan; // The DP scratch buffers are owned by the caller (start_matching_optimal) // and shared across both passes, so the seed pass leaves them in place // (`plan_buffers` is a borrow) instead of restoring them here. - self.backend.bt_mut().opt_state = opt_state; + self.bt.opt_state = opt_state; // Upstream zstd initStats_ultra keeps the collected entropy statistics but // invalidates the first-pass matchfinder history before the real pass. @@ -1773,7 +1764,7 @@ impl HcMatchGenerator { /// literal). Restore with [`BtMatcher::finish_optimal_plan`] after the /// per-segment loop. fn take_optimal_plan_buffers(&mut self) -> HcOptimalPlanBuffers { - let bt = self.backend.bt_mut(); + let bt = &mut *self.bt; let mut nodes = core::mem::take(&mut bt.opt_nodes_scratch); let mut node_prices = core::mem::take(&mut bt.opt_node_prices_scratch); let mut candidates = core::mem::take(&mut bt.opt_candidates_scratch); diff --git a/zstd/src/encoding/lazy_parse.rs b/zstd/src/encoding/lazy_parse.rs index 15428ab8..ad47ae13 100644 --- a/zstd/src/encoding/lazy_parse.rs +++ b/zstd/src/encoding/lazy_parse.rs @@ -1,19 +1,13 @@ -//! Shared, C-faithful lazy-parse decision logic. +//! The lazy-parse decision as a standalone macro, for the unit tests that probe +//! the row search outside the lazy body. //! //! Upstream zstd has a SINGLE lazy parser — `ZSTD_compressBlock_lazy_generic` //! (`lib/compress/zstd_lazy.c`) — parameterized by the search method -//! (hash-chain / binary-tree / row-hash) and the lookahead `depth`. The match -//! finders differ per strategy; the PARSE DECISION (how a candidate at the -//! current position is weighed against one a byte or two ahead) is identical -//! across all of them. -//! -//! This module hosts that one decision so the per-strategy matchers -//! (HC / Row / dfast) share it instead of each carrying a divergent copy. The -//! register-hungry part is the search kernel, which stays specialized per -//! strategy and is injected here as a closure — the decision itself is a -//! handful of comparisons per position, so sharing it adds no spill pressure. +//! (hash-chain / binary-tree / row-hash) and the lookahead `depth`; the PARSE +//! DECISION (how a candidate at the current position is weighed against one a +//! byte or two ahead) is identical across all of them. -/// C-faithful lazy commit/defer decision, ONE source for every strategy. +/// C-faithful lazy commit/defer decision. /// /// Expanded INLINE at the call site (like upstream's single /// `FORCE_INLINE_TEMPLATE ZSTD_compressBlock_lazy_generic`, zstd_lazy.c:1516): diff --git a/zstd/src/encoding/levels/config.rs b/zstd/src/encoding/levels/config.rs index c4d334ad..fdf99e40 100644 --- a/zstd/src/encoding/levels/config.rs +++ b/zstd/src/encoding/levels/config.rs @@ -56,6 +56,16 @@ pub(crate) struct RowConfig { /// Upstream `ZSTD_btlazy2`: the lazy parse searches the lazily-sorted /// binary tree (`ZSTD_BtFindBestMatch`) instead of rows / the chain. pub(crate) bt: bool, + /// Rows or the hash chain, when not the tree (`ZSTD_c_useRowMatchFinder`). + pub(crate) match_finder: crate::encoding::parameters::RowMatchFinderMode, +} + +impl RowConfig { + /// Whether a frame without a dictionary searches rows with a `window`-byte + /// window (a dictionary frame takes its dictionary's answer). + pub(crate) const fn searches_rows(&self, window: usize) -> bool { + !self.bt && self.match_finder.searches_rows(window) + } } // Only used as the default HashChain config when the test-only parse×search @@ -93,6 +103,7 @@ pub(crate) const ROW_CONFIG: RowConfig = RowConfig { mls: ROW_MIN_MATCH_LEN, chain_log: ROW_HASH_BITS, bt: false, + match_finder: crate::encoding::parameters::RowMatchFinderMode::Auto, }; // Level-5 greedy is the ONLY strategy routed to the Row backend @@ -119,6 +130,7 @@ pub(crate) const ROW_L5: RowConfig = RowConfig { mls: ROW_MIN_MATCH_LEN, chain_log: 18, bt: false, + match_finder: crate::encoding::parameters::RowMatchFinderMode::Auto, }; /// Per-level Double-Fast hash sizing, mirroring the upstream zstd `clevels.h` columns @@ -242,9 +254,7 @@ impl LevelParams { SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => { self.row.map(|row| (row.hash_bits, row.chain_log)) } - SearchMethod::HashChain | SearchMethod::BinaryTree => { - self.hc.map(|hc| (hc.hash_log, hc.chain_log)) - } + SearchMethod::BinaryTree => self.hc.map(|hc| (hc.hash_log, hc.chain_log)), }; // The reset synthesises the active config when a row lacks it, so a // missing one only means a table-less level: nothing to index. @@ -264,7 +274,6 @@ impl LevelParams { SearchMethod::Fast => self.fast.map(|fast| fast.mls as usize), SearchMethod::DoubleFast => Some(crate::encoding::match_generator::DFAST_MIN_MATCH_LEN), SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => self.row.map(|row| row.mls), - SearchMethod::HashChain => Some(crate::encoding::hc::HC_MIN_MATCH_LEN), SearchMethod::BinaryTree => self.hc.map(|hc| hc.search_mls), }; configured.unwrap_or(crate::encoding::cost_model::HC_FORMAT_MINMATCH) @@ -347,7 +356,7 @@ pub(crate) fn apply_param_overrides( let row = params.row.get_or_insert(ROW_L5); row.bt = matches!(params.search, SearchMethod::BinaryTreeLazy); } - SearchMethod::HashChain | SearchMethod::BinaryTree => { + SearchMethod::BinaryTree => { params.hc.get_or_insert(HC_OVERRIDE_DEFAULT); } } @@ -416,9 +425,10 @@ pub(crate) fn apply_param_overrides( if let Some(min_match) = ov.min_match { row.mls = min_match as usize; } + row.match_finder = ov.row_match_finder; } } - SearchMethod::HashChain | SearchMethod::BinaryTree => { + SearchMethod::BinaryTree => { if let Some(hc) = params.hc.as_mut() { if let Some(hash_log) = ov.hash_log { hc.hash_log = hash_log as usize; @@ -571,13 +581,6 @@ pub(crate) const MAX_FAST_ATTACH_DICT_REGION: usize = 1 << 24; /// fast-loop levels (L3 / Default / L0) carry the dual-probe. pub(crate) const DFAST_ATTACH_DICT_CUTOFF_LOG: u8 = 14; -/// 32 KiB (`2^15`, upstream zstd `attachDictSizeCutoffs[ZSTD_lazy2]`): small / -/// unknown-size inputs ATTACH the dict as a separate hash-chain dms (the dual -/// search in `find_best_match` walks the live input chain + the dms), larger -/// known-size inputs dense-COPY (merge the dict into the live chain and search -/// the one combined chain). -pub(crate) const HC_ATTACH_DICT_CUTOFF_LOG: u8 = 15; - /// BT/optimal attach cutoff for `btlazy2` + `btopt`: 32 KiB (`2^15`, upstream /// zstd `attachDictSizeCutoffs[ZSTD_btlazy2]` == `[ZSTD_btopt]`). Small / /// unknown-size inputs ATTACH the dict as a separate DUBT dms; larger known-size @@ -663,6 +666,7 @@ fn level_params_from_cparams(cp: crate::encoding::cparams::CParams) -> LevelPara chain_log: cp.chain_log as usize, // Upstream `ZSTD_btlazy2` (strategy 6). bt: cp.strategy == 6, + match_finder: crate::encoding::parameters::RowMatchFinderMode::Auto, }; let bt = |tag| LevelParams { strategy_tag: tag, @@ -1137,17 +1141,17 @@ fn workspace_bytes(params: &LevelParams, ldm: usize) -> usize { table_bytes(4, usize::from(d.long_hash_log)) .saturating_add(table_bytes(4, usize::from(d.short_hash_log))) }), - // The lazy backend's chain / tree finders (window <= 2^14, or a - // btlazy2 level) use a plain hash table (`4 << hash_bits`) plus the - // chain / tree table (`4 << chain_log`) instead of the row tables. + // The lazy backend's chain / tree finders use a plain hash table + // (`4 << hash_bits`) plus the chain / tree table (`4 << chain_log`) + // instead of the row tables. SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => params.row.map_or(0, |r| { - if r.bt || params.window_log <= 14 { + if !r.searches_rows(window) { table_bytes(4, r.hash_bits).saturating_add(table_bytes(4, r.chain_log)) } else { table_bytes(4, r.hash_bits).saturating_add(table_bytes(2, r.hash_bits)) } }), - SearchMethod::HashChain | SearchMethod::BinaryTree => params.hc.map_or(0, |h| { + SearchMethod::BinaryTree => params.hc.map_or(0, |h| { let hash3 = if wants_hash3 { table_bytes( 4, @@ -1397,7 +1401,7 @@ pub(crate) fn resolve_level_params_with_dict( params, Some(RowDictPlan { attach, - use_row: uses_row_match_finder(&cdict), + use_row: uses_row_match_finder(&cdict, overrides.row_match_finder), cdict, }), ) diff --git a/zstd/src/encoding/match_generator/dict_prime.rs b/zstd/src/encoding/match_generator/dict_prime.rs index fc964250..3155295c 100644 --- a/zstd/src/encoding/match_generator/dict_prime.rs +++ b/zstd/src/encoding/match_generator/dict_prime.rs @@ -301,7 +301,7 @@ impl MatchGeneratorDriver { .saturating_add(granted_retained_budget); } if self.active_backend() == super::super::strategy::BackendTag::HashChain { - // Recompute the lazy-HC attach decision made per-chunk in + // Recompute the attach decision made per-chunk in // `skip_matching_for_dictionary_priming` (stable across the prime — // `reset_size_log` does not change here). // @@ -310,12 +310,12 @@ impl MatchGeneratorDriver { // table whose dimensions differ from the copy-mode live table, so a // cross-mode restore would install mismatched table geometry and the // encoder could search past the frame window (undecodable). The two - // HC modes share identical window geometry: `max_window_size` and the + // modes share identical window geometry: `max_window_size` and the // dictionary limit are both set ABOVE this branch (the same value in - // either mode), and the live chain table dimensions come from the + // either mode), and the live tree dimensions come from the // resolved `params` the key already pins. The modes differ only in - // WHERE the committed dict lives — a single-link `dms` (attach) vs - // merged into the live chain (copy) — both producing valid matches at + // WHERE the committed dict lives — a separate `dms` tree (attach) vs + // merged into the live tree (copy) — both producing valid matches at // in-window offsets. Upstream zstd makes the same observation: attach // (`ZSTD_resetCCtx_byAttachingCDict`) and copy // (`ZSTD_resetCCtx_byCopyingCDict`) both keep the caller's @@ -336,19 +336,16 @@ impl MatchGeneratorDriver { let attach = self.hc_dict_attach_mode(); let table = &mut self.hc_matcher_mut().table; table.set_dictionary_limit_from_primed_bytes(committed_dict_budget); - // Build the dictMatchState over the committed dict (front of history) - // so `find_best_match` dual-probes it with its own compare budget — - // but ONLY in ATTACH mode. BT/optimal attach → DUBT dms; lazy-HC - // attach → single-link hash-chain dms. COPY mode (large known source, - // both BT and lazy-HC) already merged the dict into the live tree / - // chain in `skip_matching_for_dictionary_priming`, so it carries no - // separate dms — drop any stale one. - if !attach { - table.dms.invalidate(); - } else if table.uses_bt { + // Build the dictMatchState tree over the committed dict (front of + // history) so the search dual-probes it with its own compare budget — + // but ONLY in ATTACH mode. COPY mode (large known source) already + // merged the dict into the live tree in + // `skip_matching_for_dictionary_priming`, so it carries no separate + // dms — drop any stale one. + if attach { table.prime_dms_bt(committed_dict_budget); } else { - table.prime_dms_hc(committed_dict_budget); + table.dms.invalidate(); } } // CDict-equivalent: now that every dict chunk is indexed, mark the @@ -417,23 +414,6 @@ impl MatchGeneratorDriver { (MatcherStorage::Simple(live), MatcherStorage::Simple(snap)) => { live.clone_from(snap); } - // Same-variant HC lazy/greedy restore (non-BT): the snapshot keeps - // the full primed hash/chain tables (capture's non-BT full clone), - // so `clone_from` reuses the live history/hash/chain/dms buffers in - // place — upstream zstd reuses the CDict tables rather than reallocating - // them. This is the per-frame allocate+copy+drop that dominated - // small `compress-dict` HC frames (5-7x vs C). BT (`uses_bt`) - // snapshots drop their live tables, so they stay on the realloc - // path below. - (MatcherStorage::HashChain(live), MatcherStorage::HashChain(snap)) - if !snap.table.uses_bt => - { - live.table.clone_from(&snap.table); - live.hc.clone_from(&snap.hc); - live.strategy_tag = snap.strategy_tag; - // backend is `HcBackend::Hc` (zero-sized) for non-BT levels; - // the live one is already correct for this resolved key. - } // Binary tree: the history and, in COPY mode, the tree land in the // rooms the reset laid out in the workspace. An ATTACH-mode // snapshot holds no tree (see `capture_primed_dictionary`), so the @@ -510,17 +490,13 @@ impl MatchGeneratorDriver { // clone only the dict-state (history + `dms` + window/offset/dict-limit), // then move the live tables back — the snapshot keeps just what upstream zstd's // CDict keeps, and `restore_primed_dictionary` re-allocates the zeroed - // live tables. Every other case keeps the dict reachable through the live - // structure, so the snapshot must retain the full tables (full clone): - // lazy-HC attach (it DOES prime a hash-chain `dms`, but the live chain is - // still the search structure, so the tables must travel) and COPY mode for - // BOTH BT and lazy-HC (`dms` invalidated, dict merged into the live tree / - // chain). `uses_bt && dms.is_primed()` is therefore the exact "decoupled" - // signal — true only for the BT attach prime; lazy-HC attach primes `dms` - // too but is intentionally NOT decoupled. + // live tables. COPY mode keeps the dict reachable through the live tree + // (`dms` invalidated, dict merged in), so its snapshot must retain the + // full tables (full clone). `dms.is_primed()` is therefore the exact + // "decoupled" signal. let bt_decoupled = matches!( &self.storage, - MatcherStorage::HashChain(hc) if hc.table.uses_bt && hc.table.dms.is_primed() + MatcherStorage::HashChain(hc) if hc.table.dms.is_primed() ); if bt_decoupled { let MatcherStorage::HashChain(hc) = &mut self.storage else { diff --git a/zstd/src/encoding/match_generator/long_distance.rs b/zstd/src/encoding/match_generator/long_distance.rs index 323ee7f1..58cccbe2 100644 --- a/zstd/src/encoding/match_generator/long_distance.rs +++ b/zstd/src/encoding/match_generator/long_distance.rs @@ -119,7 +119,7 @@ impl MatchGeneratorDriver { } let gap_literals = gap.expect("the gap sink only ever writes Some"); self.storage.commit_piece(seq.match_length); - self.storage.skip_long_match::(); + self.storage.skip_long_match(); handle_sequence(Sequence::Triple { literal_len: gap_literals, offset: seq.offset, diff --git a/zstd/src/encoding/match_generator/mod.rs b/zstd/src/encoding/match_generator/mod.rs index 722814d2..caa1587d 100644 --- a/zstd/src/encoding/match_generator/mod.rs +++ b/zstd/src/encoding/match_generator/mod.rs @@ -207,11 +207,8 @@ enum MatcherStorage { /// `SearchMethod::BinaryTreeLazy`. The Row backend itself searches a hash /// chain instead of rows when the resolved window is small. Row(RowMatchGenerator), - /// The hash-chain matcher, for `SearchMethod::HashChain` and - /// `SearchMethod::BinaryTree`; the latter carries the BT-based optimal - /// modes (`btopt` / `btultra` / `btultra2`). The [`HcMatchGenerator`]'s - /// internal [`HcBackend`] discriminator decides whether BT scratch is - /// allocated. + /// The binary-tree matcher of the optimal parsers (`btopt` / `btultra` / + /// `btultra2`), for `SearchMethod::BinaryTree`. HashChain(HcMatchGenerator), } @@ -312,12 +309,12 @@ impl MatcherStorage { /// See the backends' `skip_long_match`. #[cfg(feature = "ldm")] - fn skip_long_match(&mut self) { + fn skip_long_match(&mut self) { match self { Self::Simple(m) => m.skip_long_match(), Self::Dfast(m) => m.skip_long_match(), Self::Row(m) => m.skip_long_match(), - Self::HashChain(m) => m.table.skip_long_match::(), + Self::HashChain(m) => m.table.skip_long_match(), } } @@ -565,15 +562,11 @@ struct PrimedKey { /// Whether a configured HashChain matcher attaches the dictionary for a frame /// of `size_log` (see [`MatchGeneratorDriver::hc_dict_attach_mode`]). fn hc_attaches_dictionary(hc: &HcMatchGenerator, size_log: Option) -> bool { - let cutoff = if hc.table.uses_bt { - match hc.strategy_tag { - super::strategy::StrategyTag::BtUltra | super::strategy::StrategyTag::BtUltra2 => { - BT_ULTRA_ATTACH_DICT_CUTOFF_LOG - } - _ => BT_OPT_ATTACH_DICT_CUTOFF_LOG, + let cutoff = match hc.strategy_tag { + super::strategy::StrategyTag::BtUltra | super::strategy::StrategyTag::BtUltra2 => { + BT_ULTRA_ATTACH_DICT_CUTOFF_LOG } - } else { - HC_ATTACH_DICT_CUTOFF_LOG + _ => BT_OPT_ATTACH_DICT_CUTOFF_LOG, }; size_log.is_none_or(|log| log <= cutoff) } @@ -925,13 +918,9 @@ impl MatchGeneratorDriver { } /// Whether the borrowed (no-copy, in-place over-window) scan is - /// implemented for the current backend + search configuration. The - /// HashChain backend serves both the lazy CHAIN parser - /// (`SearchMethod::HashChain`) and the BT/optimal parsers - /// (`SearchMethod::BinaryTree`); only the lazy chain has a borrowed scan - /// so far, so BT/optimal stay on the owned path. + /// implemented for the current backend. pub(crate) fn borrowed_supported(&self) -> bool { - use super::strategy::{BackendTag, SearchMethod}; + use super::strategy::BackendTag; // A producer reads the committed block, which the in-place scan never // commits. if self.producer.is_some() { @@ -939,18 +928,11 @@ impl MatchGeneratorDriver { } match self.active_backend() { BackendTag::Simple | BackendTag::Dfast | BackendTag::Row => true, - // The HashChain backend covers two searches: the lazy CHAIN parser - // (borrowed-capable) and the BINARY-TREE search (btlazy2 L13-15 + - // optimal BtOpt/BtUltra/BtUltra2 L16-22). btlazy2's BT-tree borrowed - // scan is byte-identical to owned (reads via live_history()), so it - // takes the in-place path. The OPTIMAL parsers stay owned: their - // cost-based DP is sensitive to candidate quality, and the borrowed - // continuous-index scan yields slightly different (ratio-worse) - // candidates than the owned evict+rehash scan — borrowed optimal - // both diverged from owned and fell outside the ffi ratio bound. - // Search-aware (not just strategy_tag) so optimal BT can never be - // staged on the borrowed path even via an internal caller. - BackendTag::HashChain => matches!(self.search, SearchMethod::HashChain), + // The optimal parsers stay owned: their cost-based DP is sensitive + // to candidate quality, and a borrowed continuous-index scan yields + // different (ratio-worse) candidates than the owned evict+rehash + // scan. + BackendTag::HashChain => false, } } @@ -1027,9 +1009,9 @@ impl MatchGeneratorDriver { super::strategy::BackendTag::Row => unsafe { self.row_matcher_mut().set_borrowed_window(buffer) }, - super::strategy::BackendTag::HashChain => unsafe { - self.hc_matcher_mut().set_borrowed_window(buffer) - }, + super::strategy::BackendTag::HashChain => { + unreachable!("the optimal parsers have no borrowed scan") + } } } @@ -1040,9 +1022,7 @@ impl MatchGeneratorDriver { super::strategy::BackendTag::Simple => self.simple_mut().clear_borrowed_window(), super::strategy::BackendTag::Dfast => self.dfast_matcher_mut().clear_borrowed_window(), super::strategy::BackendTag::Row => self.row_matcher_mut().clear_borrowed_window(), - super::strategy::BackendTag::HashChain => self.hc_matcher_mut().clear_borrowed_window(), - #[allow(unreachable_patterns)] - _ => {} + super::strategy::BackendTag::HashChain => {} } self.borrowed_pending = None; } @@ -1079,10 +1059,9 @@ impl MatchGeneratorDriver { super::strategy::BackendTag::Row => self .row_matcher_mut() .stage_borrowed_block(block_start, block_end), - super::strategy::BackendTag::HashChain => self - .hc_matcher_mut() - .table - .stage_borrowed_block(block_start, block_end), + super::strategy::BackendTag::HashChain => { + unreachable!("`borrowed_supported` refuses the optimal parsers") + } } } @@ -1335,16 +1314,13 @@ impl MatchGeneratorDriver { self.row_matcher_mut().prime_dictionary_current_block(); } super::strategy::BackendTag::HashChain => { - // Lazy-HC AND BT/optimal both follow upstream zstd `ZSTD_shouldAttachDict` - // per-strategy: ATTACH (a separate dms — hash-chain dms for lazy, - // DUBT dms for BT) for small / unknown inputs, COPY (merge the dict - // into the live chain/tree) for large known inputs. ATTACH keeps - // the dict in history but out of the live structure via - // `skip_matching_dict_bt` (the cursor advance is shared by both - // arms); COPY routes through the normal `skip_matching` (its - // `uses_bt` branch fills the live tree, the lazy branch the live - // chain). The dms is built-or-dropped to match in - // `prime_with_dictionary`. + // Upstream zstd `ZSTD_shouldAttachDict` per strategy: ATTACH (a + // separate DUBT dms) for small / unknown inputs, COPY (merge the + // dict into the live tree) for large known inputs. ATTACH keeps + // the dict in history but out of the live tree via + // `skip_matching_dict_bt`; COPY routes through the normal + // `skip_matching`, which fills the live tree. The dms is + // built-or-dropped to match in `prime_with_dictionary`. if self.hc_dict_attach_mode() { self.hc_matcher_mut().table.skip_matching_dict_bt(); } else { @@ -1536,8 +1512,8 @@ impl Matcher for MatchGeneratorDriver { params.search = search; params.lazy_depth = parse.lazy_depth(); // The matrix sweep can pair a level with a backend its native - // row doesn't populate (e.g. greedy L5, which carries only `row`, - // run on HashChain). Synthesize a default config for the + // row doesn't populate (e.g. dfast L3, which carries only `dfast`, + // run on rows). Synthesize a default config for the // overridden backend so its `configure` arm has something to read. use super::strategy::SearchMethod; match search { @@ -1551,7 +1527,7 @@ impl Matcher for MatchGeneratorDriver { let row = params.row.get_or_insert(ROW_CONFIG); row.bt = matches!(search, SearchMethod::BinaryTreeLazy); } - SearchMethod::HashChain | SearchMethod::BinaryTree => { + SearchMethod::BinaryTree => { params.hc.get_or_insert(HC_CONFIG); } } @@ -2002,19 +1978,16 @@ impl Matcher for MatchGeneratorDriver { } MatcherStorage::HashChain(hc) => { hc.table.max_window_size = max_window_size; - hc.hc.lazy_depth = params.lazy_depth; let mut hc_cfg = params.hc.expect("HashChain level row carries an HcConfig"); - // Cap the hash / chain table logs by the hinted window so a small + // Cap the hash / tree table logs by the hinted window so a small // input doesn't allocate the full level's tables (the upstream zstd // `ZSTD_adjustCParams_internal` clamp: `hashLog <= windowLog + 1`, - // and `cycleLog <= windowLog` — `cycleLog == chainLog` for the HC - // finder, `chainLog - 1` for the BT pair table, so `chainLog <= - // windowLog` (+1 for BT)). Ratio-neutral: a hinted window of - // `2^wlog` bytes holds at most `2^wlog` positions, so the slots - // beyond that are never populated — capping only sheds unused - // allocation. Was the source of L10-lazy peak-alloc ~2.15x the - // upstream zstd on a 1 MiB input. Only applied when hinted; an - // unknown-size stream keeps the full level tables. + // and `cycleLog <= windowLog` with `cycleLog == chainLog - 1` for + // the BT pair table, so `chainLog <= windowLog + 1`). + // Ratio-neutral: a hinted window of `2^wlog` bytes holds at most + // `2^wlog` positions, so the slots beyond that are never + // populated — capping only sheds unused allocation. Only applied + // when hinted; an unknown-size stream keeps the full level tables. // Skip for dictionary frames: their `hc_cfg.{hash,chain}_log` // are the CDict's (verbatim when the dictionary is copied // into the live tables, source-adjusted when it is attached); @@ -2022,15 +1995,8 @@ impl Matcher for MatchGeneratorDriver { // dictionary's tables to the small hinted source. if hinted && dict_hint.is_none() { let wlog = hc_hash_bits_for_window(table_window_size); - let uses_bt = matches!( - strategy_tag, - super::strategy::StrategyTag::Btlazy2 - | super::strategy::StrategyTag::BtOpt - | super::strategy::StrategyTag::BtUltra - | super::strategy::StrategyTag::BtUltra2 - ); hc_cfg.hash_log = hc_cfg.hash_log.min(wlog + 1); - hc_cfg.chain_log = hc_cfg.chain_log.min(if uses_bt { wlog + 1 } else { wlog }); + hc_cfg.chain_log = hc_cfg.chain_log.min(wlog + 1); } hc.configure(hc_cfg, strategy_tag, params.window_log); // A copy-mode frame merges the dictionary into the live tables, @@ -2331,9 +2297,9 @@ impl MatchGeneratorDriver { block_end, incompressible_hint, ), - super::strategy::BackendTag::HashChain => self - .hc_matcher_mut() - .skip_matching_borrowed::(block_start, block_end, incompressible_hint), + super::strategy::BackendTag::HashChain => { + unreachable!("`borrowed_supported` refuses the optimal parsers") + } } return; } @@ -2442,16 +2408,9 @@ impl MatchGeneratorDriver { &mut sink, ); } - super::strategy::BackendTag::HashChain => match self.search { - super::strategy::SearchMethod::HashChain => self - .hc_matcher_mut() - .start_matching_lazy_borrowed::(block_start, block_end, &mut sink), - // `borrowed_supported()` keeps the optimal parsers on the - // owned path and `set_borrowed_block` asserts it. - other => { - unreachable!("HashChain backend with unexpected borrowed search {other:?}") - } - }, + super::strategy::BackendTag::HashChain => { + unreachable!("`borrowed_supported` refuses the optimal parsers") + } } return; } @@ -2472,7 +2431,7 @@ impl MatchGeneratorDriver { let MatcherStorage::HashChain(hc) = &mut self.storage else { unreachable!("the optimal parser runs on the hash-chain backend"); }; - hc.backend.bt_mut().ldm_sequences.clear(); + hc.bt.ldm_sequences.clear(); } if self.start_matching_long_distance::(&mut handle_sequence, true) { return; @@ -2503,7 +2462,7 @@ impl MatchGeneratorDriver { let MatcherStorage::HashChain(hc) = &mut self.storage else { unreachable!("the optimal parser runs on the hash-chain backend"); }; - core::mem::swap(&mut self.ldm_block, &mut hc.backend.bt_mut().ldm_sequences); + core::mem::swap(&mut self.ldm_block, &mut hc.bt.ldm_sequences); } else if self.start_matching_long_distance::(&mut handle_sequence, false) { return; } @@ -2556,12 +2515,6 @@ impl MatchGeneratorDriver { // resolved at `configure`. self.row_matcher_mut().start_matching::(handle_sequence); } - SearchMethod::HashChain => { - // Greedy/lazy/lazy2 all flow through the lazy parser; it - // reads `hc.lazy_depth` (0 = greedy commit). - self.hc_matcher_mut() - .start_matching_lazy::(handle_sequence); - } SearchMethod::BinaryTree => match self.strategy_tag { StrategyTag::BtOpt => self.compress_block::(handle_sequence), StrategyTag::BtUltra => { @@ -2584,10 +2537,9 @@ impl MatchGeneratorDriver { /// the concrete opt `S: Strategy` (BtOpt / BtUltra / BtUltra2) off /// `strategy_tag`, so the optimiser keeps the cost-model predicates /// (`S::USE_BT` / `S::USE_HASH3` / `S::ACCURATE_PRICE` / - /// `S::TWO_PASS_SEED`) const-folded per strategy. The non-opt search - /// backends (Fast / DoubleFast / RowHash / HashChain) are dispatched - /// directly off the search axis and never reach this method, so all - /// strategies arriving here are HashChain-backed. + /// `S::TWO_PASS_SEED`) const-folded per strategy. The other search + /// backends are dispatched directly off the search axis and never reach + /// this method. fn compress_block( &mut self, handle_sequence: &mut impl FnMut(Sequence), @@ -2598,33 +2550,10 @@ impl MatchGeneratorDriver { "compress_block only handles the optimal (BT) path" ); self.hc_matcher_mut() - .start_matching_strategy::(handle_sequence); + .start_matching_optimal::(handle_sequence); } } -/// Stage D: backend storage discriminator. -/// -/// HC (lazy / lazy2) modes carry no extra per-frame state beyond the -/// shared `MatchTable` and `HcMatcher` runtime knobs, so the -/// [`HcBackend::Hc`] variant is zero-sized — no BT scratch is -/// allocated. BT-flavoured modes (`btopt` / `btultra` / `btultra2`) -/// hold the full [`super::bt::BtMatcher`] inside the -/// [`HcBackend::Bt`] variant (cost model, optimal-parser scratch -/// arenas, LDM candidate buffer). -/// -/// The discriminator lives next to `parse_mode` so `configure()` can -/// promote between the two on a level change without touching the -/// `MatchTable` storage. -#[derive(Clone)] -pub(crate) enum HcBackend { - /// Lazy / lazy2 modes — no per-frame backend state. - Hc, - /// BT-driven modes — owns the optimal parser's per-frame scratch. - /// Boxed so the enum stays pointer-sized: HC-only matchers pay - /// just the `Box`-niche, not the 4 KiB `BtMatcher` payload. - Bt(alloc::boxed::Box), -} - #[cfg(feature = "bench-internals")] pub(crate) fn level22_block_ranges(data: &[u8]) -> Vec<(usize, usize)> { let mut ranges = Vec::new(); diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index c1a44ddf..cf1e9b27 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -46,18 +46,6 @@ impl MatchGeneratorDriver { self.config_override = Some((search, parse)); } - /// Test-only: reset `level` routed onto the lazy HashChain pairing. - /// The lazy band runs on the Row backend in production, so HC-specific - /// behaviour (live-chain dict prime, eviction budget accounting, seed - /// pass gates) is exercised through this override-backed reset. - pub(crate) fn reset_on_hc_lazy(&mut self, level: CompressionLevel) { - self.set_config_override( - super::super::strategy::SearchMethod::HashChain, - super::super::strategy::ParseMode::Lazy2, - ); - self.reset(level); - } - /// Test-only: the tier the [`Matcher`] entry points run under, in place /// of the one detected at construction. pub(crate) fn set_tier(&mut self, tier: crate::encoding::fastpath::FastpathKernel) { @@ -97,10 +85,9 @@ fn drive_roundtrip_with_override( out } -/// Phase 1 capability proof: parse and search are decoupled, so a level -/// can run any parse mode on any non-opt search backend. Greedy-on- -/// HashChain and Lazy2-on-RowHash are pairings the legacy `strategy_tag` -/// could not express; both must reconstruct the input exactly. +/// Parse and search are decoupled, so a level can run any parse mode on any +/// non-opt search backend, including one its own row does not configure; the +/// pairings must reconstruct the input exactly. #[test] fn parse_search_matrix_decoupled_roundtrips() { use super::super::strategy::{ParseMode, SearchMethod}; @@ -111,14 +98,14 @@ fn parse_search_matrix_decoupled_roundtrips() { data.extend_from_slice(&i.to_le_bytes()); } - // Greedy parse on the HashChain search backend (legacy: Greedy was - // welded to RowHash). + // A dfast level on the rows: its row carries no row config, so the + // driver synthesizes one. let got = drive_roundtrip_with_override( - CompressionLevel::Level(5), - Some((SearchMethod::HashChain, ParseMode::Greedy)), + CompressionLevel::Level(3), + Some((SearchMethod::RowHash, ParseMode::Greedy)), &data, ); - assert_eq!(got, data, "greedy-on-hashchain diverged"); + assert_eq!(got, data, "greedy-on-rowhash from a dfast level diverged"); // Lazy2 parse on the RowHash search backend (legacy: Lazy was welded // to HashChain). @@ -144,8 +131,7 @@ fn parse_search_matrix_decoupled_roundtrips() { #[cfg(feature = "ldm")] #[test] fn hash_chain_lazy_takes_long_distance_matches() { - use super::super::parameters::CompressionParameters; - use super::super::strategy::{ParseMode, SearchMethod}; + use super::super::parameters::{CompressionParameters, RowMatchFinderMode}; // Six-bit symbols from an LCG's high bits: no repeat the chain finds. let mut state = 0x0569_C4A1u32; @@ -169,15 +155,15 @@ fn hash_chain_lazy_takes_long_distance_matches() { .window_log(22) .hash_log(12) .chain_log(12) + .row_match_finder(RowMatchFinderMode::Disable) .enable_long_distance_matching(long) .build() .unwrap() .overrides(); let mut driver = MatchGeneratorDriver::new(1 << 17, 8); driver.set_param_overrides(Some(overrides)); - driver.set_config_override(SearchMethod::HashChain, ParseMode::Lazy2); driver.reset(CompressionLevel::Level(5)); - assert_eq!(driver.search, SearchMethod::HashChain); + assert!(driver.row_matcher().uses_hash_chain()); let mut out = Vec::with_capacity(data.len()); let mut literals = 0usize; let mut consumed = 0usize; @@ -726,11 +712,11 @@ fn btultra2_seed_pass_initializes_opt_state() { hc.table.commit_input(&data); hc.start_matching(|_| {}); assert!( - hc.backend.bt_mut().opt_state.lit_length_sum > 0, + hc.bt.opt_state.lit_length_sum > 0, "btultra2 first block should seed non-zero sequence statistics" ); assert!( - hc.backend.bt_mut().opt_state.off_code_sum > 0, + hc.bt.opt_state.off_code_sum > 0, "btultra2 first block should seed offset-code statistics" ); } @@ -1016,7 +1002,7 @@ fn dictionary_entropy_seed_initializes_opt_state_from_tables() { let of = crate::fse::fse_encoder::default_of_table(); hc.seed_dictionary_entropy(Some(&huff), Some(&*ll), Some(&*ml), Some(&*of)); - hc.backend.bt_mut().opt_state.rescale_freqs( + hc.bt.opt_state.rescale_freqs( b"abcd", ::ACCURATE_PRICE, ); @@ -1027,22 +1013,15 @@ fn dictionary_entropy_seed_initializes_opt_state_from_tables() { ]; assert_ne!( - hc.backend.bt_mut().opt_state.lit_length_freq, - base_ll_freqs, + hc.bt.opt_state.lit_length_freq, base_ll_freqs, "dictionary entropy should override fallback LL bootstrap frequencies" ); assert!( - hc.backend - .bt_mut() - .opt_state - .match_length_freq - .iter() - .any(|&v| v != 1), + hc.bt.opt_state.match_length_freq.iter().any(|&v| v != 1), "dictionary entropy should seed non-uniform ML frequencies" ); assert_ne!( - hc.backend.bt_mut().opt_state.off_code_freq[0], - 6, + hc.bt.opt_state.off_code_freq[0], 6, "dictionary entropy should override fallback OF bootstrap frequencies" ); } @@ -1061,7 +1040,7 @@ fn dictionary_fse_seed_applies_without_huffman_seed() { let ml = crate::fse::fse_encoder::default_ml_table(); let of = crate::fse::fse_encoder::default_of_table(); hc.seed_dictionary_entropy(None, Some(&*ll), Some(&*ml), Some(&*of)); - hc.backend.bt_mut().opt_state.rescale_freqs( + hc.bt.opt_state.rescale_freqs( b"abcd", ::ACCURATE_PRICE, ); @@ -1071,22 +1050,15 @@ fn dictionary_fse_seed_applies_without_huffman_seed() { 1, 1, 1, 1, 1, 1, ]; assert_ne!( - hc.backend.bt_mut().opt_state.lit_length_freq, - base_ll_freqs, + hc.bt.opt_state.lit_length_freq, base_ll_freqs, "FSE seed should still override LL bootstrap frequencies without huffman seed" ); assert!( - hc.backend - .bt_mut() - .opt_state - .match_length_freq - .iter() - .any(|&v| v != 1), + hc.bt.opt_state.match_length_freq.iter().any(|&v| v != 1), "FSE seed should still seed non-uniform ML frequencies" ); assert_ne!( - hc.backend.bt_mut().opt_state.off_code_freq[0], - 6, + hc.bt.opt_state.off_code_freq[0], 6, "FSE seed should still override OF bootstrap frequencies without huffman seed" ); } @@ -1105,15 +1077,12 @@ fn dictionary_seed_overrides_predef_price_mode_on_tiny_input() { let ml = crate::fse::fse_encoder::default_ml_table(); let of = crate::fse::fse_encoder::default_of_table(); hc.seed_dictionary_entropy(None, Some(&*ll), Some(&*ml), Some(&*of)); - hc.backend.bt_mut().opt_state.rescale_freqs( + hc.bt.opt_state.rescale_freqs( b"abc", ::ACCURATE_PRICE, ); assert!( - matches!( - hc.backend.bt_mut().opt_state.price_type, - HcOptPriceType::Dynamic - ), + matches!(hc.bt.opt_state.price_type, HcOptPriceType::Dynamic), "dictionary-seeded first block should stay in dynamic mode even for tiny src" ); } @@ -1217,7 +1186,7 @@ fn btultra2_seed_pass_disabled_after_stats_initialized() { super::super::strategy::StrategyTag::BtUltra2, 26, ); - hc.backend.bt_mut().opt_state.lit_length_sum = 1; + hc.bt.opt_state.lit_length_sum = 1; assert!( !hc.should_run_btultra2_seed_pass::( HC_PREDEF_THRESHOLD + 32 @@ -1264,7 +1233,7 @@ fn btultra2_seed_pass_runs_alongside_ldm_candidates() { let block_len = HC_PREDEF_THRESHOLD + 64; hc.table.window_size = block_len; hc.table.chunk_lens.push_back(block_len); - hc.backend.bt_mut().ldm_sequences.push(HcRawSeq { + hc.bt.ldm_sequences.push(HcRawSeq { lit_length: 8, offset: 16, match_length: 32, @@ -1343,48 +1312,45 @@ fn dictionary_huffman_seed_ignored_when_literals_uncompressed() { ); } +/// The optimal search probes the repeat offsets in their literal-length order +/// (upstream `ZSTD_insertBtAndGetAllMatches`, zstd_opt.c:657): after a literal +/// `rep[0]` is the first repeat, while at a literal length of zero it is not a +/// repeat at all and `rep[0] - 1` takes its place. #[test] fn hc_repcode_candidates_respect_litlen_dependent_rep_order() { - let mut hc = HcMatchGenerator::new(64); - hc.table.history = b"xxxxxxABCDEFABCDEF".to_vec().into(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - - let abs_pos = 12usize; // points at second "ABCDEF" - let current_abs_end = hc.table.history.len(); - let reps = [6u32, 3u32, 9u32]; - - let mut lit_pos_candidates = Vec::new(); - with_host_tier!(K => hc.hc.for_each_repcode_candidate_with_reps::( - &hc.table, - abs_pos, - 1, - reps, - current_abs_end, - HC_OPT_MIN_MATCH_LEN, - |c| { - lit_pos_candidates.push(c.offset); - }, - )); + let candidates = |lit_len: usize| { + let mut hc = HcMatchGenerator::new(64); + hc.strategy_tag = crate::encoding::strategy::StrategyTag::BtOpt; + // Only the repeats: no tree compare. + hc.hc.search_depth = 0; + hc.table.search_depth = 0; + // Eight bytes past the searched position, as the parser leaves every + // position it searches. + hc.table.history = b"xxxxxxABCDEFABCDEFzzzzzzzz".to_vec().into(); + hc.table.history_start = 0; + hc.table.history_abs_start = 0; + let abs_pos = 12usize; // the second "ABCDEF" + let current_abs_end = hc.table.history.len(); + let mut out = Vec::new(); + with_host_tier!(K => hc.collect_optimal_candidates::( + abs_pos, + current_abs_end, + usize::MAX / 2, + HcCandidateQuery { + reps: [6, 3, 9], + lit_len, + ldm_candidate: None, + }, + &mut out, + )); + out.iter().map(|c| c.offset).collect::>() + }; assert!( - lit_pos_candidates.contains(&6), + candidates(1).contains(&6), "when lit_len>0, rep0 should be considered and match" ); - - let mut ll0_candidates = Vec::new(); - with_host_tier!(K => hc.hc.for_each_repcode_candidate_with_reps::( - &hc.table, - abs_pos, - 0, - reps, - current_abs_end, - HC_OPT_MIN_MATCH_LEN, - |c| { - ll0_candidates.push(c.offset); - }, - )); assert!( - !ll0_candidates.contains(&6), + !candidates(0).contains(&6), "when lit_len==0, rep0 is not directly eligible (ll0 semantics)" ); } @@ -1579,9 +1545,10 @@ fn hc_collect_optimal_candidates_rep_tail_match_skips_chain_probe() { hc.hc.search_depth = 32; let abs_pos = 6usize; hc.table.ensure_tables(); - hc.table.insert_positions(0, abs_pos); - hc.table.search_depth = 32; + let end = hc.table.history.len(); + with_host_tier!(K => hc.table.bt_update_tree_until::(abs_pos, end)); + let mut out = Vec::new(); with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, @@ -1613,10 +1580,11 @@ fn hc_collect_optimal_candidates_long_chain_match_advances_skip_window() { hc.hc.search_depth = 32; let abs_pos = 9usize; hc.table.ensure_tables(); - hc.table.insert_positions(0, abs_pos); + hc.table.search_depth = 32; + let end = hc.table.history.len(); + with_host_tier!(K => hc.table.bt_update_tree_until::(abs_pos, end)); hc.table.skip_insert_until_abs = 0; - hc.table.search_depth = 32; let mut out = Vec::new(); with_host_tier!(K => hc.collect_optimal_candidates::( abs_pos, @@ -1770,8 +1738,8 @@ fn hc_ldm_candidate_survives_the_search_early_exit() { #[test] fn btultra_and_btultra2_both_keep_dictionary_candidates() { // Routes the BtUltra2 / BtUltra fixture through the production - // `configure()` path so derived state (`hash3_log`, `is_btultra2`, - // `uses_bt`, `backend`) stays consistent — manually flipping the + // `configure()` path so derived state (`hash3_log`, `is_btultra2`) + // stays consistent — manually flipping the // strategy flags here used to leave `hash3_log` / `hash3_table` in // the previous mode's shape and trip the // `Strategy::USE_HASH3 ⇒ hash3_log != 0` debug invariant inside @@ -1802,8 +1770,8 @@ fn btultra_and_btultra2_both_keep_dictionary_candidates() { hc.table.history_abs_start = 0; hc.table.position_base = 0; hc.table.ensure_tables(); - hc.table.insert_positions(0, abs_pos); hc.table.dictionary_limit_abs = Some(64); + // The search threads the tree from here up to `abs_pos` itself. hc.table.skip_insert_until_abs = 0; }; @@ -2487,9 +2455,9 @@ fn driver_dictionary_frame_runs_a_strategy_override() { #[test] fn driver_chain_log_override_survives_row_to_hc_fallback() { - // Regression: when a RowHash level is forced onto the HashChain backend - // (resolved window <= 14, upstream `ZSTD_resolveRowMatchFinderMode`), the - // synthesised HC chain table must honour an explicit `chain_log` override. + // Regression: when a RowHash level searches the hash chain (resolved + // window <= 14, upstream `ZSTD_resolveRowMatchFinderMode`), the chain + // table must honour an explicit `chain_log` override. // The RowHash override arm drops `chain_log` (Row has no chain table), so // the synthesis previously replaced the caller's `chain_log` with the upstream zstd // `hashLog - 1`, silently ignoring it on small-window frames. @@ -2571,6 +2539,70 @@ fn driver_small_source_hint_shrinks_row_hash_tables() { ); } +/// `RowMatchFinderMode` replaces the window rule for the greedy / lazy band: +/// `Disable` searches the chain under a wide window, `Enable` the rows under a +/// narrow one, and a btlazy2 level keeps its tree under either (upstream +/// `ZSTD_rowMatchFinderSupported` covers greedy..lazy2 only). +#[test] +fn row_match_finder_mode_picks_the_lazy_finder() { + use crate::encoding::RowMatchFinderMode; + let finder = |level: i32, hint: u64, mode: RowMatchFinderMode| { + let mut driver = MatchGeneratorDriver::new(32, 2); + driver.set_source_size_hint(hint); + driver.set_param_overrides(Some(super::super::parameters::ParamOverrides { + row_match_finder: mode, + ..Default::default() + })); + driver.reset(CompressionLevel::Level(level)); + let row = driver.row_matcher(); + (row.uses_hash_chain(), row.uses_binary_tree()) + }; + let wide = 1 << 20; + let narrow = 1 << 12; + for level in [5, 7] { + assert_eq!( + finder(level, wide, RowMatchFinderMode::Auto), + (false, false) + ); + assert_eq!( + finder(level, wide, RowMatchFinderMode::Disable), + (true, false) + ); + assert_eq!( + finder(level, narrow, RowMatchFinderMode::Auto), + (true, false) + ); + assert_eq!( + finder(level, narrow, RowMatchFinderMode::Enable), + (false, false) + ); + } + assert_eq!(finder(13, wide, RowMatchFinderMode::Disable), (false, true)); + assert_eq!(finder(13, wide, RowMatchFinderMode::Enable), (false, true)); +} + +/// A dictionary frame searches what its dictionary resolved, under the +/// dictionary's mode (upstream `ZSTD_resetCCtx_usingCDict` takes +/// `cdict->useRowMatchFinder`): a dictionary that forces the chain keeps it on +/// a source wide enough for rows. +#[test] +fn row_match_finder_mode_reaches_a_dictionary_frame() { + use crate::encoding::RowMatchFinderMode; + let finder = |mode: RowMatchFinderMode| { + let mut driver = MatchGeneratorDriver::new(32, 2); + driver.set_source_size_hint(1 << 20); + driver.set_dictionary_size_hint(crate::encoding::DictionarySizes::raw_content(1 << 16)); + driver.set_param_overrides(Some(super::super::parameters::ParamOverrides { + row_match_finder: mode, + ..Default::default() + })); + driver.reset(CompressionLevel::Level(7)); + driver.row_matcher().uses_hash_chain() + }; + assert!(!finder(RowMatchFinderMode::Auto)); + assert!(finder(RowMatchFinderMode::Disable)); +} + /// btlazy2 levels run on the lazy (Row) backend with the binary-tree finder, /// wherever the source-size tier puts strategy 6: L13 at tier 0, L10 at the /// <= 16 KiB tier (where L11 is already btopt, an optimal level on the @@ -2869,12 +2901,13 @@ fn source_hint_clamps_driver_slice_size_to_window() { fn driver_best_to_fastest_releases_oversized_hc_tables() { let mut driver = MatchGeneratorDriver::new(32, 2); - // Initialize at Best routed onto HashChain via the test-only override - // (production `Best` sits on level 13, whose native backend differs) — - // allocates large HC tables (4M hash, 2M chain) so the swap below - // exercises the HC drain path this test pins. - driver.reset_on_hc_lazy(CompressionLevel::Best); - assert_eq!(driver.window_size(), (1u64 << 22)); + // An optimal level allocates large tables on the HashChain backend, so the + // swap below exercises the drain path this test pins. + driver.reset(CompressionLevel::Level(19)); + assert_eq!( + driver.active_backend(), + super::super::strategy::BackendTag::HashChain + ); // Feed data so tables are actually allocated via ensure_tables(). driver.commit_input(b"abcabcabcabc"); @@ -2949,7 +2982,7 @@ fn prime_with_dictionary_applies_offset_history_even_when_content_is_empty() { #[test] fn hc_prime_with_empty_dictionary_disables_btultra2_seed_pass() { let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset_on_hc_lazy(CompressionLevel::Better); + driver.reset(CompressionLevel::Level(19)); driver.prime_with_dictionary(&[], [11, 7, 3]); @@ -3071,7 +3104,7 @@ fn a_btultra2_seed_pass_near_the_top_of_the_address_space_parses_like_a_fresh_on #[test] fn hc_prime_with_dictionary_disables_btultra2_seed_pass() { let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset_on_hc_lazy(CompressionLevel::Better); + driver.reset(CompressionLevel::Level(19)); driver.prime_with_dictionary(b"abcdefgh", [1, 4, 8]); @@ -3899,16 +3932,6 @@ fn hc_hash3_position_matches_hash3_formula() { ); } -#[test] -fn hc_hash_position_matches_hash4_formula() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure(HC_CONFIG, super::super::strategy::StrategyTag::Lazy, 22); - let bytes = *b"abcd"; - let read32 = u32::from_le_bytes(bytes); - let expected = ((read32.wrapping_mul(HC_PRIME4BYTES)) >> (32 - hc.table.hash_log)) as usize; - assert_eq!(hc.table.hash_position(&bytes), expected); -} - #[test] fn btultra2_main_hash_uses_hash4_formula() { let mut hc = HcMatchGenerator::new(1 << 20); @@ -3949,8 +3972,9 @@ fn hc_reset_advances_floor_past_prior_frame_entries() { let mut hc = HcMatchGenerator::new(32); hc.table.commit_input(b"abcdeabcde"); hc.table.ensure_tables(); - // Populate real hash / chain entries for the first frame's positions. - hc.table.insert_positions(0, 6); + hc.table.search_depth = 4; + // Populate real hash / tree entries for the first frame's positions. + with_host_tier!(K => hc.table.bt_update_tree_until::(6, 10)); let prev_end = hc.table.history_abs_end(); assert_eq!(prev_end, 10); assert!(hc.table.hash_table().iter().any(|&v| v != HC_EMPTY)); @@ -4019,52 +4043,6 @@ fn deterministic_high_entropy_bytes(seed: u64, len: usize) -> Vec { out } -#[test] -fn hc_sparse_skip_matching_preserves_tail_cross_block_match() { - let mut matcher = HcMatchGenerator::new(1 << 22); - let tail = b"Qz9kLm2Rp"; - let mut first = deterministic_high_entropy_bytes(0xD1B5_4A32_9C77_0E19, 4096); - let tail_start = first.len() - tail.len(); - first[tail_start..].copy_from_slice(tail); - matcher.table.commit_input(&first); - with_host_tier!(K => matcher.skip_matching::(Some(true))); - - let mut second = tail.to_vec(); - second.extend_from_slice(b"after-tail-literals"); - matcher.table.commit_input(&second); - - let mut first_sequence = None; - matcher.start_matching(|seq| { - if first_sequence.is_some() { - return; - } - first_sequence = Some(match seq { - Sequence::Literals { len } => (len, 0usize, 0usize), - Sequence::Triple { - literal_len, - offset, - match_len, - } => (literal_len, offset, match_len), - }); - }); - - let (literals_len, offset, match_len) = - first_sequence.expect("expected at least one sequence after sparse skip"); - assert_eq!( - literals_len, 0, - "first sequence should start at block boundary" - ); - assert_eq!( - offset, - tail.len(), - "first match should reference previous tail" - ); - assert!( - match_len >= tail.len(), - "tail-aligned cross-block match must be preserved" - ); -} - #[test] fn btultra2_sparse_skip_matching_preserves_tail_cross_block_match() { let mut matcher = HcMatchGenerator::new(1 << 20); @@ -4159,58 +4137,6 @@ fn hc_compact_history_drains_when_threshold_crossed() { assert_eq!(&hc.table.history[..], b"qrstuvwxyz"); } -#[test] -fn hc_insert_position_no_rebase_returns_when_relative_pos_unavailable() { - let mut hc = HcMatchGenerator::new(32); - hc.table.history = b"abcdefghijklmnop".to_vec().into(); - hc.table.history_abs_start = 0; - hc.table.position_base = 1; - hc.table.ensure_tables(); - let before_hash = hc.table.hash_table().to_vec(); - let before_chain = hc.table.chain_table().to_vec(); - - hc.table.insert_position_no_rebase(0); - - assert_eq!(hc.table.hash_table(), before_hash); - assert_eq!(hc.table.chain_table(), before_chain); -} - -#[test] -fn hc_insert_positions_advances_next_to_update3_for_contiguous_range() { - let mut hc = HcMatchGenerator::new(64); - hc.table.history = b"abcdefghijklmnopqrstuvwxyz".to_vec().into(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - hc.table.position_base = 0; - hc.table.ensure_tables(); - hc.table.next_to_update3 = 0; - - hc.table.insert_positions(0, 9); - - assert_eq!( - hc.table.next_to_update3, 9, - "contiguous insert_positions should advance hash3 update cursor" - ); -} - -#[test] -fn hc_insert_positions_with_step_keeps_next_to_update3_cursor_for_sparse_ranges() { - let mut hc = HcMatchGenerator::new(64); - hc.table.history = b"abcdefghijklmnopqrstuvwxyz".to_vec().into(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - hc.table.position_base = 0; - hc.table.ensure_tables(); - hc.table.next_to_update3 = 0; - - hc.table.insert_positions_with_step(0, 16, 4); - - assert_eq!( - hc.table.next_to_update3, 0, - "sparse insert_positions_with_step must not mark skipped positions as hash3-updated" - ); -} - #[test] fn prime_with_dictionary_budget_shrinks_after_dfast_eviction() { let mut driver = MatchGeneratorDriver::new(8, 1); @@ -4242,25 +4168,29 @@ fn prime_with_dictionary_budget_shrinks_after_dfast_eviction() { #[test] fn hc_prime_with_dictionary_preserves_history_for_first_full_block() { - let mut driver = MatchGeneratorDriver::new(8, 1); - // Route onto HashChain explicitly — `Better` resolves to the Row - // backend in production, and this test pins HC dict-prime behaviour. - driver.reset_on_hc_lazy(CompressionLevel::Better); + let mut driver = MatchGeneratorDriver::new(64, 1); + driver.reset(CompressionLevel::Level(19)); - driver.prime_with_dictionary(b"abcdefgh", [1, 4, 8]); + let dict = b"abcdefghijklmnopqrstuvwxyz012345"; + driver.prime_with_dictionary(dict, [1, 4, 8]); - // Repeat the dictionary content so the HC matcher can find it. - // HC_MIN_MATCH_LEN is 5, so an 8-byte match is well above threshold. - driver.commit_input(b"abcdefgh"); + // Repeat the dictionary content so the search can find it, a block long + // enough for the optimal parser to search. + driver.commit_input(dict); + // The parse starts one byte into the block: the attached dictionary leaves + // the prefix starting at the frame's first byte, which the optimal parser + // never searches (upstream zstd_opt.c:1116, `ip += (ip==prefixStart)`). let mut saw_match = false; driver.start_matching(|seq| { if let Sequence::Triple { - literal_len: 0, - offset: 8, + literal_len, + offset, match_len, } = seq - && match_len >= HC_MIN_MATCH_LEN + && literal_len <= 1 + && offset == dict.len() + && literal_len + match_len == dict.len() { saw_match = true; } @@ -4268,14 +4198,14 @@ fn hc_prime_with_dictionary_preserves_history_for_first_full_block() { assert!( saw_match, - "hash-chain backend should match dictionary-primed history in first full block" + "the optimal backend should match dictionary-primed history in first full block" ); } #[test] fn prime_with_dictionary_budget_shrinks_after_hc_eviction() { let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset_on_hc_lazy(CompressionLevel::Better); + driver.reset(CompressionLevel::Level(19)); // Use a small live window so dictionary-primed slices are evicted quickly. driver.hc_matcher_mut().table.max_window_size = 8; driver.reported_window_size = 8; @@ -4313,7 +4243,7 @@ fn resident_reapply_restores_retained_dictionary_budget() { let mut driver = MatchGeneratorDriver::new(1 << 16, 1); let dict = b"abcdefghABCDEFGHijklmnopqrstuvwxyz0123456789"; driver.set_dictionary_size_hint(crate::encoding::DictionarySizes::raw_content(dict.len())); - driver.reset_on_hc_lazy(CompressionLevel::Better); + driver.reset(CompressionLevel::Level(19)); driver.prime_with_dictionary(dict, [1, 4, 8]); let base = driver.reported_window_size; assert!( @@ -4323,7 +4253,7 @@ fn resident_reapply_restores_retained_dictionary_budget() { // Second frame: the reset detects the resident dict and re-borrows it. driver.set_dictionary_size_hint(crate::encoding::DictionarySizes::raw_content(dict.len())); - driver.reset_on_hc_lazy(CompressionLevel::Better); + driver.reset(CompressionLevel::Level(19)); assert!( driver.dictionary_is_resident(), "the second frame must re-borrow the resident dictionary" @@ -4355,7 +4285,7 @@ fn hc_commit_without_eviction_retires_no_dictionary_budget() { // would prematurely retire dictionary budget even when the window is // nowhere near full. let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset_on_hc_lazy(CompressionLevel::Better); + driver.reset(CompressionLevel::Level(19)); // A large live window so a small committed block evicts nothing. driver.hc_matcher_mut().table.max_window_size = 1 << 20; driver.reported_window_size = 1 << 20; @@ -4475,6 +4405,7 @@ fn hc_rebase_rebuilds_only_inserted_prefix() { let mut matcher = HcMatchGenerator::new(64); matcher.table.commit_input(b"abcdeabcdeabcde"); matcher.table.ensure_tables(); + matcher.table.search_depth = 4; matcher.table.position_base = 0; let history_abs_start: usize = match (u64::from(u32::MAX) + 64).try_into() { Ok(value) => value, @@ -4483,15 +4414,20 @@ fn hc_rebase_rebuilds_only_inserted_prefix() { matcher.table.history_abs_start = history_abs_start; let abs_pos = matcher.table.history_abs_start + 6; + // The tree a walk from the rebased origin builds up to `abs_pos`, with the + // compare the rebase replays on. let mut expected = HcMatchGenerator::new(64); expected.table.commit_input(b"abcdeabcdeabcde"); expected.table.ensure_tables(); + expected.table.search_depth = 4; expected.table.history_abs_start = history_abs_start; expected.table.position_base = expected.table.history_abs_start; + expected.table.skip_insert_until_abs = history_abs_start; expected.table.tables.fill(HC_EMPTY); - for pos in expected.table.history_abs_start..abs_pos { - expected.table.insert_position_no_rebase(pos); - } + let end = history_abs_start + expected.table.window_size; + expected + .table + .bt_update_tree_until::(abs_pos, end); matcher.table.maybe_rebase_positions(abs_pos); @@ -4507,7 +4443,7 @@ fn hc_rebase_rebuilds_only_inserted_prefix() { assert_eq!( matcher.table.chain_table(), expected.table.chain_table(), - "future positions must not be pre-seeded into HC chains during rebase" + "future positions must not be pre-seeded into the tree during rebase" ); } @@ -4705,53 +4641,53 @@ fn dfast_inserts_tail_positions_for_next_block_matching() { assert_eq!(history, b"012345bcdeabcdeabcdeab"); } -/// Regression for #49 — locks down `MatchTable::backfill_boundary_positions` -/// for the [`HcMatchGenerator`] lazy path. `backfill_boundary_positions` -/// seeds ONLY the last `< 4` bytes of the previous slice (positions in -/// `[current_abs_start - 3, current_abs_start)`) — the bytes that -/// `insert_position` could not hash at the time because hashing needs -/// 4 bytes of lookahead. The existing 8 MiB window roundtrip test -/// exercises cross-slice behaviour end-to-end, but does not isolate -/// the backfill of those final 1-3 unhashable bytes. +/// Regression for #49 — locks down the boundary backfill of the lazy +/// hash-chain search (the Row backend's chain finder). It seeds the last +/// `< 4` bytes of the previous slice (positions in +/// `[current_abs_start - 3, current_abs_start)`) — the bytes that could not be +/// hashed at the time because hashing needs 4 bytes of lookahead. The +/// existing 8 MiB window roundtrip test exercises cross-slice behaviour +/// end-to-end, but does not isolate the backfill of those final 1-3 +/// unhashable bytes. /// /// Fixture is built so the cross-block match's candidate position /// MUST lie in `[block_1_end - 3, block_1_end)`: /// -/// - Block 1 = `b"PQRSTBCD"` (8 bytes). Block 1's `start_matching` -/// hashes positions 0..=4 (each has 4 bytes of forward context); -/// positions 5/6/7 are the unhashable tail. +/// - Block 1 = `b"PQRSTBCD"` (8 bytes). Block 1's parse hashes positions +/// 0..=4 (each has 4 bytes of forward context); positions 5/6/7 are the +/// unhashable tail. /// - Block 2 = `b"BCDBCDBCDB"` (10 bytes). At absolute position 8 /// (block 2 start) the 4-byte window is `b"BCDB"`. The ONLY place /// `b"BCDB"` was inserted in the hash + chain tables is position 5 -/// — via `backfill_boundary_positions` on the next-slice entry -/// (the 4-byte window at position 5 is `data[5..9] = b"BCD" + -/// block_2[0] = b"BCDB"`). +/// — via the backfill on the next-slice entry (the 4-byte window at +/// position 5 is `data[5..9] = b"BCD" + block_2[0] = b"BCDB"`). /// -/// If `backfill_boundary_positions` regresses, position 5 is never -/// hashed, position 8's lookup misses, and the lazy parser falls -/// through to a leading literals run — `offset == 3, match_len >= 4` -/// would no longer hold. +/// If the backfill regresses, position 5 is never hashed, position 8's +/// lookup misses, and the lazy parser falls through to a leading literals +/// run — `offset == 3, match_len >= 4` would no longer hold. #[test] fn hashchain_inserts_tail_positions_for_next_block_matching() { - let mut matcher = HcMatchGenerator::new(1 << 22); - matcher.configure(HC_CONFIG, super::super::strategy::StrategyTag::Lazy, 22); + // A window of 2^14 searches the hash chain. + let mut matcher = RowMatchGenerator::new(1 << 14); + matcher.configure(ROW_CONFIG); + assert!(matcher.uses_hash_chain()); - matcher.table.commit_input(b"PQRSTBCD"); + matcher.commit_input(b"PQRSTBCD"); let mut history = alloc::vec::Vec::new(); let mut replay = BlockReplay::new(b"PQRSTBCD"); - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { assert!( matches!(seq, Sequence::Literals { .. }), "first block has no internal repeats" ); replay.apply(&mut history, seq); - }); + }))); assert_eq!(history, b"PQRSTBCD"); - matcher.table.commit_input(b"BCDBCDBCDB"); + matcher.commit_input(b"BCDBCDBCDB"); let mut first_sequence_offset: Option = None; let mut first_sequence_match_len: Option = None; - matcher.start_matching(|seq| { + with_host_tier!(K => matcher.start_matching::(dyn_sink(&mut |seq| { if first_sequence_offset.is_some() { return; } @@ -4759,7 +4695,7 @@ fn hashchain_inserts_tail_positions_for_next_block_matching() { Sequence::Literals { .. } => { panic!( "expected tail-anchored cross-block match before any literals — \ - backfill_boundary_positions did not seed positions 5/6/7" + the boundary backfill did not seed positions 5/6/7" ) } Sequence::Triple { @@ -4772,16 +4708,15 @@ fn hashchain_inserts_tail_positions_for_next_block_matching() { first_sequence_match_len = Some(match_len); } } - }); + }))); - let offset = first_sequence_offset.expect( - "expected tail-anchored cross-block match emitted from backfill_boundary_positions", - ); + let offset = first_sequence_offset + .expect("expected tail-anchored cross-block match emitted from the boundary backfill"); assert!( (1..=3).contains(&offset), "boundary match offset {offset} must point into the unhashable tail \ (positions 5/6/7 of an 8-byte block 1) so the test specifically \ - locks down backfill_boundary_positions", + locks down the boundary backfill", ); assert_eq!( offset, 3, diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index 242bd6b8..cd32269c 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -1,19 +1,9 @@ -//! Shared match-finder storage. +//! Match-finder storage of the optimal parsers. //! -//! `MatchTable` owns every byte of state that both the hash-chain (HC / -//! lazy / lazy2) and binary-tree (BT / optimal parser) backends touch: -//! the rolling window, the contiguous `history` mirror, the absolute -//! position cursors, the hash / hash3 / chain (or BT pointer-pair) -//! tables, and the dictionary-priming flags. Both backends operate on -//! the same physical buffers; the only difference is the semantics of -//! `chain_table` entries — HC mode threads single-link chain pointers -//! through it, BT mode lays out pairs of pointers per node — and that -//! interpretation is the matcher's concern, not the table's. -//! -//! Extracted from `HcMatchGenerator` in #111 Phase 1d Stage 1 so the -//! follow-up stages can pull the HC and BT matchers into their own -//! modules without dragging this shared storage around as a forest of -//! `&mut Vec` arguments. +//! `MatchTable` owns every byte of state the binary-tree search touches: the +//! rolling window, the contiguous `history` mirror, the absolute position +//! cursors, the hash / hash3 / binary-tree pointer-pair tables, and the +//! dictionary-priming flags. use alloc::collections::VecDeque; use alloc::vec::Vec; @@ -176,33 +166,24 @@ pub(crate) struct BtCoords { pub(crate) bt_mask: usize, } -/// Shared storage backing every match finder. Holds the contiguous -/// Immutable dictionary match structure (upstream zstd `ZSTD_dictMatchState`) for -/// the binary-tree / optimal path. A hash + single-link chain over the -/// dictionary content concat range, searched in addition to the live BT -/// with its OWN compare budget so dictionary candidates are reachable even -/// when the live tree's budget is spent on recent positions. Slots store -/// `dict_rel + 1` (dictionary-relative concat index); `0` is empty. -/// Which matcher built a [`DmsDictTables`]: the hash-chain (`prime_dms_hc`) and -/// the binary-tree (`prime_dms_bt`) populate `hash_table` / `chain_table` with -/// incompatible meanings, so a cached dms must only be reused by the SAME -/// builder. Part of the cache-reuse key alongside `mls` / `hash_log` / region so -/// a reused compressor that switches level across the HC↔BT boundary (same -/// `mls` / `hash_log`) rebuilds instead of reinterpreting the other layout. +/// Whether a [`DmsDictTables`] holds a built tree; part of the cache-reuse key +/// alongside `mls` / `hash_log` / region. #[derive(Clone, Copy, PartialEq, Eq, Default, Debug)] pub(crate) enum DmsDictLayout { /// Unbuilt / invalidated — never matches a reuse probe. #[default] None, - /// Hash-chain dms (`prime_dms_hc`): single-link chain for the lazy backend. - Hc, /// Binary-tree dms (`prime_dms_bt`): unsorted DUBT for the optimal backend. Bt, } +/// Immutable dictionary match structure (upstream zstd `ZSTD_dictMatchState`) +/// of the optimal path: a binary tree over the dictionary content, searched in +/// addition to the live tree. Slots store `dict_rel + 1` (dictionary-relative +/// concat index); `0` is empty. #[derive(Clone, Default)] pub(crate) struct DmsDictTables { - /// Builder that populated the tables (HC vs BT), part of the reuse key. + /// Whether the tables are built, part of the reuse key. pub(crate) layout: DmsDictLayout, /// Per-hash-bucket binary-tree root (`1 << hash_log` entries), packed /// `dict_rel + 1` (`0` = empty). @@ -220,10 +201,8 @@ pub(crate) struct DmsDictTables { pub(crate) mls: usize, } -/// history buffer, the rolling window, and the hash / chain / hash3 -/// tables. Methods on this struct contain only logic that's identical -/// between HC and BT modes — backend-specific table interpretation -/// lives in the matcher modules. +/// Storage of the optimal parsers' binary-tree search: the contiguous history +/// buffer, the rolling window, and the hash / tree / hash3 tables. pub(crate) struct MatchTable { pub(crate) max_window_size: usize, /// Per-chunk lengths of the live window, in add order. The bytes @@ -291,17 +270,11 @@ pub(crate) struct MatchTable { /// `configure()` so the BT walker and rebase machinery can stay /// on `MatchTable` without consulting the outer generator. pub(crate) is_btultra2: bool, - /// Whether the active backend is one of the BT parsers (`btopt`, - /// `btultra`, `btultra2`). Mirrored from the active strategy for - /// the same reason as `is_btultra2`. - pub(crate) uses_bt: bool, /// Binary-tree finder hash width (`mls`), upstream zstd `ZSTD_hashPtr` mls = - /// `BOUNDED(4, cParams.minMatch, 6)`. clevels.h gives the btlazy2 / btopt - /// band (levels 13-16) `minMatch = 5` → a 5-byte (8-byte read) hash that - /// shortens the chains the BT walks (speed), so it is set per level in - /// `configure`. Only the BT body reads it (the BT hash sites already - /// guarantee 8 readable bytes); the HC `hash_position` stays 4-byte. - /// Defaults to `4`. + /// `BOUNDED(4, cParams.minMatch, 6)`. clevels.h gives btopt level 16 + /// `minMatch = 5` → a 5-byte (8-byte read) hash that shortens the chains + /// the BT walks (speed), so it is set per level in `configure`. The BT hash + /// sites guarantee 8 readable bytes. Defaults to `4`. pub(crate) search_mls: usize, /// Tree coordinates of the block being parsed, taken by /// [`Self::capture_block_coords`] before the parse. @@ -315,15 +288,6 @@ pub(crate) struct MatchTable { /// searched by the BT/optimal collect alongside the live tree. `Some` /// once primed from a non-empty dictionary on a BT level. pub(crate) dms: DictAttach, - /// Borrowed (no-copy) one-shot input window `(ptr, len)`. When set, the - /// scan reads candidate/cursor bytes straight from the caller's slice - /// instead of the owned `history` mirror, so an over-window one-shot - /// input is matched in place (no input->mirror memmove). Raw pointer: - /// live until `clear_borrowed_window` / `reset`. - pub(crate) borrowed_input: Option<(*const u8, usize)>, - /// Active borrowed block range `[start, end)` within `borrowed_input`, - /// staged before each borrowed scan. - pub(crate) borrowed_block: Option<(usize, usize)>, /// Set by [`Self::retire_history`] ahead of a layout and taken by the next /// [`Self::reset`], which would otherwise read the floor off a history /// already cut down to what it keeps. @@ -364,15 +328,12 @@ impl Clone for MatchTable { dict_resident: self.dict_resident, search_depth: self.search_depth, is_btultra2: self.is_btultra2, - uses_bt: self.uses_bt, search_mls: self.search_mls, block_coords: self.block_coords, // Points into this table's history, not the copy's; taken again // when the copy parses a block. block_live: (core::ptr::null(), 0), dms: self.dms.clone(), - borrowed_input: self.borrowed_input, - borrowed_block: self.borrowed_block, retired: self.retired, } } @@ -411,13 +372,10 @@ impl Clone for MatchTable { self.dict_resident = source.dict_resident; self.search_depth = source.search_depth; self.is_btultra2 = source.is_btultra2; - self.uses_bt = source.uses_bt; self.search_mls = source.search_mls; self.block_coords = source.block_coords; // The source's view points into the source's history. self.block_live = (core::ptr::null(), 0); - self.borrowed_input = source.borrowed_input; - self.borrowed_block = source.borrowed_block; self.retired = source.retired; } } @@ -439,16 +397,14 @@ pub(crate) fn dms_hash_log(region: usize, hash_log: usize) -> usize { (usize::BITS - (region - 1).leading_zeros()).clamp(10.min(hash_log), hash_log) as usize } -/// Shared scaffold for the two `prime_dms_*` dictionary-match-state builders. +/// Scaffold of the dictionary-match-state builder. /// -/// Both resolve the dict `region` (capped to the live history), bail on an -/// undersized dict, derive the dict-sized hash log, short-circuit when a -/// same-shape table is already primed, recycle the prior tables' capacity, run -/// the layout-specific `$build` over the dict bytes at the front of the live -/// history, then publish the tables and mark the cache primed. Only the build -/// pass (HC single-link chain vs BT binary tree), the minimum readable region, -/// and the chain entries per dict position differ — those are the macro -/// parameters. `$build` sees `concat` (the dict bytes), `&mut` `hash_table` / +/// Resolves the dict `region` (capped to the live history), bails on an +/// undersized dict, derives the dict-sized hash log, short-circuits when a +/// same-shape table is already primed, recycles the prior tables' capacity, +/// runs `$build` over the dict bytes at the front of the live history, then +/// publishes the tables and marks the cache primed. `$build` sees `concat` +/// (the dict bytes), `&mut` `hash_table` / /// `chain_table`, `dms_hash_log`, and `region` in scope; `mls` and any /// build-only widths come from the caller's surrounding locals. macro_rules! build_dms { @@ -545,13 +501,10 @@ impl MatchTable { dict_resident: false, search_depth: 0, is_btultra2: false, - uses_bt: false, search_mls: 4, block_coords: BtCoords::default(), block_live: (core::ptr::null(), 0), dms: DictAttach::new(), - borrowed_input: None, - borrowed_block: None, retired: None, } } @@ -646,15 +599,8 @@ impl MatchTable { return; } let idx = abs_pos - self.history_abs_start; - // Borrowed-aware: hash the SAME bytes the finder reads via - // `live_history()` (borrowed window in no-copy mode, owned mirror - // otherwise). Reading `self.history[history_start..]` directly hashed - // the EMPTY owned mirror on the borrowed one-shot path, so every hash3 - // insert early-returned and the HC3 side table stayed empty — btultra2 - // short-match probing then found nothing. This is the identical bug - // already fixed for `insert_position_no_rebase`; the hash3 inserter was - // missed. `hash3` is the only value needed past the borrow, so it ends - // before the table write. + // `hash3` is the only value needed past the borrow, so it ends before + // the table write. let hash3 = { let concat = self.live_history(); if idx + 4 > concat.len() { @@ -668,60 +614,12 @@ impl MatchTable { self.hash3_table_mut()[hash3] = relative_pos + 1; } - /// Insert a position into the main hash / chain table without - /// running the rebase check. Caller pre-validates that the - /// position is representable as a `(rel + 1)` u32, either via - /// `maybe_rebase_positions` (HC) or `bt_update_tree_until` (BT). - /// Upstream zstd parity: `ZSTD_insertAndFindFirstIndex` inner body. - #[inline] - pub(crate) fn insert_position_no_rebase(&mut self, abs_pos: usize) { - let idx = abs_pos.wrapping_sub(self.history_abs_start); - // Borrowed-aware: read the SAME bytes the finder hashes via - // `live_history()` (borrowed window in no-copy mode, owned mirror - // otherwise). Reading `self.history[history_start..]` directly would - // hash the empty owned mirror on the borrowed HC lazy/greedy path - // (live=input, mirror=0) — the insert would skip or hash garbage while - // the finder hashes the real input, so the chain stays empty and no - // matches are ever found. `hash` is the only value needed past this - // borrow, so it ends before the table mutation below. - let hash = { - let concat = self.live_history(); - if idx + 4 > concat.len() { - return; - } - Self::hash_position_at(concat, idx, self.hash_log, 4) - }; - let Some(relative_pos) = self.relative_position(abs_pos) else { - return; - }; - let stored = relative_pos + 1; - let chain_mask = (1usize << self.chain_log) - 1; - let chain_idx = relative_pos as usize & chain_mask; - // SAFETY: `hash` is produced by `hash_value_with_mls` which masks - // the result down to `hash_log` bits, and `hash_table.len() == 1 << - // hash_log` (`ensure_tables`). `chain_idx` is `& chain_mask` so - // `< chain_table.len() == 1 << chain_log`. Both indices are - // provably in bounds, so the elided bounds checks save ~4 - // instructions per call on this per-byte-of-input hot path. - debug_assert!(hash < self.chain_off); - debug_assert!(chain_idx < self.hash3_off - self.chain_off); - // Both regions come from the one buffer; the split ends with this - // statement, so it costs the same two base pointers the separate - // vectors did. - let (hash_tbl, chain_tbl) = self.hash_and_chain_mut(); - unsafe { - let prev = *hash_tbl.get_unchecked(hash); - *chain_tbl.get_unchecked_mut(chain_idx) = prev; - *hash_tbl.get_unchecked_mut(hash) = stored; - } - } - - /// Allocate the hash / chain / hash3 tables sized to the current + /// Allocate the hash / tree / hash3 tables sized to the current /// `hash_log` / `chain_log` / `hash3_log` configuration. Each table /// is reallocated whenever its current length doesn't match the - /// width implied by its `*_log` field — `insert_position_no_rebase` - /// reaches via `get_unchecked`, so a stale-width table after a - /// level change would index out of bounds (UB) on the next encode. + /// width implied by its `*_log` field — the tree walk reaches them + /// through raw pointers, so a stale-width table after a level change + /// would index out of bounds (UB) on the next encode. pub(crate) fn ensure_tables(&mut self) { let (hash_size, chain_size, total) = self.table_sizes(); // The three regions share one buffer, so a width change on any of them @@ -921,13 +819,6 @@ impl MatchTable { Self::hash_value_with_mls(value, hash_log, mls) } - /// Main hash for the current matcher (4-byte MLS, `hash_log` from - /// the table's configuration). - #[inline(always)] - pub(crate) fn hash_position(&self, data: &[u8]) -> usize { - Self::hash_position_with_mls(data, self.hash_log, 4) - } - /// 3-byte hash used by the HC3 side table. Test-only — the /// production path uses inlined per-kernel variants. #[cfg(test)] @@ -967,8 +858,8 @@ impl MatchTable { /// position (`[smaller_child, larger_child]`), `hash_table` the per-bucket /// tree roots. `dict_rel + 1` packing, `0` sentinel. Dict positions are /// concat indices at the front of the shared buffer, so the walk's offset is - /// `idx - dict_idx` directly (no upstream zstd `dmsIndexDelta`). Called on BT levels - /// (`uses_bt`); cached across frames via `DictAttach::is_primed`. + /// `idx - dict_idx` directly (no upstream zstd `dmsIndexDelta`). Cached across + /// frames via `DictAttach::is_primed`. pub(crate) fn prime_dms_bt(&mut self, region_len: usize) { // Upstream zstd `ZSTD_dictMatchState` searches the dict at the live cParams // `minMatch` (= our `search_mls`), NOT a fixed 3 — a fixed 3 surfaces @@ -1044,44 +935,6 @@ impl MatchTable { ); } - /// Build a SEPARATE dictionary match state for the lazy hash-chain path - /// (upstream `ZSTD_dictMatchState`, the dms HC4 walk in - /// `ZSTD_HcFindBestMatch` `zstd_lazy.c:748-770`) over the first `region_len` - /// bytes of the live history (the dictionary at the front). Unlike - /// [`Self::prime_dms_bt`] this is a single-link hash chain (one `next` entry - /// per dict position, like the live HC chain), not a binary tree: the lazy - /// parser walks it with a bounded compare budget rather than descending to - /// the longest match. `hash_table` holds per-bucket chain heads, packed - /// `dict_rel + 1` (`0` = empty); `chain_table[dict_rel]` is the previous - /// dict position in the same bucket, same packing. Dict positions are concat - /// indices at the front of the shared buffer, so the offset is `idx - - /// dict_idx` directly. Hashed at `HC_MIN_MATCH_LEN` (4), matching the live - /// chain. Cached across frames via `DictAttach::is_primed`. - pub(crate) fn prime_dms_hc(&mut self, region_len: usize) { - let mls = HC_MIN_MATCH_LEN; - let floor = self.dict_index_floor; - // Single `next` link per dict position (HC4 chain, not the BT's 2/pos). - build_dms!( - self, - region_len, - DmsDictLayout::Hc, - mls, - mls, - 1, - |concat, hash_table, chain_table, dms_hash_log, region| { - let mut current = floor; - while current + mls <= region { - let h = Self::hash_position_at(concat, current, dms_hash_log, mls); - // Prepend `current` to its bucket chain (upstream zstd - // `ZSTD_insertAndFindFirstIndex` head insert). - chain_table[current] = hash_table[h]; - hash_table[h] = (current + 1) as u32; - current += 1; - } - } - ); - } - /// Heap bytes this table owns: history, the hash / hash3 / chain tables, /// the chunk-length deque, and any attached immutable dictionary tables. pub(crate) fn heap_size(&self) -> usize { @@ -1237,21 +1090,6 @@ impl MatchTable { /// `history` Vec. #[inline] pub(crate) fn live_history(&self) -> &[u8] { - // Borrowed one-shot: expose `[0, block_end)` of the caller's input so - // candidate/cursor reads land in place (the owned `history` mirror is - // empty under the no-copy path). Loop-invariant branch, inlines. - if let Some((_start, end)) = self.borrowed_block { - let (ptr, total) = self - .borrowed_input - .expect("borrowed_block set without a registered borrowed window"); - debug_assert!( - end <= total, - "borrowed block end {end} exceeds window {total}" - ); - // SAFETY: `ptr` is the registered window start (live by contract); - // `end <= total` in bounds. - return unsafe { core::slice::from_raw_parts(ptr, end) }; - } // Stop at the committed end, not at `history.len()`: the next block's // bytes may already be read into the tail, and a scan must not see // past the block it is compressing. @@ -1267,65 +1105,20 @@ impl MatchTable { /// the most recent buffer in the rolling window — panics if no /// data has been committed yet. pub(crate) fn get_last_space(&self) -> &[u8] { - if let (Some((ptr, _total)), Some((block_start, block_end))) = - (self.borrowed_input, self.borrowed_block) - { - // SAFETY: borrowed liveness contract; range validated when staged. - return unsafe { - core::slice::from_raw_parts(ptr.add(block_start), block_end - block_start) - }; - } let last = *self.chunk_lens.back().unwrap(); &self.history[self.history.len() - self.uncommitted_len - last ..self.history.len() - self.uncommitted_len] } - /// Register the borrowed input window. Zeroes `history_abs_start` so - /// borrowed positions are absolute input offsets (the owned floor-advance - /// reset leaves it non-zero across frames; a borrowed frame commits - /// nothing, so it stays 0 for the frame). Every probed candidate is byte-verified, so - /// stale table entries from a prior frame are rejected (or, if their bytes - /// coincidentally match, are genuine in-window matches). - /// - /// # Safety - /// `buffer` must stay live and unmodified until `clear_borrowed_window` - /// or `reset`. - pub(crate) unsafe fn set_borrowed_window(&mut self, buffer: &[u8]) { - self.borrowed_input = Some((buffer.as_ptr(), buffer.len())); - self.borrowed_block = None; - self.history_abs_start = 0; - } - - pub(crate) fn clear_borrowed_window(&mut self) { - self.borrowed_input = None; - self.borrowed_block = None; - } - - /// Stage `[block_start, block_end)` as the active borrowed block. - pub(crate) fn stage_borrowed_block(&mut self, block_start: usize, block_end: usize) { - let (_ptr, total) = self - .borrowed_input - .expect("stage_borrowed_block requires a registered borrowed window"); - assert!( - block_start <= block_end && block_end <= total, - "borrowed block bounds out of range: start={block_start} end={block_end} total={total}", - ); - self.borrowed_block = Some((block_start, block_end)); - } - - /// `(current_abs_start, current_len)` for the active scan. Borrowed: the - /// staged block range. Owned: the last committed chunk in the live window, - /// or an empty range at the window's end when none has been committed. + /// `(current_abs_start, current_len)` for the active scan: the last + /// committed chunk in the live window, or an empty range at the window's + /// end when none has been committed. pub(crate) fn current_block_range(&self) -> (usize, usize) { - if let Some((start, end)) = self.borrowed_block { - (start, end - start) - } else { - let current_len = self.chunk_lens.back().copied().unwrap_or(0); - ( - self.history_abs_start + self.window_size - current_len, - current_len, - ) - } + let current_len = self.chunk_lens.back().copied().unwrap_or(0); + ( + self.history_abs_start + self.window_size - current_len, + current_len, + ) } /// Test-only: append a chunk to the live window without eviction or @@ -1587,10 +1380,6 @@ impl MatchTable { self.window_size = 0; self.offset_hist = [1, 4, 8]; self.skip_insert_until_abs = 0; - // Clear borrowed-window state so a following OWNED frame reads the - // owned mirror; a borrowed frame re-arms via `set_borrowed_window`. - self.borrowed_input = None; - self.borrowed_block = None; self.dictionary_primed_for_frame = false; if let Some(region) = reborrow_region { // Keep `[0, region)` (the dict); drop the previous frame's input. @@ -2138,11 +1927,9 @@ impl MatchTable { } } - /// Cold rebase: clear the hash / hash3 / chain tables and replay - /// the inserted history prefix through the active backend's walker - /// so the new `position_base` is consistent. The `uses_bt` flag - /// (mirrored from `HcParseMode`) selects between the HC and BT - /// replay variants. + /// Cold rebase: clear the hash / hash3 / tree tables and replay the + /// inserted history prefix through the tree walk so the new + /// `position_base` is consistent. #[cold] #[inline(never)] pub(crate) fn rebase_positions_cold(&mut self, abs_pos: usize) { @@ -2155,14 +1942,10 @@ impl MatchTable { // below every position insert, and the tiers' compares return the same // lengths, so the tree is the same and only this rare pass's speed // depends on it. - if self.uses_bt { - self.replay_history_for_rebase_bt::( - history_start, - abs_pos, - ); - } else { - self.replay_history_for_rebase_hc(history_start, abs_pos); - } + self.replay_history_for_rebase_bt::( + history_start, + abs_pos, + ); // begin_rebase() also clears `hash3_table`. Rewind `next_to_update3` // to the inserted prefix start so `update_hash3_until` re-fills the // HC3 side table — without this, every HC3 probe up to `abs_pos` @@ -2174,164 +1957,9 @@ impl MatchTable { } } - /// Insert a single position into the hash / chain tables, rebasing - /// first if required. - #[inline] - pub(crate) fn insert_position(&mut self, abs_pos: usize) { - self.maybe_rebase_positions(abs_pos); - self.insert_position_no_rebase(abs_pos); - } - - /// Insert every position in `[start, end)` into the hash / chain - /// tables and advance the hash3 fill cursor past `end`. - /// - /// The rebase guard is hoisted out of the per-position loop: a single - /// `(rel + 1)`-representability check on both ends of the range decides - /// whether the whole span fits without a rebase. The check is monotone - /// in `pos` (rebase only fires as the relative position approaches - /// `u32::MAX`), so when neither end needs a rebase no interior position - /// can either, and the - /// fill runs as a tight loop with raw `(pos - position_base)` index - /// arithmetic and hoisted base pointers. This mirrors the upstream zstd's - /// once-per-block `ZSTD_window_correctOverflow` followed by an - /// unchecked fill, and matters on highly repetitive inputs where a - /// single long match makes this loop fill the entire block. When a - /// rebase is required the cold per-position path runs unchanged. - pub(crate) fn insert_positions(&mut self, start: usize, end: usize) { - if start < end { - if self.needs_rebase(start) || self.needs_rebase(end - 1) { - // Cold path: at least one position in the range needs a - // rebase. Defer to the guarded per-position insert, which - // rebases exactly when each position requires it. - for pos in start..end { - self.insert_position(pos); - } - } else { - self.fill_hash_chain_positions(start, end); - } - } - self.next_to_update3 = self.next_to_update3.max(end); - } - - /// Index a just-emitted match span with the upstream zstd skip-threshold cap - /// (`ZSTD_row_update_internal`, `zstd_lazy.c:922-940`): when the span - /// exceeds `SKIP_THRESHOLD` positions only the first `MAX_START` and last - /// `MAX_END` are chained, the interior is skipped. Indexing every interior - /// byte of a long match is O(matchlen) and dominates HashChain (lazy) - /// encode time on periodic inputs where one match can span a whole block; - /// upstream zstd's hash-chain finder chains nothing in the interior at all, so the - /// 96 + 32 cap is a conservative (ratio-preserving) mirror that still keeps - /// boundary anchors for the following search. - pub(crate) fn insert_match_span(&mut self, start: usize, end: usize) { - const SKIP_THRESHOLD: usize = 384; - const MAX_START: usize = 96; - const MAX_END: usize = 32; - if end.saturating_sub(start) > SKIP_THRESHOLD { - // Raw arithmetic is correct by design here, NOT masked with - // `saturating_*`. `start` / `end` are absolute stream - // positions, and `check_stream_abs_headroom` guarantees - // `abs_pos + STREAM_ABS_HEADROOM (= 4112) <= usize::MAX` for - // every position in the frame, so `start + MAX_START` (96) - // cannot overflow even on a 32-bit target. In this branch - // `end - start > SKIP_THRESHOLD (384)`, so `end > 384 > - // MAX_END (32)` and `end - MAX_END` cannot underflow. - self.insert_positions(start, start + MAX_START); - self.insert_positions(end - MAX_END, end); - } else { - self.insert_positions(start, end); - } - } - - /// Tight hash/chain fill for `[start, end)` when the caller has already - /// proven every position is `(rel + 1)`-representable (so no rebase can - /// occur). Equivalent to looping - /// [`Self::insert_position_no_rebase`], but with the table base pointers - /// and config hoisted out of the loop and the relative position derived - /// by a raw subtraction instead of the checked `relative_position` - /// arithmetic. Upstream zstd parity: the unchecked `ZSTD_insertAndFindFirstIndex` - /// fill body. - #[inline] - fn fill_hash_chain_positions(&mut self, start: usize, end: usize) { - let history_abs_start = self.history_abs_start; - let position_base = self.position_base; - let index_shift = self.index_shift; - let hash_log = self.hash_log; - let chain_mask = (1usize << self.chain_log) - 1; - // Borrowed-aware source: `live_history()` is the borrowed input window - // in no-copy mode (owned mirror empty) and `history[history_start..]` - // otherwise. Reading `self.history` directly would hash the empty - // mirror on the borrowed HC lazy/greedy path. The raw ptr is copied - // out so it holds no borrow across the `hash_table` / `chain_table` - // mutations below; the window stays valid for the loop (borrowed input - // is live by contract, owned `history` is not realloc'd mid-fill). - let (concat_ptr, concat_len) = { - let live = self.live_history(); - (live.as_ptr(), live.len()) - }; - debug_assert!(self.chain_off == 1usize << hash_log); - debug_assert!(self.hash3_off - self.chain_off == 1usize << self.chain_log); - // Base pointers into the one buffer, hoisted before the loop below. - // SAFETY: `chain_off` is in bounds (one past the hash region is the - // first chain slot), asserted just above. - let hash_ptr = self.tables.as_mut_ptr(); - let chain_ptr = unsafe { hash_ptr.add(self.chain_off) }; - // The last `< 4` bytes of the live window can't be hashed. Compute - // the hashable upper bound once instead of branching per position: a - // position `pos` is hashable iff `pos - history_abs_start + 4 <= - // concat_len`, i.e. `pos < history_abs_start + (concat_len - 3)`. - let hashable_end = end.min(history_abs_start + concat_len.saturating_sub(3)); - if start >= hashable_end { - return; - } - // Hoist the source pointer and relative index out of the loop and - // advance both by one per iteration, mirroring the upstream zstd's - // `ip++ / idx++` fill rather than recomputing them from `pos`. - let mut src = unsafe { concat_ptr.add(start - history_abs_start) }; - // `rel` cannot reach `u32::MAX` because the caller proved - // `!needs_rebase(end - 1)`, and the `wrapping_add` below keeps the - // overflow branch off this per-byte hot loop. The distance from the - // floor comes first so the sum stays in range on a 32-bit word. - let mut rel = (start - position_base + index_shift) as u32; - for _ in start..hashable_end { - // SAFETY: every `src` in `[start, hashable_end)` is at least 4 - // bytes from the end of `history`, so the unaligned 4-byte read is - // in range. `hash` is masked to `hash_log` bits and `chain_idx` to - // `chain_log` bits, both within the table lengths asserted above. - unsafe { - let value = Self::read_le_u32_ptr(src); - let hash = Self::hash_value_with_mls(value, hash_log, 4); - let chain_idx = (rel as usize) & chain_mask; - let prev = *hash_ptr.add(hash); - *chain_ptr.add(chain_idx) = prev; - *hash_ptr.add(hash) = rel + 1; - src = src.add(1); - } - rel = rel.wrapping_add(1); - } - } - - /// Insert every `step`-th position in `[start, end)` — the sparse - /// counterpart to [`Self::insert_positions`]. Skipped positions are - /// *not* advanced through `next_to_update3` (the upstream zstd's behaviour - /// for the "incompressible block" skip path). - pub(crate) fn insert_positions_with_step(&mut self, start: usize, end: usize, step: usize) { - if step == 0 { - return; - } - let mut pos = start; - while pos < end { - self.insert_position(pos); - let next = pos.saturating_add(step); - if next <= pos { - break; - } - pos = next; - } - } - /// Backfill the last `< 4` bytes of the previous slice (which couldn't - /// be hashed at the time because `insert_position` needs 4 bytes of - /// lookahead) before the next slice's matching pass. + /// be hashed at the time, a position needs 4 bytes of lookahead) before + /// the next slice's matching pass. pub(crate) fn backfill_boundary_positions( &mut self, current_abs_start: usize, @@ -2341,11 +1969,7 @@ impl MatchTable { .saturating_sub(3) .max(self.history_abs_start); if backfill_start < current_abs_start { - if self.uses_bt { - self.bt_update_tree_until::(current_abs_start, current_abs_end); - } else { - self.insert_positions(backfill_start, current_abs_start); - } + self.bt_update_tree_until::(current_abs_start, current_abs_end); } } @@ -2353,16 +1977,13 @@ impl MatchTable { /// stretch of `skip_insert_until_abs`. Cap the gap at 384 to bound /// the worst-case BT tree update on the next match search. pub(crate) fn apply_limited_update_after_long_match(&mut self, current_abs_start: usize) { - if !self.uses_bt { - return; - } let gap = current_abs_start.saturating_sub(self.skip_insert_until_abs); if gap > 384 { self.skip_insert_until_abs = current_abs_start - (gap - 384).min(192); } } - /// BT-mode counterpart of the HC sparse skip path. Inserts every + /// Sparse skip of an incompressible block. Inserts every /// `INCOMPRESSIBLE_SKIP_STEP`-th position through the BT walker /// (which threads the binary tree) and then densely inserts the /// final `HC_MIN_MATCH_LEN + INCOMPRESSIBLE_SKIP_STEP` positions so @@ -2402,9 +2023,6 @@ impl MatchTable { self.next_to_update3 = self.next_to_update3.max(current_abs_end); } - /// `skip_matching` body — backfill the slice boundary and then walk - /// the current slice in either dense or sparse mode (driven by the - /// `incompressible_hint`). BT and HC modes branch via `uses_bt`. /// Dict-priming for BT / optimal levels (upstream zstd `ZSTD_dictMatchState`): /// the dictionary content stays in `history` (so the dms chain can read it /// and offsets are computed across the dict→input boundary) but is NOT @@ -2450,28 +2068,16 @@ impl MatchTable { self.uncommitted_len += len; } - /// Index the last committed block as the body of a long-distance match on - /// the lazy chain: only its tail, which is what upstream's lazy search - /// catches up on after one (`ZSTD_ldm_limitTableUpdate` moves - /// `nextToUpdate` to it), since the match itself is never searched. + /// Index the last committed block as the body of a long-distance match. + /// The tree gets here only behind a producer, which parses in place of the + /// optimal parser: it resumes near the body's end, as after a long match + /// of its own (`ZSTD_ldm_limitTableUpdate`). #[cfg(feature = "ldm")] - pub(crate) fn skip_long_match(&mut self) { + pub(crate) fn skip_long_match(&mut self) { self.ensure_tables(); let (start, len) = self.current_block_range(); - let end = start + len; - if self.uses_bt { - // The tree gets here only behind a producer, which parses in place - // of the optimal parser: it resumes near the body's end, as after - // a long match of its own. - self.apply_limited_update_after_long_match(end); - self.next_to_update3 = self.next_to_update3.max(self.skip_insert_until_abs); - return; - } - self.backfill_boundary_positions::(start, end); - self.insert_positions( - end - len.min(crate::encoding::ldm::LONG_MATCH_INDEXED_TAIL), - end, - ); + self.apply_limited_update_after_long_match(start + len); + self.next_to_update3 = self.next_to_update3.max(self.skip_insert_until_abs); } /// Carry a long-distance match the driver emitted into the repeat @@ -2494,33 +2100,11 @@ impl MatchTable { let floor = (self.history_abs_start + self.dict_index_floor).min(current_abs_end); let current_abs_start = current_abs_start.max(floor); self.backfill_boundary_positions::(current_abs_start, current_abs_end); - if self.uses_bt { - if incompressible_hint == Some(true) { - self.bt_insert_sparse_incompressible_block::(current_abs_start, current_abs_end); - return; - } - self.bt_update_tree_until::(current_abs_end, current_abs_end); - return; - } if incompressible_hint == Some(true) { - self.insert_positions_with_step( - current_abs_start, - current_abs_end, - INCOMPRESSIBLE_SKIP_STEP, - ); - let dense_tail = HC_MIN_MATCH_LEN + INCOMPRESSIBLE_SKIP_STEP; - let tail_start = current_abs_end - .saturating_sub(dense_tail) - .max(self.history_abs_start); - let tail_start = tail_start.max(current_abs_start); - for pos in tail_start..current_abs_end { - if !(pos - current_abs_start).is_multiple_of(INCOMPRESSIBLE_SKIP_STEP) { - self.insert_position(pos); - } - } - } else { - self.insert_positions(current_abs_start, current_abs_end); + self.bt_insert_sparse_incompressible_block::(current_abs_start, current_abs_end); + return; } + self.bt_update_tree_until::(current_abs_end, current_abs_end); } /// Reset the rebase-derived bookkeeping (rolling `position_base` / @@ -2536,17 +2120,6 @@ impl MatchTable { self.tables.fill(HC_EMPTY); } - /// HC-side history replay after [`Self::begin_rebase`]. Re-inserts every - /// position from `history_start` (inclusive) to `abs_pos` - /// (exclusive) into the HC chain/hash tables without re-checking - /// the rebase guard — the caller has just rebased, so positions - /// are by construction representable. - pub(crate) fn replay_history_for_rebase_hc(&mut self, history_start: usize, abs_pos: usize) { - for pos in history_start..abs_pos { - self.insert_position_no_rebase(pos); - } - } - /// Upstream zstd parity: replay an optimal-parser plan into the consumer's /// sequence sink over a block of `current_len` bytes, advancing /// `offset_hist` exactly like the upstream zstd block-store walker. diff --git a/zstd/src/encoding/match_table/storage/storage_tests.rs b/zstd/src/encoding/match_table/storage/storage_tests.rs index 8117211d..cc2099ea 100644 --- a/zstd/src/encoding/match_table/storage/storage_tests.rs +++ b/zstd/src/encoding/match_table/storage/storage_tests.rs @@ -37,50 +37,11 @@ fn set_dictionary_limit_from_primed_bytes_offsets_from_history_start() { assert_eq!(t.dictionary_limit_abs, Some(140)); } -#[test] -fn dms_cache_rebuilds_across_hc_bt_layout_switch() { - // A reused compressor that changes level across the HC↔BT boundary lands - // back in prime_dms_* with the SAME (region, mls, hash_log) but the OTHER - // builder. Without a layout discriminator in the cache key, the second - // prime would reuse the first builder's tables verbatim (HC single-link - // chain reinterpreted as a BT DUBT, or vice versa) — a silent corruption. - // The cache key must include the layout so the switch forces a rebuild. - let mut t = new_table(64); - // dms_hash_log clamps to [10, hash_log]; keep hash_log above the floor. - t.hash_log = 12; - // Match HC's fixed mls (HC_MIN_MATCH_LEN == 4) so the BT prime resolves - // the SAME (region, mls, hash_log) as the HC prime — then the LAYOUT field - // is the only differing key, which is exactly what this test guards. With - // search_mls != 4 the rebuild would happen via the mls mismatch and the - // test would pass even without the layout discriminator. - t.search_mls = 4; - t.push_test_chunk(vec![7u8; 48]); - t.ensure_tables(); - let region = 48; - - t.prime_dms_hc(region); - assert_eq!(t.dms.table().unwrap().layout, DmsDictLayout::Hc); - // HC chain has one `next` per dict position. - assert_eq!(t.dms.table().unwrap().chain_table.len(), region); - - // Same region/mls/hash_log, but the BT builder must NOT reuse the HC - // tables: it rebuilds to the BT layout (2 children per dict position). - t.prime_dms_bt(region); - assert_eq!(t.dms.table().unwrap().layout, DmsDictLayout::Bt); - assert_eq!(t.dms.table().unwrap().chain_table.len(), 2 * region); - - // And back to HC rebuilds again. - t.prime_dms_hc(region); - assert_eq!(t.dms.table().unwrap().layout, DmsDictLayout::Hc); - assert_eq!(t.dms.table().unwrap().chain_table.len(), region); -} - #[test] fn skip_matching_bt_incompressible_routes_through_sparse_block() { let mut t = new_table(32); t.push_test_chunk(vec![0u8; 32]); t.ensure_tables(); - t.uses_bt = true; t.is_btultra2 = false; t.search_depth = 4; let before_skip_until = t.skip_insert_until_abs; @@ -97,7 +58,6 @@ fn skip_matching_bt_dense_routes_through_bt_update_tree() { let mut t = new_table(32); t.push_test_chunk(vec![1u8; 32]); t.ensure_tables(); - t.uses_bt = true; t.is_btultra2 = false; t.search_depth = 4; // `incompressible_hint = None` → dense bt_update_tree_until path @@ -119,7 +79,6 @@ fn replay_history_for_rebase_bt_walks_inserted_prefix() { t.window_size = 64; t.position_base = 0; t.search_depth = 4; - t.uses_bt = true; t.ensure_tables(); // Replay the first 32 positions; the BT walker writes entries // into the hash table (via `hash_table[hash] = stored`) so the @@ -346,7 +305,6 @@ fn bt_table() -> MatchTable { t.window_size = 64; t.position_base = 0; t.search_depth = 4; - t.uses_bt = true; t.ensure_tables(); t } @@ -400,58 +358,11 @@ fn a_tree_walk_past_the_index_range_rebases_then_matches_a_plain_walk() { ); } -#[test] -fn insert_positions_with_step_zero_step_is_noop() { - let mut t = new_table(32); - t.history = vec![0u8; 32].into(); - t.push_test_chunk(vec![0u8; 32]); - t.ensure_tables(); - let next_to_update3_before = t.next_to_update3; - // step=0 must early-return without touching anything. - t.insert_positions_with_step(0, 16, 0); - assert!(t.hash_table().iter().all(|&v| v == HC_EMPTY)); - assert_eq!(t.next_to_update3, next_to_update3_before); -} - -#[test] -fn insert_positions_with_step_saturating_step_breaks_loop() { - // step = usize::MAX so first iteration overflows - // `pos.saturating_add(step)` to usize::MAX, then the `next <= pos` - // guard breaks out of the loop after one insert. - let mut t = new_table(32); - t.history = vec![1u8; 32].into(); - t.push_test_chunk(vec![1u8; 32]); - t.ensure_tables(); - t.insert_positions_with_step(0, 16, usize::MAX); - // Exactly one position should have been inserted before the - // loop terminated — observe that only one slot is non-empty. - let non_empty = t.hash_table().iter().filter(|&&v| v != HC_EMPTY).count(); - assert!( - non_empty <= 1, - "step=usize::MAX must break after the first insert" - ); -} - -#[test] -fn apply_limited_update_after_long_match_hc_mode_is_noop() { - // HC mode (`uses_bt = false`) — function must early-return - // without mutating `skip_insert_until_abs`. - let mut t = new_table(32); - t.uses_bt = false; - t.skip_insert_until_abs = 100; - t.apply_limited_update_after_long_match(1000); - assert_eq!( - t.skip_insert_until_abs, 100, - "HC mode must not adjust skip cursor" - ); -} - #[test] fn apply_limited_update_after_long_match_bt_mode_caps_gap_at_384() { - // BT mode with gap > 384 → cap the skip cursor so future - // `bt_update_tree_until` doesn't walk an unbounded prefix. + // A gap > 384 → cap the skip cursor so future `bt_update_tree_until` + // doesn't walk an unbounded prefix. let mut t = new_table(32); - t.uses_bt = true; t.skip_insert_until_abs = 0; // current_abs_start = 1000 → gap = 1000 → cap subtracts // (gap - 384).min(192) = 192, so result is 1000 - 192 = 808. @@ -462,7 +373,6 @@ fn apply_limited_update_after_long_match_bt_mode_caps_gap_at_384() { #[test] fn apply_limited_update_after_long_match_small_gap_is_noop() { let mut t = new_table(32); - t.uses_bt = true; t.skip_insert_until_abs = 800; // gap = 200 < 384 → no change. t.apply_limited_update_after_long_match(1000); diff --git a/zstd/src/encoding/mod.rs b/zstd/src/encoding/mod.rs index 7c8d45f1..e2d40846 100644 --- a/zstd/src/encoding/mod.rs +++ b/zstd/src/encoding/mod.rs @@ -70,6 +70,7 @@ pub(crate) mod bt; pub(crate) mod cost_model; pub(crate) mod dfast; pub(crate) mod hc; +#[cfg(test)] pub(crate) mod lazy_parse; // LDM hashes each `min_match_length` window with XXH64 (upstream zstd // `zstd_ldm.c:315`), so the `ldm` feature implies `hash` and the @@ -111,7 +112,7 @@ pub use levels::config::{ pub use match_generator::MatchGeneratorDriver; pub use parameters::{ Bounds, CParameter, CompressionParameters, CompressionParametersBuilder, LevelParameters, - LiteralCompressionMode, ParameterError, Strategy, + LiteralCompressionMode, ParameterError, RowMatchFinderMode, Strategy, }; pub use streaming_encoder::{CompressionContext, StreamingEncoder}; pub use workspace::HistoryBuf; diff --git a/zstd/src/encoding/parameters.rs b/zstd/src/encoding/parameters.rs index 024307fd..b9b1d9aa 100644 --- a/zstd/src/encoding/parameters.rs +++ b/zstd/src/encoding/parameters.rs @@ -154,6 +154,35 @@ pub enum LiteralCompressionMode { Disable, } +/// Which finder the greedy, lazy and lazy2 strategies search, the drop-in +/// equivalent of C zstd's `ZSTD_c_useRowMatchFinder` (`ZSTD_ps_auto` / +/// `ZSTD_ps_enable` / `ZSTD_ps_disable`). The other strategies have a finder +/// of their own and ignore it. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub enum RowMatchFinderMode { + /// Rows above a 16 KiB window, a hash chain at or below it, where the rows + /// cost more to set up than they save (upstream + /// `ZSTD_resolveRowMatchFinderMode`, zstd_compress.c:246). + #[default] + Auto, + /// Search rows at every window size. + Enable, + /// Search a hash chain at every window size. + Disable, +} + +impl RowMatchFinderMode { + /// Whether a greedy / lazy / lazy2 frame with a `window`-byte window + /// searches rows. + pub(crate) const fn searches_rows(self, window: usize) -> bool { + match self { + Self::Auto => window > (1 << 14), + Self::Enable => true, + Self::Disable => false, + } + } +} + /// One tunable compression parameter — the analogue of a C zstd /// `ZSTD_cParameter`. Used to query bounds via [`CParameter::bounds`]. #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -435,6 +464,8 @@ pub(crate) struct ParamOverrides { pub(crate) ldm: Option, /// Whether literals are entropy-coded; `Auto` leaves it to the level. pub(crate) literal_compression: LiteralCompressionMode, + /// The greedy / lazy band's finder; `Auto` leaves it to the window. + pub(crate) row_match_finder: RowMatchFinderMode, } impl ParamOverrides { @@ -451,6 +482,7 @@ impl ParamOverrides { && self.strategy.is_none() && self.ldm.is_none() && self.literal_compression == LiteralCompressionMode::Auto + && self.row_match_finder == RowMatchFinderMode::Auto } } @@ -483,6 +515,7 @@ impl CompressionParameters { enable_ldm: false, ldm: LdmOverride::default(), literal_compression: LiteralCompressionMode::Auto, + row_match_finder: RowMatchFinderMode::Auto, } } @@ -501,6 +534,11 @@ impl CompressionParameters { self.overrides.literal_compression } + /// Which finder the greedy / lazy band searches (see [`RowMatchFinderMode`]). + pub fn row_match_finder_mode(&self) -> RowMatchFinderMode { + self.overrides.row_match_finder + } + /// The window, `log2`, the encoder gives a frame of `source_size` bytes /// (`None` = unknown) with `dictionary_size` bytes of dictionary content /// under these parameters: resolved exactly as the encoder resolves it. @@ -580,9 +618,37 @@ pub struct CompressionParametersBuilder { enable_ldm: bool, ldm: LdmOverride, literal_compression: LiteralCompressionMode, + row_match_finder: RowMatchFinderMode, } impl CompressionParametersBuilder { + /// Choose the finder the greedy, lazy and lazy2 strategies search instead + /// of leaving it to the window size. C `ZSTD_c_useRowMatchFinder`. + /// + /// ```rust + /// use structured_zstd::encoding::{ + /// compress_with_parameters, CompressionLevel, CompressionParameters, + /// RowMatchFinderMode, + /// }; + /// use structured_zstd::decoding::FrameDecoder; + /// + /// let data = b"a row or a chain, the frame decodes the same ".repeat(2000); + /// let chain = CompressionParameters::builder(CompressionLevel::Level(7)) + /// .row_match_finder(RowMatchFinderMode::Disable) + /// .build() + /// .unwrap(); + /// let frame = compress_with_parameters(&data[..], &chain); + /// let mut decoded = Vec::with_capacity(data.len()); + /// FrameDecoder::new() + /// .decode_all_to_vec(&frame, &mut decoded) + /// .unwrap(); + /// assert_eq!(decoded, data); + /// ``` + pub fn row_match_finder(mut self, mode: RowMatchFinderMode) -> Self { + self.row_match_finder = mode; + self + } + /// Decide whether literals are entropy-coded, overriding the level's own /// choice. C `ZSTD_c_literalCompressionMode`. /// @@ -741,6 +807,7 @@ impl CompressionParametersBuilder { strategy: self.strategy, ldm, literal_compression: self.literal_compression, + row_match_finder: self.row_match_finder, }, }) } diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index d686dcc1..0bd6215c 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -2785,16 +2785,17 @@ impl RowMatchGenerator { self.row_hash_mls = self.mls.clamp(4, 6) as u32; // A btlazy2 level (its own or the CDict's strategy) searches the // binary tree. Otherwise upstream `ZSTD_resolveRowMatchFinderMode`: - // rows only above a 2^14 window; the window is source-size-adjusted, - // so inputs of 16 KiB or less search the hash chain. A dictionary - // frame inherits the CDict's decision (`ZSTD_resetCCtx_usingCDict`), - // and a copied dictionary brings the CDict's salt (0) with its tables. + // the mode the caller set, or rows only above a 2^14 window; the + // window is source-size-adjusted, so inputs of 16 KiB or less search + // the hash chain. A dictionary frame inherits the CDict's decision + // (`ZSTD_resetCCtx_usingCDict`), and a copied dictionary brings the + // CDict's salt (0) with its tables. self.finder = if config.bt { LazyFinder::Tree } else { let rows = match self.dict_plan { Some(plan) => plan.use_row, - None => self.max_window_size > (1usize << 14), + None => config.searches_rows(self.max_window_size), }; if rows { LazyFinder::Rows diff --git a/zstd/src/encoding/strategy.rs b/zstd/src/encoding/strategy.rs index 32ef81d3..097cc535 100644 --- a/zstd/src/encoding/strategy.rs +++ b/zstd/src/encoding/strategy.rs @@ -12,9 +12,7 @@ //! Matcher::start_matching // 7-arm match on StrategyTag (per block) //! └─ compress_block:: // S::BACKEND const match //! ├─ Simple/Dfast/Row // backends without parse_mode -//! └─ HcMatchGenerator::start_matching_strategy:: -//! ├─ S::USE_BT == false → start_matching_lazy -//! └─ S::USE_BT == true → start_matching_optimal:: +//! └─ HcMatchGenerator::start_matching_optimal:: //! ├─ HcOptimalCostProfile::const_for_strategy::() //! ├─ should_run_btultra2_seed_pass:: // const false unless S = BtUltra2 //! └─ select_kernel() once, then one pass over the block: @@ -48,7 +46,7 @@ pub(crate) enum BackendTag { Dfast, /// `RowMatchGenerator` — level 4. Row, - /// `HcMatchGenerator` — levels 5-22. + /// `HcMatchGenerator` — the optimal parsers (btopt, btultra, btultra2). HashChain, } @@ -93,19 +91,17 @@ impl ParseMode { } /// Search method — how match candidates are produced for a position. -/// Orthogonal to [`ParseMode`] (upstream zstd `cParams.strategy` search half plus -/// the `useRowMatchFinder` cParam that swaps `HashChain` for `RowHash` in -/// the greedy/lazy band). +/// Orthogonal to [`ParseMode`] (upstream zstd `cParams.strategy` search half). #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub(crate) enum SearchMethod { /// Single-table fast finder (upstream zstd `ZSTD_fast`). Fast, /// Two parallel hash tables (upstream zstd `ZSTD_dfast`). DoubleFast, - /// Row-hash SIMD-tag finder (upstream zstd row matchfinder). + /// The greedy / lazy / lazy2 finders: rows or the hash chain + /// (`ZSTD_RowFindBestMatch` / `ZSTD_HcFindBestMatch`, chosen by + /// `useRowMatchFinder`), searched by one lazy parse. RowHash, - /// Hash-chain finder (upstream zstd `ZSTD_HcFindBestMatch`). - HashChain, /// Binary-tree finder for the optimal parser (upstream zstd `ZSTD_BtGetAllMatches`). BinaryTree, /// Lazily-sorted binary-tree finder inside the lazy parse (upstream zstd @@ -115,15 +111,14 @@ pub(crate) enum SearchMethod { impl SearchMethod { /// Storage backend (matcher variant) this search method runs in. - /// `BinaryTree` shares the `HashChain` storage (the BT scratch lives - /// inside the hash-chain matcher, gated by `uses_bt`); `BinaryTreeLazy` - /// is the third finder of the lazy (Row) backend. + /// `BinaryTree` runs in the `HashChain` storage, whose tables hold the + /// tree; `BinaryTreeLazy` is the third finder of the lazy (Row) backend. pub(crate) const fn backend(self) -> BackendTag { match self { Self::Fast => BackendTag::Simple, Self::DoubleFast => BackendTag::Dfast, Self::RowHash | Self::BinaryTreeLazy => BackendTag::Row, - Self::HashChain | Self::BinaryTree => BackendTag::HashChain, + Self::BinaryTree => BackendTag::HashChain, } } }