diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index 7a4d76af..3824222f 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -731,8 +731,45 @@ fn builtin_block_sequences( code_offsets: bool, hist: &mut [u32; 3], ) { - driver - .start_matching_with::(|seq| record_sequence(seq, tail, sequences, code_offsets, hist)); + driver.start_matching_with::(|seq| { + record_builtin_sequence(seq, tail, sequences, code_offsets, hist) + }); +} + +/// [`record_sequence`] for the built-in matcher, whose reports need no +/// checking: it reports the block's trailing literals last, and a block of at +/// most 128 KiB bounds every length far below 32 bits. Both hold by +/// construction of the driver and are asserted in debug builds; a custom +/// matcher's reports go through the checked path. +#[inline(always)] +fn record_builtin_sequence( + seq: Sequence, + tail: &mut usize, + sequences: &mut RegionVec, + code_offsets: bool, + hist: &mut [u32; 3], +) { + match seq { + Sequence::Literals { len } => *tail += len, + Sequence::Triple { + literal_len, + offset, + match_len, + } => { + debug_assert_eq!(*tail, 0, "literals reported before a sequence"); + debug_assert!((literal_len | match_len | offset) <= u32::MAX as usize); + let off_base = if code_offsets { + encode_offset_with_history_fast(offset as u32, literal_len as u32, hist) + } else { + offset as u32 + }; + sequences.push(RawSequence { + ll: literal_len as u32, + ml: match_len as u32, + off_base, + }); + } + } } /// Append a block's parse to `capture` as external sequences with plain diff --git a/zstd/src/encoding/blocks/compressed/literal_runs.rs b/zstd/src/encoding/blocks/compressed/literal_runs.rs index aacd0d85..b36073c9 100644 --- a/zstd/src/encoding/blocks/compressed/literal_runs.rs +++ b/zstd/src/encoding/blocks/compressed/literal_runs.rs @@ -100,19 +100,27 @@ macro_rules! gather_body { // SAFETY (both arms): `written <= pos`, and the capacity check in // `gather_literals` gives the buffer `src_len` bytes past // `start`. The chunked arm reads and writes at most - // `len + $chunk - 1 < len + WILDCOPY_SLACK <= room` bytes from - // `pos` / `written`; the exact arm, `len <= room` bytes. + // `max($chunk, len + $chunk - 1) <= len + WILDCOPY_SLACK <= room` + // bytes from `pos` / `written` (`$chunk <= WILDCOPY_SLACK`); the + // exact arm, `len <= room` bytes. unsafe { let s = src.add(pos); let d = out.add(written); if room >= WILDCOPY_SLACK && len <= room - WILDCOPY_SLACK { - // A repcode match right after the last one has no - // literals, and the wild copy would still run its loop - // checks for none (skipping it measured -1.0% at level - // 1 and -0.5% at level 4 on decodecorpus z000033). + // Upstream's shape (`ZSTD_wildcopy`): one chunk + // unconditionally, which is the whole run for most of + // them, and a loop only for the rest of a longer one. + // The first chunk is a constant-length copy, a single + // load and store, so an empty run costs less to copy + // than to test for. if len < LITERAL_INLINE_COPY_MAX { - if len != 0 { - $wild(s, d, len.next_multiple_of($chunk)); + $wild(s, d, $chunk); + if len > $chunk { + $wild( + s.add($chunk), + d.add($chunk), + (len - $chunk).next_multiple_of($chunk), + ); } } else { core::ptr::copy_nonoverlapping(s, d, len); diff --git a/zstd/src/encoding/blocks/compressed/tests.rs b/zstd/src/encoding/blocks/compressed/tests.rs index aa83bb16..29a2bce0 100644 --- a/zstd/src/encoding/blocks/compressed/tests.rs +++ b/zstd/src/encoding/blocks/compressed/tests.rs @@ -638,11 +638,11 @@ fn discarded_block_keeps_the_entry_tables() { }, ); assert_eq!( - state.last_huff_table.as_ref().unwrap().packed_codes(), + state.last_huff_table.as_ref().unwrap().packed_codes()[..], entry_codes ); assert_eq!( - state.huff_table_spare.as_ref().unwrap().packed_codes(), + state.huff_table_spare.as_ref().unwrap().packed_codes()[..], built_codes ); let previous = [ diff --git a/zstd/src/fse/fse_encoder.rs b/zstd/src/fse/fse_encoder.rs index 22d42a36..f3f35b79 100644 --- a/zstd/src/fse/fse_encoder.rs +++ b/zstd/src/fse/fse_encoder.rs @@ -149,7 +149,10 @@ pub(crate) struct SymbolTT { /// fixed-point math. Upstream zstd `fse_compress.c` uses `U32` throughout /// the same arithmetic for the same reason. pub(crate) delta_nb_bits: u32, - pub(crate) delta_find_state: isize, + /// `i32` as upstream's `int deltaFindState`: its magnitude is under the + /// 512-entry table, and the pair then packs into 8 bytes, half a cache line + /// for two symbols where an `isize` padded it to 16. + pub(crate) delta_find_state: i32, } /// Largest next-state table any FSE encoder table can carry: the accuracy log @@ -237,10 +240,20 @@ impl FSETable { let tt = self.symbol_tt[symbol as usize]; let value = (self.table_size + idx) as u32; let nb_bits = ((value + tt.delta_nb_bits) >> 16) as usize; - let slot = ((value >> nb_bits) as isize + tt.delta_find_state) as usize; + let slot = ((value >> nb_bits) as i32 + tt.delta_find_state) as usize; + // Inside `0..table_size` by construction, as upstream's + // `FSE_encodeSymbol` relies on: the transition of a state of a present + // symbol lands on one of that symbol's own slots. + debug_assert!( + slot < self.table_size, + "next-state slot {slot} outside a {}-entry table", + self.table_size + ); + // SAFETY: `slot < table_size <= MAX_FSE_TABLE_SIZE`, asserted above + // and held by the build. State { num_bits: nb_bits as u8, - index: self.state_table_flat[slot] as usize, + index: unsafe { *self.state_table_flat.get_unchecked(slot) } as usize, } } @@ -444,7 +457,7 @@ pub fn build_table_from_data( build_table_from_counts(&counts[..=max_symbol], max_log, avoid_0_numbit) } -#[cfg(any(test, feature = "fuzz-exports"))] +#[cfg(test)] pub(crate) fn build_table_from_symbol_counts( counts: &[usize], max_log: u8, @@ -1048,10 +1061,12 @@ pub(super) fn build_table_from_probabilities_into(probs: &[i32], acc_log: u8, ou }; continue; } - let (delta_nb_bits, delta_find_state): (u32, isize) = match prob { + // `total` and `prob` are at most the 512-entry table, so the + // difference fits `i32`. + let (delta_nb_bits, delta_find_state): (u32, i32) = match prob { -1 | 1 => ( ((acc_log as u32) << 16).saturating_sub(1u32 << acc_log), - total as isize - 1, + total as i32 - 1, ), p if p > 1 => { let p_u32 = p as u32; @@ -1059,7 +1074,7 @@ pub(super) fn build_table_from_probabilities_into(probs: &[i32], acc_log: u8, ou let min_state_plus = p_u32 << max_bits_out; ( (max_bits_out << 16).saturating_sub(min_state_plus), - total as isize - p_u32 as isize, + total as i32 - p, ) } _ => unreachable!("probability is one of {{-1, 1+}} after the prob==0 gate above"), @@ -1075,7 +1090,7 @@ pub(super) fn build_table_from_probabilities_into(probs: &[i32], acc_log: u8, ou // where `nbBitsOut = (deltaNbBits + (1<<15)) >> 16`. let init_nb_bits_out = (delta_nb_bits + (1 << 15)) >> 16; let init_value = (init_nb_bits_out << 16).saturating_sub(delta_nb_bits); - let state_table_index = (init_value >> init_nb_bits_out) as isize + delta_find_state; + let state_table_index = (init_value >> init_nb_bits_out) as i32 + delta_find_state; // Upstream zstd `FSE_initCState2` guarantees this index is in // `0..table_size` by construction (`delta_find_state` is bounded // by `total - probability`, and `(value >> nb_bits_out)` is diff --git a/zstd/src/huff0/huf_cstream.rs b/zstd/src/huff0/huf_cstream.rs index a8a6acf7..66d21614 100644 --- a/zstd/src/huff0/huf_cstream.rs +++ b/zstd/src/huff0/huf_cstream.rs @@ -196,30 +196,29 @@ macro_rules! encode_unrolled_body { } debug_assert!(n.is_multiple_of(2 * $ku)); - // Phase 3: dual-container main loop. - while n > 0 { + // Phase 3: dual-container main loop over `data[..n]` from the end, + // `2 * K_UNROLL` symbols at a time. A chunk of constant length lets + // every symbol load go unchecked, as upstream's `ip[n - u]` does. + for chunk in $data[..n].rchunks_exact(2 * $ku) { for u in 1..$ku { - huf_add!(bc0, bp0, $table[$data[n - u] as usize], true); + huf_add!(bc0, bp0, $table[chunk[2 * $ku - u] as usize], true); } - huf_add!(bc0, bp0, $table[$data[n - $ku] as usize], $klf); + huf_add!(bc0, bp0, $table[chunk[$ku] as usize], $klf); huf_flush!(bc0, bp0, cursor, overflow, out_base, end_ptr, $kff); bc1 = 0; bp1 = 0; for u in 1..$ku { - huf_add!(bc1, bp1, $table[$data[n - $ku - u] as usize], true); + huf_add!(bc1, bp1, $table[chunk[$ku - u] as usize], true); } - huf_add!(bc1, bp1, $table[$data[n - $ku - $ku] as usize], $klf); + huf_add!(bc1, bp1, $table[chunk[0] as usize], $klf); // merge_index1: fold container 1 into container 0. let nb_bits_1 = bp1 & 0xFF; bc0 >>= nb_bits_1; bc0 |= bc1; bp0 = bp0.wrapping_add(bp1); huf_flush!(bc0, bp0, cursor, overflow, out_base, end_ptr, $kff); - - n -= 2 * $ku; } - debug_assert_eq!(n, 0); // Write the hoisted state back so `close()` sees the final values. $self.bit_container[0] = bc0; @@ -363,7 +362,7 @@ impl<'a> HufCStream<'a> { const K_LAST_FAST: bool, >( &mut self, - table: &[u64], + table: &[u64; 256], data: &[u8], ) { crate::encoding::fastpath::tier_match!(K { @@ -390,7 +389,7 @@ impl<'a> HufCStream<'a> { const K_LAST_FAST: bool, >( &mut self, - table: &[u64], + table: &[u64; 256], data: &[u8], ) { encode_unrolled_body!(self, table, data, K_UNROLL, K_FAST_FLUSH, K_LAST_FAST); @@ -413,7 +412,7 @@ impl<'a> HufCStream<'a> { const K_LAST_FAST: bool, >( &mut self, - table: &[u64], + table: &[u64; 256], data: &[u8], ) { encode_unrolled_body!(self, table, data, K_UNROLL, K_FAST_FLUSH, K_LAST_FAST); diff --git a/zstd/src/huff0/huff0_encoder.rs b/zstd/src/huff0/huff0_encoder.rs index 64a07d4f..913598bd 100644 --- a/zstd/src/huff0/huff0_encoder.rs +++ b/zstd/src/huff0/huff0_encoder.rs @@ -253,7 +253,7 @@ impl>> HuffmanEncoder<'_, '_, V> { /// `kLastFast` matching upstream zstd's 64-bit branch in /// `huf_compress.c:1092-1110`. fn encode_one_stream( - table: &[u64], + table: &[u64; 256], bit_c: &mut super::huf_cstream::HufCStream<'_>, data: &[u8], table_log: u32, @@ -308,7 +308,7 @@ impl>> HuffmanEncoder<'_, '_, V> { const K_FAST_FLUSH: bool, const K_LAST_FAST: bool, >( - table: &[u64], + table: &[u64; 256], bit_c: &mut super::huf_cstream::HufCStream<'_>, data: &[u8], ) { @@ -539,6 +539,7 @@ pub struct HuffmanTable { /// dual-container [`super::huf_cstream::HufCStream`] can index /// symbols with a single u64 load (no per-symbol shift+combine). /// See `huf_compress.c:208-221` for the upstream zstd format. + /// Always 256 entries, absent symbols zero (see [`Self::packed_codes`]). packed_codes: Vec, /// Active Huffman table-log (1..=12). Stored explicitly so /// `encode4x_reference` can dispatch to the correct `kUnroll` @@ -1077,8 +1078,8 @@ impl HuffmanTable { spare.codes.reserve_exact(weights.len()); spare.codes.resize(weights.len(), (0, 0)); spare.packed_codes.clear(); - spare.packed_codes.reserve_exact(weights.len()); - spare.packed_codes.resize(weights.len(), 0u64); + spare.packed_codes.reserve_exact(256); + spare.packed_codes.resize(256, 0u64); spare.table_log = table_log as u32; #[cfg(feature = "std")] { @@ -1091,7 +1092,7 @@ impl HuffmanTable { } None => HuffmanTable { codes: alloc::vec![(0, 0); weights.len()], - packed_codes: alloc::vec![0u64; weights.len()], + packed_codes: alloc::vec![0u64; 256], table_log: table_log as u32, #[cfg(feature = "std")] cached_encoded_weight_description: WeightDescription::default(), @@ -1126,11 +1127,15 @@ impl HuffmanTable { } /// Upstream zstd-format packed code table for the hot encode loop. - /// One `HUF_CElt` (`u64`) per symbol — see - /// `huf_compress.c:208-221` for the layout. + /// One `HUF_CElt` (`u64`) per byte value, absent symbols zero — see + /// `huf_compress.c:208-221` for the layout. A full 256 entries, as + /// upstream's `CTable`, so a symbol byte indexes it without a check. #[inline(always)] - pub(crate) fn packed_codes(&self) -> &[u64] { - &self.packed_codes + pub(crate) fn packed_codes(&self) -> &[u64; 256] { + self.packed_codes + .as_slice() + .try_into() + .expect("the packed code table is built with 256 entries") } /// Active Huffman table-log (1..=12) — drives the `kUnroll`