perf(encode): bring the level-1 entropy stage to libzstd's pace

zstd/src/encoding/blocks/compressed.rs

@@ -731,8 +731,45 @@ fn builtin_block_sequences<K: EncodeTier>(
     code_offsets: bool,
     hist: &mut [u32; 3],
 ) {
-    driver
-        .start_matching_with::<K>(|seq| record_sequence(seq, tail, sequences, code_offsets, hist));
+    driver.start_matching_with::<K>(|seq| {
+        record_builtin_sequence(seq, tail, sequences, code_offsets, hist)
+    });
+}
+
+/// [`record_sequence`] for the built-in matcher, whose reports need no
+/// checking: it reports the block's trailing literals last, and a block of at
+/// most 128 KiB bounds every length far below 32 bits. Both hold by
+/// construction of the driver and are asserted in debug builds; a custom
+/// matcher's reports go through the checked path.
+#[inline(always)]
+fn record_builtin_sequence(
+    seq: Sequence,
+    tail: &mut usize,
+    sequences: &mut RegionVec<RawSequence>,
+    code_offsets: bool,
+    hist: &mut [u32; 3],
+) {
+    match seq {
+        Sequence::Literals { len } => *tail += len,
+        Sequence::Triple {
+            literal_len,
+            offset,
+            match_len,
+        } => {
+            debug_assert_eq!(*tail, 0, "literals reported before a sequence");
+            debug_assert!((literal_len | match_len | offset) <= u32::MAX as usize);
+            let off_base = if code_offsets {
+                encode_offset_with_history_fast(offset as u32, literal_len as u32, hist)
+            } else {
+                offset as u32
+            };
+            sequences.push(RawSequence {
+                ll: literal_len as u32,
+                ml: match_len as u32,
+                off_base,
+            });
+        }
+    }
 }
 
 /// Append a block's parse to `capture` as external sequences with plain

zstd/src/encoding/blocks/compressed/literal_runs.rs

@@ -100,19 +100,27 @@ macro_rules! gather_body {
                 // SAFETY (both arms): `written <= pos`, and the capacity check in
                 // `gather_literals` gives the buffer `src_len` bytes past
                 // `start`. The chunked arm reads and writes at most
-                // `len + $chunk - 1 < len + WILDCOPY_SLACK <= room` bytes from
-                // `pos` / `written`; the exact arm, `len <= room` bytes.
+                // `max($chunk, len + $chunk - 1) <= len + WILDCOPY_SLACK <= room`
+                // bytes from `pos` / `written` (`$chunk <= WILDCOPY_SLACK`); the
+                // exact arm, `len <= room` bytes.
                 unsafe {
                     let s = src.add(pos);
                     let d = out.add(written);
                     if room >= WILDCOPY_SLACK && len <= room - WILDCOPY_SLACK {
-                        // A repcode match right after the last one has no
-                        // literals, and the wild copy would still run its loop
-                        // checks for none (skipping it measured -1.0% at level
-                        // 1 and -0.5% at level 4 on decodecorpus z000033).
+                        // Upstream's shape (`ZSTD_wildcopy`): one chunk
+                        // unconditionally, which is the whole run for most of
+                        // them, and a loop only for the rest of a longer one.
+                        // The first chunk is a constant-length copy, a single
+                        // load and store, so an empty run costs less to copy
+                        // than to test for.
                         if len < LITERAL_INLINE_COPY_MAX {
-                            if len != 0 {
-                                $wild(s, d, len.next_multiple_of($chunk));
+                            $wild(s, d, $chunk);
+                            if len > $chunk {
+                                $wild(
+                                    s.add($chunk),
+                                    d.add($chunk),
+                                    (len - $chunk).next_multiple_of($chunk),
+                                );
                             }
                         } else {
                             core::ptr::copy_nonoverlapping(s, d, len);

zstd/src/encoding/blocks/compressed/tests.rs

@@ -638,11 +638,11 @@ fn discarded_block_keeps_the_entry_tables() {
         },
     );
     assert_eq!(
-        state.last_huff_table.as_ref().unwrap().packed_codes(),
+        state.last_huff_table.as_ref().unwrap().packed_codes()[..],
         entry_codes
     );
     assert_eq!(
-        state.huff_table_spare.as_ref().unwrap().packed_codes(),
+        state.huff_table_spare.as_ref().unwrap().packed_codes()[..],
         built_codes
     );
     let previous = [

zstd/src/fse/fse_encoder.rs

@@ -149,7 +149,10 @@ pub(crate) struct SymbolTT {
     /// fixed-point math. Upstream zstd `fse_compress.c` uses `U32` throughout
     /// the same arithmetic for the same reason.
     pub(crate) delta_nb_bits: u32,
-    pub(crate) delta_find_state: isize,
+    /// `i32` as upstream's `int deltaFindState`: its magnitude is under the
+    /// 512-entry table, and the pair then packs into 8 bytes, half a cache line
+    /// for two symbols where an `isize` padded it to 16.
+    pub(crate) delta_find_state: i32,
 }
 
 /// Largest next-state table any FSE encoder table can carry: the accuracy log
@@ -237,10 +240,20 @@ impl FSETable {
         let tt = self.symbol_tt[symbol as usize];
         let value = (self.table_size + idx) as u32;
         let nb_bits = ((value + tt.delta_nb_bits) >> 16) as usize;
-        let slot = ((value >> nb_bits) as isize + tt.delta_find_state) as usize;
+        let slot = ((value >> nb_bits) as i32 + tt.delta_find_state) as usize;
+        // Inside `0..table_size` by construction, as upstream's
+        // `FSE_encodeSymbol` relies on: the transition of a state of a present
+        // symbol lands on one of that symbol's own slots.
+        debug_assert!(
+            slot < self.table_size,
+            "next-state slot {slot} outside a {}-entry table",
+            self.table_size
+        );
+        // SAFETY: `slot < table_size <= MAX_FSE_TABLE_SIZE`, asserted above
+        // and held by the build.
         State {
             num_bits: nb_bits as u8,
-            index: self.state_table_flat[slot] as usize,
+            index: unsafe { *self.state_table_flat.get_unchecked(slot) } as usize,
         }
     }
 
@@ -444,7 +457,7 @@ pub fn build_table_from_data(
     build_table_from_counts(&counts[..=max_symbol], max_log, avoid_0_numbit)
 }
 
-#[cfg(any(test, feature = "fuzz-exports"))]
+#[cfg(test)]
 pub(crate) fn build_table_from_symbol_counts(
     counts: &[usize],
     max_log: u8,
@@ -1048,18 +1061,20 @@ pub(super) fn build_table_from_probabilities_into(probs: &[i32], acc_log: u8, ou
             };
             continue;
         }
-        let (delta_nb_bits, delta_find_state): (u32, isize) = match prob {
+        // `total` and `prob` are at most the 512-entry table, so the
+        // difference fits `i32`.
+        let (delta_nb_bits, delta_find_state): (u32, i32) = match prob {
             -1 | 1 => (
                 ((acc_log as u32) << 16).saturating_sub(1u32 << acc_log),
-                total as isize - 1,
+                total as i32 - 1,
             ),
             p if p > 1 => {
                 let p_u32 = p as u32;
                 let max_bits_out = (acc_log as u32) - (p_u32 - 1).ilog2();
                 let min_state_plus = p_u32 << max_bits_out;
                 (
                     (max_bits_out << 16).saturating_sub(min_state_plus),
-                    total as isize - p_u32 as isize,
+                    total as i32 - p,
                 )
             }
             _ => unreachable!("probability is one of {{-1, 1+}} after the prob==0 gate above"),
@@ -1075,7 +1090,7 @@ pub(super) fn build_table_from_probabilities_into(probs: &[i32], acc_log: u8, ou
         // where `nbBitsOut = (deltaNbBits + (1<<15)) >> 16`.
         let init_nb_bits_out = (delta_nb_bits + (1 << 15)) >> 16;
         let init_value = (init_nb_bits_out << 16).saturating_sub(delta_nb_bits);
-        let state_table_index = (init_value >> init_nb_bits_out) as isize + delta_find_state;
+        let state_table_index = (init_value >> init_nb_bits_out) as i32 + delta_find_state;
         // Upstream zstd `FSE_initCState2` guarantees this index is in
         // `0..table_size` by construction (`delta_find_state` is bounded
         // by `total - probability`, and `(value >> nb_bits_out)` is

zstd/src/huff0/huf_cstream.rs

@@ -196,30 +196,29 @@ macro_rules! encode_unrolled_body {
         }
         debug_assert!(n.is_multiple_of(2 * $ku));
 
-        // Phase 3: dual-container main loop.
-        while n > 0 {
+        // Phase 3: dual-container main loop over `data[..n]` from the end,
+        // `2 * K_UNROLL` symbols at a time. A chunk of constant length lets
+        // every symbol load go unchecked, as upstream's `ip[n - u]` does.
+        for chunk in $data[..n].rchunks_exact(2 * $ku) {
             for u in 1..$ku {
-                huf_add!(bc0, bp0, $table[$data[n - u] as usize], true);
+                huf_add!(bc0, bp0, $table[chunk[2 * $ku - u] as usize], true);
             }
-            huf_add!(bc0, bp0, $table[$data[n - $ku] as usize], $klf);
+            huf_add!(bc0, bp0, $table[chunk[$ku] as usize], $klf);
             huf_flush!(bc0, bp0, cursor, overflow, out_base, end_ptr, $kff);
 
             bc1 = 0;
             bp1 = 0;
             for u in 1..$ku {
-                huf_add!(bc1, bp1, $table[$data[n - $ku - u] as usize], true);
+                huf_add!(bc1, bp1, $table[chunk[$ku - u] as usize], true);
             }
-            huf_add!(bc1, bp1, $table[$data[n - $ku - $ku] as usize], $klf);
+            huf_add!(bc1, bp1, $table[chunk[0] as usize], $klf);
             // merge_index1: fold container 1 into container 0.
             let nb_bits_1 = bp1 & 0xFF;
             bc0 >>= nb_bits_1;
             bc0 |= bc1;
             bp0 = bp0.wrapping_add(bp1);
             huf_flush!(bc0, bp0, cursor, overflow, out_base, end_ptr, $kff);
-
-            n -= 2 * $ku;
         }
-        debug_assert_eq!(n, 0);
 
         // Write the hoisted state back so `close()` sees the final values.
         $self.bit_container[0] = bc0;
@@ -363,7 +362,7 @@ impl<'a> HufCStream<'a> {
         const K_LAST_FAST: bool,
     >(
         &mut self,
-        table: &[u64],
+        table: &[u64; 256],
         data: &[u8],
     ) {
         crate::encoding::fastpath::tier_match!(K {
@@ -390,7 +389,7 @@ impl<'a> HufCStream<'a> {
         const K_LAST_FAST: bool,
     >(
         &mut self,
-        table: &[u64],
+        table: &[u64; 256],
         data: &[u8],
     ) {
         encode_unrolled_body!(self, table, data, K_UNROLL, K_FAST_FLUSH, K_LAST_FAST);
@@ -413,7 +412,7 @@ impl<'a> HufCStream<'a> {
         const K_LAST_FAST: bool,
     >(
         &mut self,
-        table: &[u64],
+        table: &[u64; 256],
         data: &[u8],
     ) {
         encode_unrolled_body!(self, table, data, K_UNROLL, K_FAST_FLUSH, K_LAST_FAST);

zstd/src/huff0/huff0_encoder.rs

@@ -253,7 +253,7 @@ impl<V: AsMut<Vec<u8>>> HuffmanEncoder<'_, '_, V> {
     /// `kLastFast` matching upstream zstd's 64-bit branch in
     /// `huf_compress.c:1092-1110`.
     fn encode_one_stream<K: EncodeTier>(
-        table: &[u64],
+        table: &[u64; 256],
         bit_c: &mut super::huf_cstream::HufCStream<'_>,
         data: &[u8],
         table_log: u32,
@@ -308,7 +308,7 @@ impl<V: AsMut<Vec<u8>>> HuffmanEncoder<'_, '_, V> {
         const K_FAST_FLUSH: bool,
         const K_LAST_FAST: bool,
     >(
-        table: &[u64],
+        table: &[u64; 256],
         bit_c: &mut super::huf_cstream::HufCStream<'_>,
         data: &[u8],
     ) {
@@ -539,6 +539,7 @@ pub struct HuffmanTable {
     /// dual-container [`super::huf_cstream::HufCStream`] can index
     /// symbols with a single u64 load (no per-symbol shift+combine).
     /// See `huf_compress.c:208-221` for the upstream zstd format.
+    /// Always 256 entries, absent symbols zero (see [`Self::packed_codes`]).
     packed_codes: Vec<u64>,
     /// Active Huffman table-log (1..=12). Stored explicitly so
     /// `encode4x_reference` can dispatch to the correct `kUnroll`
@@ -1077,8 +1078,8 @@ impl HuffmanTable {
                 spare.codes.reserve_exact(weights.len());
                 spare.codes.resize(weights.len(), (0, 0));
                 spare.packed_codes.clear();
-                spare.packed_codes.reserve_exact(weights.len());
-                spare.packed_codes.resize(weights.len(), 0u64);
+                spare.packed_codes.reserve_exact(256);
+                spare.packed_codes.resize(256, 0u64);
                 spare.table_log = table_log as u32;
                 #[cfg(feature = "std")]
                 {
@@ -1091,7 +1092,7 @@ impl HuffmanTable {
             }
             None => HuffmanTable {
                 codes: alloc::vec![(0, 0); weights.len()],
-                packed_codes: alloc::vec![0u64; weights.len()],
+                packed_codes: alloc::vec![0u64; 256],
                 table_log: table_log as u32,
                 #[cfg(feature = "std")]
                 cached_encoded_weight_description: WeightDescription::default(),
@@ -1126,11 +1127,15 @@ impl HuffmanTable {
     }
 
     /// Upstream zstd-format packed code table for the hot encode loop.
-    /// One `HUF_CElt` (`u64`) per symbol — see
-    /// `huf_compress.c:208-221` for the layout.
+    /// One `HUF_CElt` (`u64`) per byte value, absent symbols zero — see
+    /// `huf_compress.c:208-221` for the layout. A full 256 entries, as
+    /// upstream's `CTable`, so a symbol byte indexes it without a check.
     #[inline(always)]
-    pub(crate) fn packed_codes(&self) -> &[u64] {
-        &self.packed_codes
+    pub(crate) fn packed_codes(&self) -> &[u64; 256] {
+        self.packed_codes
+            .as_slice()
+            .try_into()
+            .expect("the packed code table is built with 256 entries")
     }
 
     /// Active Huffman table-log (1..=12) — drives the `kUnroll`