[pull] master from ruby:master

benchmark/string_tr.yml

@@ -6,9 +6,11 @@ prelude: |
   STR = ((("a" * 31) + "<") * 1000).freeze
 
   ESCAPED_CHARS = {
-    ">" => '\u003e',
-    "<" => '\u003c',
-    "&" => '\u0026',
+    "\\" => "&#39",
+    "\"" => "&quot;",
+    ">" => '&gt;',
+    "<" => '&lt;',
+    "&" => '&amp;',
   }.freeze
   ESCAPE_PATTERN = Regexp.union(ESCAPED_CHARS.keys)
 

internal/simd.h

@@ -0,0 +1,18 @@
+#ifndef INTERNAL_SIMD_H
+#define INTERNAL_SIMD_H
+
+#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64)
+#ifdef HAVE_X86INTRIN_H
+#include <x86intrin.h>
+#define HAVE_SIMD 1
+#define HAVE_SIMD_SSE2 1
+#endif
+#endif
+
+#if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(__aarch64__) || defined(_M_ARM64)
+#define HAVE_SIMD 1
+#define HAVE_SIMD_NEON 1
+#include <arm_neon.h>
+#endif
+
+#endif /* INTERNAL_SIMD_H */

string.c

@@ -39,6 +39,7 @@
 #include "internal/proc.h"
 #include "internal/re.h"
 #include "internal/sanitizers.h"
+#include "internal/simd.h"
 #include "internal/string.h"
 #include "internal/transcode.h"
 #include "probes.h"
@@ -9211,7 +9212,7 @@ tr_buffer_ensure_capa(struct tr_buffer *buffer, size_t extra_capa)
         size_t new_capa = buffer->capa ? buffer->capa : buffer->initial_capa;
         RUBY_ASSERT(new_capa >= 32); // Lower would cause infinite loop
         while (new_capa < required_capa) {
-            new_capa *= 1.2;
+            new_capa = (size_t)(new_capa * 1.2);
         }
         SIZED_REALLOC_N(buffer->buf, unsigned char, new_capa, buffer->capa);
         buffer->ptr = buffer->buf + offset;
@@ -9220,7 +9221,7 @@ tr_buffer_ensure_capa(struct tr_buffer *buffer, size_t extra_capa)
 }
 
 static inline void
-tr_buffer_append(struct tr_buffer *buffer, unsigned char *ptr, size_t len)
+tr_buffer_append(struct tr_buffer *buffer, const unsigned char *ptr, size_t len)
 {
     if (len) {
         tr_buffer_ensure_capa(buffer, len);
@@ -9286,6 +9287,164 @@ tr_trans_pairs_coerce_i(st_data_t key, st_data_t value, st_data_t _args)
     return ST_CONTINUE;
 }
 
+#define TR_TRANS_PAIRS_SIMD_MAX_NEEDLES 16
+
+struct tr_trans_pairs_search {
+    const unsigned char *s;
+    const unsigned char *send;
+
+#ifdef HAVE_SIMD
+    unsigned char needles[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES];
+    int needles_count;
+#ifdef HAVE_SIMD_NEON
+    uint64_t matches_bitmap;
+#endif
+#ifdef HAVE_SIMD_SSE2
+    int matches_bitmap;
+#endif
+#endif
+
+    VALUE trans_table[256];
+};
+
+static inline VALUE
+tr_trans_pairs_search_basic(struct tr_trans_pairs_search *search)
+{
+    while (search->s < search->send) {
+        VALUE repl = search->trans_table[*search->s];
+        if (UNLIKELY(repl)) {
+            return repl;
+        }
+
+        search->s++;
+    }
+
+    return 0;
+}
+
+#ifdef HAVE_SIMD_SSE2
+static inline bool
+tr_trans_pairs_next_match_sse2(struct tr_trans_pairs_search *search)
+{
+    size_t next_match_offset = ntz_int32(search->matches_bitmap);
+    search->matches_bitmap >>= (next_match_offset + 1);
+    search->s += next_match_offset;
+    if (search->s > search->send) {
+        search->s = search->send;
+        return false;
+    }
+    return true;
+}
+
+static inline VALUE
+tr_trans_pairs_search_sse2(struct tr_trans_pairs_search *search)
+{
+    RBIMPL_ASSERT_OR_ASSUME(search->needles_count > 0);
+    RBIMPL_ASSERT_OR_ASSUME(search->needles_count < TR_TRANS_PAIRS_SIMD_MAX_NEEDLES);
+
+    if (search->matches_bitmap) {
+        return tr_trans_pairs_next_match_sse2(search);
+    }
+
+    if ((size_t)(search->send - search->s) >= sizeof(__m128i)) {
+        int i;
+        __m128i masks[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES];
+        for (i = 0; i < search->needles_count; i++) {
+            masks[i] = _mm_set1_epi8(search->needles[i]);
+        }
+
+        do {
+            const __m128i bytes = _mm_loadu_si128((__m128i const *)search->s);
+
+            __m128i matches[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES];
+            for (i = 0; i < search->needles_count; i++) {
+                matches[i] = _mm_cmpeq_epi8(bytes, masks[i]);
+            }
+
+            for (i = i; i < search->needles_count; i++) {
+                matches[0] = _mm_or_si128(matches[0], matches[i]);
+            }
+
+            const int bitmap = _mm_movemask_epi8(matches[0]);
+
+            if (bitmap) {
+                search->matches_bitmap = bitmap;
+                return tr_trans_pairs_next_match_sse2(search);
+            }
+            search->s += sizeof(__m128i);
+        } while ((size_t)(search->send - search->s) >= sizeof(__m128i));
+    }
+    return tr_trans_pairs_search_basic(search);
+}
+
+#define tr_trans_pairs_search_impl tr_trans_pairs_search_sse2
+#endif
+
+#ifdef HAVE_SIMD_NEON
+static inline bool
+tr_trans_pairs_next_match_neon(struct tr_trans_pairs_search *search)
+{
+    size_t next_match_offset = ntz_int64(search->matches_bitmap) / 4;
+    search->matches_bitmap >>= (next_match_offset + 1) * 4;
+    search->s += next_match_offset;
+    if (search->s > search->send) {
+        search->s = search->send;
+        return false;
+    }
+    return true;
+}
+
+static inline VALUE
+tr_trans_pairs_search_neon(struct tr_trans_pairs_search *search)
+{
+    if (search->needles_count) {
+        RBIMPL_ASSERT_OR_ASSUME(search->needles_count > 0);
+        RBIMPL_ASSERT_OR_ASSUME(search->needles_count <= TR_TRANS_PAIRS_SIMD_MAX_NEEDLES);
+
+        if (search->matches_bitmap) {
+            return tr_trans_pairs_next_match_neon(search);
+        }
+
+        if ((size_t)(search->send - search->s) >= sizeof(uint8x16_t)) {
+            int i;
+            uint8x16_t masks[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES];
+            for (i = 0; i < search->needles_count; i++) {
+                masks[i] = vdupq_n_u8(search->needles[i]);
+            }
+
+            do {
+                const uint8x16_t bytes = vld1q_u8(search->s);
+
+                uint8x16_t matches[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES];
+                for (i = 0; i < search->needles_count; i++) {
+                    matches[i] = vceqq_u8(bytes, masks[i]);
+                }
+
+                for (i = i; i < search->needles_count; i++) {
+                    matches[0] = vorrq_u8(matches[0], matches[i]);
+                }
+
+                const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(matches[0]), 4);
+                const uint64_t bitmap = vget_lane_u64(vreinterpret_u64_u8(res), 0) & 0x8888888888888888ull;
+
+                if (bitmap) {
+                    search->matches_bitmap = bitmap;
+                    return tr_trans_pairs_next_match_neon(search);
+                }
+                search->s += sizeof(uint8x16_t);
+            } while ((size_t)(search->send - search->s) >= sizeof(uint8x16_t));
+        }
+    }
+    return tr_trans_pairs_search_basic(search);
+}
+
+#define tr_trans_pairs_search_impl tr_trans_pairs_search_neon
+#endif
+
+#ifndef tr_trans_pairs_search_impl
+#define tr_trans_pairs_search_impl tr_trans_pairs_search_basic
+#endif
+
 static VALUE
 tr_trans_pairs(VALUE str, VALUE pairs_val)
 {
@@ -9311,72 +9470,84 @@ tr_trans_pairs(VALUE str, VALUE pairs_val)
 
     VALUE hash = 0;
 
-    unsigned char *sstart = (unsigned char *)RSTRING_PTR(str);
-    unsigned char *s = sstart;
-    unsigned char *send = sstart + RSTRING_LEN(str);
+    const unsigned char *sstart = (unsigned char *)RSTRING_PTR(str);
+    long str_len = RSTRING_LEN(str);
     int termlen = rb_enc_mbminlen(e1);
 
     struct tr_buffer buffer;
-    tr_buffer_init(&buffer, send - s);
+    tr_buffer_init(&buffer, str_len);
     bool modify = false;
 
     if (RB_LIKELY(rb_str_encindex_fastpath(rb_enc_to_index(e1)))) {
-        VALUE trans_table[256] = { 0 };
+        struct tr_trans_pairs_search search = {
+            .s = sstart,
+            .send = sstart + str_len,
+        };
 
         for (size_t index = 0; index < pairs_count; index++) {
             struct tr_pair *pair = &pairs[index];
 
             char *ptr = RSTRING_PTR(pair->search);
             unsigned int codepoint = rb_enc_mbc_to_codepoint(ptr, RSTRING_END(pair->search), e1);
 
+            const unsigned char first_byte = (unsigned char)*ptr;
+
+#ifdef HAVE_SIMD
+            if (pairs_count <= TR_TRANS_PAIRS_SIMD_MAX_NEEDLES) {
+                search.needles[index] = first_byte;
+                search.needles_count++;
+            }
+#endif
+
             if (rb_enc_codelen(codepoint, e1) == 1) {
-                trans_table[(unsigned char)*ptr] = pair->replace;
+                search.trans_table[first_byte] = pair->replace;
             }
             else {
-                trans_table[(unsigned char)*ptr] = Qundef;
+                search.trans_table[first_byte] = Qundef;
                 if (!hash) {
                     hash = rb_obj_hide(rb_hash_new_capa(pairs_count));
                 }
                 rb_hash_aset(hash, UINT2NUM(codepoint), pair->replace);
             }
         }
 
-        unsigned char *checkpoint = s;
-        while (s < send) {
-            VALUE repl = trans_table[*s];
-
+        const unsigned char *checkpoint = search.s;
+        VALUE repl;
+        while ((repl = tr_trans_pairs_search_impl(&search))) {
             int clen = 1;
 
             if (UNLIKELY(repl == Qundef)) {
-                unsigned int c = rb_enc_mbc_to_codepoint((char *)s, (char *)send, e1);
+                unsigned int c = rb_enc_mbc_to_codepoint((char *)search.s, (char *)search.send, e1);
                 clen = rb_enc_codelen(c, e1);
                 repl = rb_hash_lookup2(hash, UINT2NUM(c), 0);
-            }
-
-            if (LIKELY(repl == 0)) {
-                s += clen;
-                continue;
+                if (!repl) {
+                    search.s += clen;
+                    continue;
+                }
             }
 
             modify = true;
 
-            if (checkpoint < s) {
-                tr_buffer_append(&buffer, checkpoint, s - checkpoint);
+            if (checkpoint < search.s) {
+                tr_buffer_append(&buffer, checkpoint, search.s - checkpoint);
             }
             tr_buffer_append_str(&buffer, repl);
-            s += clen;
-            checkpoint = s;
+            search.s += clen;
+            checkpoint = search.s;
 
             if (cr == ENC_CODERANGE_7BIT && rb_enc_str_coderange(repl) != ENC_CODERANGE_7BIT) {
-                cr == ENC_CODERANGE_VALID;
+                cr = ENC_CODERANGE_VALID;
             }
         }
 
-        if (modify && checkpoint < s) {
-            tr_buffer_append(&buffer, checkpoint, s - checkpoint);
+        if (modify && checkpoint < search.s) {
+            tr_buffer_append(&buffer, checkpoint, search.s - checkpoint);
         }
     }
     else {
+        const unsigned char *s = sstart;
+        const unsigned char *send = sstart + str_len;
+
         hash = rb_obj_hide(rb_hash_new_capa(pairs_count));
 
         for (size_t index = 0; index < pairs_count; index++) {