x86: SSE2/SSSE3 vec_dot for PTQ1_0 and PQ2_0

docs/build.md

@@ -92,6 +92,29 @@ cmake --build build --config Release
   - **Fedora / RHEL / Rocky / Alma:** `sudo dnf install openssl-devel`
   - **Arch / Manjaro:** `sudo pacman -S openssl`
 
+### x86 CPU instruction sets
+
+To build for x86-64 CPUs with SSSE3 and no AVX requirement, use a fresh build directory:
+
+```bash
+cmake -B build-ssse3 -DGGML_NATIVE=OFF -DGGML_SSSE3=ON \
+    -DGGML_SSE42=OFF -DGGML_AVX=OFF -DGGML_AVX2=OFF \
+    -DGGML_FMA=OFF -DGGML_F16C=OFF -DGGML_BMI2=OFF
+cmake --build build-ssse3 --config Release -j 8
+```
+
+SSSE3 is a separate extension from SSE3. Set `GGML_SSSE3=OFF` in this configuration for the x86-64 SSE2 baseline. Avoid `-march=native` in custom compiler flags when targeting older CPUs.
+
+For automatic selection at startup, build the CPU variants as shared backends:
+
+```bash
+cmake -B build-auto -DGGML_NATIVE=OFF -DBUILD_SHARED_LIBS=ON \
+    -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON
+cmake --build build-auto --config Release -j 8
+```
+
+On x86-64, this includes SSE2, SSSE3, SSE4.2, and AVX-family variants. Keep the backend libraries with the executable when distributing the build; the loader selects the highest-ranked variant supported by the CPU.
+
 ## BLAS Build
 
 Building the program with BLAS support may lead to some performance improvements in prompt processing using batch sizes higher than 32 (the default is 512). Using BLAS doesn't affect the generation performance. There are currently several different BLAS implementations available for build and use:

ggml/CMakeLists.txt

@@ -151,6 +151,7 @@ message(DEBUG "INS_ENB             : ${INS_ENB}")
 option(GGML_CPU_HBM          "ggml: use memkind for CPU HBM" OFF)
 option(GGML_CPU_REPACK       "ggml: use runtime weight conversion of Q4_0 to Q4_X_X" ON)
 option(GGML_CPU_KLEIDIAI     "ggml: use KleidiAI optimized kernels if applicable" OFF)
+option(GGML_SSSE3            "ggml: enable SSSE3"            OFF)
 option(GGML_SSE42            "ggml: enable SSE 4.2"          ${INS_ENB})
 option(GGML_AVX              "ggml: enable AVX"              ${INS_ENB})
 option(GGML_AVX_VNNI         "ggml: enable AVX-VNNI"         OFF)

ggml/src/CMakeLists.txt

@@ -441,7 +441,7 @@ function(ggml_add_cpu_backend_variant tag_name)
     # other: OPENMP LLAMAFILE CPU_HBM
     if (GGML_SYSTEM_ARCH STREQUAL "x86")
         foreach (feat NATIVE
-                      SSE42
+                      SSSE3 SSE42
                       AVX AVX2 BMI2 AVX_VNNI FMA F16C
                       AVX512 AVX512_VBMI AVX512_VNNI AVX512_BF16
                       AMX_TILE AMX_INT8 AMX_BF16)
@@ -490,6 +490,7 @@ if (GGML_CPU_ALL_VARIANTS)
     endif()
     if (GGML_SYSTEM_ARCH STREQUAL "x86")
         ggml_add_cpu_backend_variant(x64)
+        ggml_add_cpu_backend_variant(ssse3              SSSE3)
         ggml_add_cpu_backend_variant(sse42              SSE42)
         ggml_add_cpu_backend_variant(sandybridge        SSE42 AVX)
         if (NOT MSVC)

ggml/src/ggml-cpu/CMakeLists.txt

@@ -294,6 +294,12 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
                 list(APPEND ARCH_FLAGS /arch:SSE4.2)
                 list(APPEND ARCH_DEFINITIONS GGML_SSE42)
             endif()
+            if (GGML_SSSE3)
+                list(APPEND ARCH_DEFINITIONS GGML_SSSE3)
+                if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
+                    list(APPEND ARCH_FLAGS -mssse3)
+                endif()
+            endif()
             if (GGML_AVX_VNNI)
                 list(APPEND ARCH_DEFINITIONS __AVXVNNI__ GGML_AVX_VNNI)
             endif()
@@ -305,6 +311,10 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
             if (GGML_NATIVE)
                 list(APPEND ARCH_FLAGS -march=native)
             else ()
+                if (GGML_SSSE3)
+                    list(APPEND ARCH_FLAGS -mssse3)
+                    list(APPEND ARCH_DEFINITIONS GGML_SSSE3)
+                endif()
                 if (GGML_SSE42)
                     list(APPEND ARCH_FLAGS -msse4.2)
                     list(APPEND ARCH_DEFINITIONS GGML_SSE42)

ggml/src/ggml-cpu/arch-fallback.h

@@ -98,8 +98,6 @@
 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K
 #define ggml_gemm_pq2_0_4x8_q8_0_generic ggml_gemm_pq2_0_4x8_q8_0
 #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)
-// PTQ1_0 currently has only the generic vec_dot; alias it here until a SIMD version lands
-#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0
 // quants.c
 #define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0
 // repack.cpp

ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp

@@ -266,6 +266,10 @@ static int ggml_backend_cpu_x86_score() {
     int score = 1;
     cpuid_x86 is;
 
+#ifdef GGML_SSSE3
+    if (!is.SSSE3()) { return 0; }
+    score += 1;
+#endif
 #ifdef GGML_FMA
     if (!is.FMA()) { return 0; }
     score += 1;

ggml/src/ggml-cpu/arch/x86/quants.c

@@ -558,6 +558,46 @@ static inline __m128i get_scale_shuffle(int i) {
 #  define GGML_DPBUSD_256 _mm256_dpbusd_avx_epi32
 #endif
 
+#if defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)
+static inline __m128i mul_sum_i8_pairs_sse2(const __m128i x, const __m128i y) {
+    const __m128i sx = _mm_cmpgt_epi8(_mm_setzero_si128(), x);
+    const __m128i sy = _mm_cmpgt_epi8(_mm_setzero_si128(), y);
+    const __m128i lo = _mm_madd_epi16(_mm_unpacklo_epi8(x, sx), _mm_unpacklo_epi8(y, sy));
+    const __m128i hi = _mm_madd_epi16(_mm_unpackhi_epi8(x, sx), _mm_unpackhi_epi8(y, sy));
+    return _mm_add_epi32(lo, hi);
+}
+
+static inline int hsum_i32_4_sse2(__m128i x) {
+    x = _mm_add_epi32(x, _mm_srli_si128(x, 8));
+    x = _mm_add_epi32(x, _mm_srli_si128(x, 4));
+    return _mm_cvtsi128_si32(x);
+}
+
+static inline __m128i decode_trits_sse2(__m128i * x) {
+    const __m128i triple = _mm_add_epi16(*x, _mm_add_epi16(*x, *x));
+    *x = _mm_and_si128(triple, _mm_set1_epi16(255));
+    return _mm_sub_epi16(_mm_srli_epi16(triple, 8), _mm_set1_epi16(1));
+}
+#endif
+
+#if defined(__SSSE3__)
+static inline __m128i mul_sum_ternary_pairs_ssse3(const __m128i codes, const __m128i y) {
+    // Subtract the code offset without negating -128 activations.
+    const __m128i dot = _mm_maddubs_epi16(codes, y);
+    const __m128i sum = _mm_maddubs_epi16(_mm_set1_epi8(1), y);
+    return _mm_madd_epi16(_mm_sub_epi16(dot, sum), _mm_set1_epi16(1));
+}
+
+static inline __m128i decode_trits_ssse3(__m128i * x) {
+    // Extract floor(3*x/256) and keep the low byte for the next trit.
+    const __m128i biased = _mm_xor_si128(*x, _mm_set1_epi8(-128));
+    const __m128i ge1 = _mm_cmpgt_epi8(biased, _mm_set1_epi8(85 - 128));
+    const __m128i ge2 = _mm_cmpgt_epi8(biased, _mm_set1_epi8(170 - 128));
+    *x = _mm_add_epi8(*x, _mm_add_epi8(*x, *x));
+    return _mm_sub_epi8(_mm_setzero_si128(), _mm_add_epi8(ge1, ge2));
+}
+#endif
+
 void ggml_vec_dot_pq2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
     const int qk = QK_PQ2_0;
     const int nb = n / qk;
@@ -619,6 +659,35 @@ void ggml_vec_dot_pq2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo
         acc = _mm256_fmadd_ps(_mm256_set1_ps(d0), acc_block, acc);
     }
     sumf = hsum_float_8(acc);
+#elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)
+    const __m128i mask = _mm_set1_epi8(3);
+    for (int i = 0; i < nb; i++) {
+        const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d);
+        float sumi = 0.0f;
+        for (int k = 0; k < 4; k++) {
+            const block_q8_0 * GGML_RESTRICT yb = &y[i * 4 + k];
+            const __m128i qs = _mm_loadl_epi64((const __m128i *) &x[i].qs[k * 8]);
+            const __m128i q0 = _mm_and_si128(qs, mask);
+            const __m128i q1 = _mm_and_si128(_mm_srli_epi16(qs, 2), mask);
+            const __m128i q2 = _mm_and_si128(_mm_srli_epi16(qs, 4), mask);
+            const __m128i q3 = _mm_and_si128(_mm_srli_epi16(qs, 6), mask);
+            const __m128i q01 = _mm_unpacklo_epi8(q0, q1);
+            const __m128i q23 = _mm_unpacklo_epi8(q2, q3);
+            const __m128i lo = _mm_unpacklo_epi16(q01, q23);
+            const __m128i hi = _mm_unpackhi_epi16(q01, q23);
+#if defined(__SSSE3__)
+            const __m128i dot0 = mul_sum_ternary_pairs_ssse3(lo, _mm_loadu_si128((const __m128i *) &yb->qs[0]));
+            const __m128i dot1 = mul_sum_ternary_pairs_ssse3(hi, _mm_loadu_si128((const __m128i *) &yb->qs[16]));
+#else
+            const __m128i ones = _mm_set1_epi8(1);
+            const __m128i dot0 = mul_sum_i8_pairs_sse2(_mm_sub_epi8(lo, ones), _mm_loadu_si128((const __m128i *) &yb->qs[0]));
+            const __m128i dot1 = mul_sum_i8_pairs_sse2(_mm_sub_epi8(hi, ones), _mm_loadu_si128((const __m128i *) &yb->qs[16]));
+#endif
+            const int dot = hsum_i32_4_sse2(_mm_add_epi32(dot0, dot1));
+            sumi += GGML_CPU_FP16_TO_FP32(yb->d) * dot;
+        }
+        sumf += d0 * sumi;
+    }
 #else
     for (int i = 0; i < nb; i++) {
         const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d);
@@ -645,6 +714,84 @@ void ggml_vec_dot_pq2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo
     *s = sumf;
 }
 
+void ggml_vec_dot_ptq1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+#if defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)
+    assert(n % QK_PTQ1_0 == 0);
+    assert(nrc == 1);
+    UNUSED(nrc);
+    UNUSED(bs);
+    UNUSED(bx);
+    UNUSED(by);
+
+    const block_ptq1_0 * GGML_RESTRICT x = vx;
+    const block_q8_0 * GGML_RESTRICT y = vy;
+    const __m128i zero = _mm_setzero_si128();
+    float sumf = 0.0f;
+    for (int i = 0; i < n / QK_PTQ1_0; ++i) {
+        const block_q8_0 * yb = &y[i * 4];
+        __m128i sums[4] = {zero, zero, zero, zero};
+        const __m128i qs = _mm_loadu_si128((const __m128i *) x[i].qs);
+#if defined(__SSSE3__)
+        __m128i packed = qs;
+#else
+        __m128i lo = _mm_unpacklo_epi8(qs, zero);
+        __m128i hi = _mm_unpackhi_epi8(qs, zero);
+#endif
+
+        // The first 16 packed bytes decode to five groups of 16 weights.
+        for (int j = 0; j < 5; ++j) {
+            const __m128i qy = _mm_loadu_si128((const __m128i *) &yb[j / 2].qs[(j % 2) * 16]);
+#if defined(__SSSE3__)
+            const __m128i dot = mul_sum_ternary_pairs_ssse3(decode_trits_ssse3(&packed), qy);
+            sums[j / 2] = _mm_add_epi32(sums[j / 2], dot);
+#else
+            const __m128i ql = decode_trits_sse2(&lo);
+            const __m128i qh = decode_trits_sse2(&hi);
+            const __m128i sy = _mm_cmpgt_epi8(zero, qy);
+            const __m128i dot0 = _mm_madd_epi16(ql, _mm_unpacklo_epi8(qy, sy));
+            const __m128i dot1 = _mm_madd_epi16(qh, _mm_unpackhi_epi8(qy, sy));
+            sums[j / 2] = _mm_add_epi32(sums[j / 2], _mm_add_epi32(dot0, dot1));
+#endif
+        }
+
+        // The next eight bytes hold weights 80..119; qh holds weights 120..127.
+        __m128i tail = _mm_loadl_epi64((const __m128i *) &x[i].qs[16]);
+#if !defined(__SSSE3__)
+        tail = _mm_unpacklo_epi8(tail, zero);
+#endif
+        for (int j = 0; j < 5; ++j) {
+            const int offset = 80 + j * 8;
+            const __m128i qy = _mm_loadl_epi64((const __m128i *) &yb[offset / 32].qs[offset % 32]);
+#if defined(__SSSE3__)
+            const __m128i dot = mul_sum_ternary_pairs_ssse3(decode_trits_ssse3(&tail), qy);
+#else
+            const __m128i q = decode_trits_sse2(&tail);
+            const __m128i sy = _mm_cmpgt_epi8(zero, qy);
+            const __m128i dot = _mm_madd_epi16(q, _mm_unpacklo_epi8(qy, sy));
+#endif
+            sums[offset / 32] = _mm_add_epi32(sums[offset / 32], dot);
+        }
+        uint16_t qh;
+        memcpy(&qh, x[i].qh, sizeof(qh));
+        tail = _mm_unpacklo_epi8(_mm_set1_epi16((int16_t) qh), zero);
+        tail = _mm_and_si128(_mm_mullo_epi16(tail, _mm_setr_epi16(1, 1, 3, 3, 9, 9, 27, 27)), _mm_set1_epi16(255));
+        const __m128i q = decode_trits_sse2(&tail);
+        const __m128i qy = _mm_loadl_epi64((const __m128i *) &yb[3].qs[24]);
+        const __m128i sy = _mm_cmpgt_epi8(zero, qy);
+        sums[3] = _mm_add_epi32(sums[3], _mm_madd_epi16(q, _mm_unpacklo_epi8(qy, sy)));
+
+        float sumi = 0.0f;
+        for (int k = 0; k < 4; ++k) {
+            sumi += GGML_CPU_FP16_TO_FP32(yb[k].d) * hsum_i32_4_sse2(sums[k]);
+        }
+        sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * sumi;
+    }
+    *s = sumf;
+#else
+    ggml_vec_dot_ptq1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
+#endif
+}
+
 void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
     const int qk = QK1_0;
     const int nb = n / qk;
@@ -4251,6 +4398,38 @@ void ggml_vec_dot_pq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo
         acc = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[i].d) * yb->d), _mm256_cvtepi32_ps(s32), acc);
     }
     sumf = hsum_float_8(acc);
+#elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)
+    // Same structure as the AVX2 path at half the width: the four sub-block dots of a
+    // 128-weight block accumulate in int32 and the block scale is applied once. The unpack
+    // puts byte b's bit-pair j at element 4b+j, which is the order q8 is already in.
+    const __m128i mask = _mm_set1_epi8(3);
+    for (int i = 0; i < nb; i++) {
+        const block_q8_K * GGML_RESTRICT yb = &y[i >> 1];
+        const int8_t * GGML_RESTRICT q8 = yb->qs + 128 * (i & 1);
+        __m128i acc32 = _mm_setzero_si128();
+        for (int k = 0; k < 4; k++) {
+            const __m128i qs  = _mm_loadl_epi64((const __m128i *) &x[i].qs[8 * k]);
+            const __m128i q0  = _mm_and_si128(qs, mask);
+            const __m128i q1  = _mm_and_si128(_mm_srli_epi16(qs, 2), mask);
+            const __m128i q2  = _mm_and_si128(_mm_srli_epi16(qs, 4), mask);
+            const __m128i q3  = _mm_and_si128(_mm_srli_epi16(qs, 6), mask);
+            const __m128i q01 = _mm_unpacklo_epi8(q0, q1);
+            const __m128i q23 = _mm_unpacklo_epi8(q2, q3);
+            const __m128i lo  = _mm_unpacklo_epi16(q01, q23);
+            const __m128i hi  = _mm_unpackhi_epi16(q01, q23);
+            const __m128i y0  = _mm_loadu_si128((const __m128i *) (q8 + 32 * k));
+            const __m128i y1  = _mm_loadu_si128((const __m128i *) (q8 + 32 * k + 16));
+#if defined(__SSSE3__)
+            acc32 = _mm_add_epi32(acc32, mul_sum_ternary_pairs_ssse3(lo, y0));
+            acc32 = _mm_add_epi32(acc32, mul_sum_ternary_pairs_ssse3(hi, y1));
+#else
+            const __m128i ones = _mm_set1_epi8(1);
+            acc32 = _mm_add_epi32(acc32, mul_sum_i8_pairs_sse2(_mm_sub_epi8(lo, ones), y0));
+            acc32 = _mm_add_epi32(acc32, mul_sum_i8_pairs_sse2(_mm_sub_epi8(hi, ones), y1));
+#endif
+        }
+        sumf += (GGML_CPU_FP16_TO_FP32(x[i].d) * yb->d) * (float) hsum_i32_4_sse2(acc32);
+    }
 #else
     ggml_vec_dot_pq2_0_q8_K_generic(n, &sumf, bs, vx, bx, vy, by, nrc);
 #endif

ggml/src/ggml-cpu/ggml-cpu-impl.h

@@ -52,8 +52,8 @@ struct ggml_compute_params {
 #endif
 #endif
 
-// __SSE3__ and __SSSE3__ are not defined in MSVC, but SSE3/SSSE3 are present when AVX/AVX2/AVX512 are available
-#if defined(_MSC_VER) && (defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__))
+// MSVC does not define SSE3/SSSE3 feature macros.
+#if defined(_MSC_VER) && (defined(GGML_SSSE3) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__))
 #ifndef __SSE3__
 #define __SSE3__
 #endif