Add native Windows CPU support and tested CI binaries

.github/workflows/ci.yml

@@ -4,21 +4,128 @@ on:
   push:
     branches: [main]
   pull_request:
+  workflow_dispatch:
 
 permissions:
   contents: read
 
 jobs:
   portable:
-    name: Portable checks (${{ matrix.os }})
+    name: CPU binary (${{ matrix.platform }})
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-14]
+        include:
+          - os: ubuntu-24.04
+            platform: linux-x86_64
+          - os: ubuntu-24.04-arm
+            platform: linux-arm64
+          - os: macos-15
+            platform: darwin-arm64
+          - os: windows-2022
+            platform: windows-x86_64
+    defaults:
+      run:
+        shell: bash
     steps:
-      - uses: actions/checkout@v4
-      - name: Run model-free checks
-        run: make check
-      - name: Build CPU server
-        run: make
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - name: Set up native Windows compiler
+        if: runner.os == 'Windows'
+        uses: msys2/setup-msys2@ec48f7c5447b3140e2b088413ae3a55687bccb6e # v2
+        with:
+          msystem: UCRT64
+          update: true
+          install: make mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-python
+      - name: Run model-free checks and build
+        if: runner.os != 'Windows'
+        run: make check && make
+      - name: Run native Windows checks and build
+        if: runner.os == 'Windows'
+        shell: msys2 {0}
+        run: make check && make && make test-windows-download
+      - name: Download public upstream test model
+        run: |
+          mkdir -p model
+          curl -fL --retry 3 -o model/embeddinggemma-300M-qat-Q4_0.gguf 'https://huggingface.co/ggml-org/embeddinggemma-300M-qat-q4_0-GGUF/resolve/8dd0ca2a66a8f14470acb0e2a71f801afbc5fb73/embeddinggemma-300M-qat-Q4_0.gguf'
+          python -c 'import hashlib,pathlib; p=pathlib.Path("model/embeddinggemma-300M-qat-Q4_0.gguf"); assert hashlib.sha256(p.read_bytes()).hexdigest()=="50d28e22432a148f6f8a86eab3700f92add5d1f54baf7790675a2a4dadbccf26"'
+      - name: Run embedding and HTTP conformance checks
+        if: runner.os != 'Windows'
+        run: make test
+      - name: Run native Windows embedding and HTTP checks
+        if: runner.os == 'Windows'
+        shell: msys2 {0}
+        run: make test
+      - name: Verify actual executable format and architecture
+        env:
+          PLATFORM: ${{ matrix.platform }}
+        run: |
+          suffix=''
+          if [ "$RUNNER_OS" = Windows ]; then suffix='.exe'; fi
+          python scripts/check-binary-platform.py "$PLATFORM" "build/embeddinggemma${suffix}"
+      - name: Verify AVX2 code stays in runtime-dispatched kernels
+        if: matrix.platform == 'linux-x86_64'
+        run: python3 scripts/check-avx2-isolation.py build/embeddinggemma
+      - name: Verify Windows AVX2 code stays in runtime-dispatched kernels
+        if: runner.os == 'Windows'
+        shell: msys2 {0}
+        run: python scripts/check-avx2-isolation.py build/embeddinggemma.exe
+      - name: Inspect Windows runtime dependencies
+        if: runner.os == 'Windows'
+        shell: msys2 {0}
+        run: |
+          objdump -f build/embeddinggemma.exe
+          objdump -p build/embeddinggemma.exe | grep 'DLL Name'
+          if objdump -p build/embeddinggemma.exe | grep -Ei 'DLL Name:.*(msys-|cygwin|libwinpthread|libgcc|libstdc)'; then
+            echo 'unexpected non-system runtime DLL dependency' >&2
+            exit 1
+          fi
+      - name: Inspect Unix executable
+        if: runner.os != 'Windows'
+        run: uname -m && file build/embeddinggemma && cc --version
+      - name: Strip Windows executable
+        if: runner.os == 'Windows'
+        shell: msys2 {0}
+        run: strip build/embeddinggemma.exe
+      - name: Strip Unix executable
+        if: runner.os != 'Windows'
+        run: strip build/embeddinggemma
+      - name: Preserve Windows runtime notices and toolchain provenance
+        if: runner.os == 'Windows'
+        shell: msys2 {0}
+        run: |
+          mkdir -p dist/licenses
+          packages=(mingw-w64-ucrt-x86_64-crt mingw-w64-ucrt-x86_64-winpthreads mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-libgcc)
+          pacman -Q "${packages[@]}" > dist/toolchain.txt
+          gcc --version >> dist/toolchain.txt
+          while IFS= read -r notice; do
+            if [ -f "$notice" ]; then
+              relative="${notice#*/share/licenses/}"
+              mkdir -p "dist/licenses/$(dirname "$relative")"
+              cp "$notice" "dist/licenses/$relative"
+            fi
+          done < <(pacman -Qql "${packages[@]}" | grep '/share/licenses/')
+          test -n "$(find dist/licenses -type f -print -quit)"
+      - name: Preserve Unix toolchain provenance
+        if: runner.os != 'Windows'
+        run: mkdir -p dist && cc --version > dist/toolchain.txt
+      - name: Stage executable and checksum
+        env:
+          PLATFORM: ${{ matrix.platform }}
+        run: |
+          mkdir -p dist
+          suffix=''
+          if [ "$RUNNER_OS" = Windows ]; then suffix='.exe'; fi
+          name="embeddinggemma-${PLATFORM}-cpu${suffix}"
+          cp "build/embeddinggemma${suffix}" "dist/$name"
+          python -c 'import hashlib,pathlib; p=next(pathlib.Path("dist").glob("embeddinggemma-*")); p.with_name(p.name+".sha256").write_text(hashlib.sha256(p.read_bytes()).hexdigest()+"  "+p.name+"\n")'
+          cp LICENSE dist/LICENSE
+          git rev-parse HEAD > dist/SOURCE_REVISION
+      - name: Upload verified executable
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: embeddinggemma-${{ matrix.platform }}-cpu
+          path: dist/*
+          if-no-files-found: error
+          retention-days: 30

CONTRIBUTING.md

@@ -87,8 +87,10 @@ make
 ```
 
 The output is `build/embeddinggemma`. The CPU route includes scalar C, ARM64
-NEON, AVX2, and SSSE3 kernels and selects the best available implementation at
-runtime.
+NEON, SSE2/SSSE3, and AVX2 kernels. NEON and SSE2/SSSE3 follow the compiler
+target; on x86-64 the AVX2 kernels (`src/kernels_avx2.c`, with per-function
+`target("avx2,fma,f16c")` attributes) are selected at runtime when the CPU and
+OS support AVX2, FMA, and F16C, so one baseline binary runs on any x86-64 CPU.
 
 Build the self-contained Metal server:
 
@@ -231,6 +233,9 @@ Diagnostic controls:
 
 - `EI_THREADS` (1..1024; the server's `--threads` sets it): override the CPU
   thread count. The default is every core up to 32.
+- `EI_CPU_ISA=auto|baseline|avx2`: x86-64 kernel family. `baseline` forces the
+  SSE2 route on an AVX2 host; `avx2` fails at startup when unsupported.
+  `test_cpu_dispatch` compares the two routes in one process.
 - `EI_CPU_SHORT_THREADS`: override the short-projection worker width.
 - `EI_CPU_DUAL_PROJECTION`: control paired projection accumulation.
 - `EI_CPU_FUSED_RMS_QUANT=0`: materialize normalized activations before Q8.

Makefile

@@ -3,6 +3,13 @@ CC      ?= cc
 CXX     ?= c++
 CFLAGS  ?= -std=c11 -O2 -Wall -Wextra -Werror -g
 LDLIBS  ?= -lm -pthread
+ifneq ($(findstring MINGW,$(shell uname -s)),)
+CFLAGS += -D_WIN32_WINNT=0x0601 -D_CRT_RAND_S -D__USE_MINGW_ANSI_STDIO=1
+# Win64 GCC does not realign the stack for 32-byte AVX spills (GCC bug 54412);
+# assembling aligned vector moves as unaligned keeps the AVX2 kernels safe.
+CFLAGS += -Wa,-muse-unaligned-vector-move
+LDLIBS += -lws2_32 -static
+endif
 NVCC    ?= nvcc
 CUDA_HOME ?= /usr/local/cuda
 NVCCFLAGS ?= -std=c++17 -O3 --use_fast_math -lineinfo
@@ -99,7 +106,8 @@ HDRS_NATIVE := $(wildcard src/*.h src/vendor/*.h)
 SRCS_CORE := src/gguf.c
 SRCS_MODEL := $(SRCS_CORE) src/model.c
 SRCS_TOKENIZER := $(SRCS_MODEL) src/tokenizer.c
-SRCS_ENGINE := $(SRCS_TOKENIZER) src/quants.c src/kernels.c src/parallel.c src/engine.c src/gemma2.c src/gemma2_fast.c src/gemma2_fast_x86.c src/media.c src/response_cache.c
+SRCS_KERNELS := src/quants.c src/kernels.c src/kernels_avx2.c
+SRCS_ENGINE := $(SRCS_TOKENIZER) $(SRCS_KERNELS) src/parallel.c src/engine.c src/gemma2.c src/gemma2_fast.c src/gemma2_fast_x86.c src/media.c src/response_cache.c
 SRCS_SERVICE := src/inference_service.c
 SRCS_SERVER := src/server.c src/float_format.c
 
@@ -184,8 +192,11 @@ $(BUILD)/test_embed: src/test_embed.c $(SRCS_ENGINE) $(HDRS_NATIVE) | $(BUILD)
 $(BUILD)/test_batch: src/test_batch.c $(SRCS_ENGINE) $(HDRS_NATIVE) | $(BUILD)
 	$(CC) $(CFLAGS) -o $@ src/test_batch.c $(SRCS_ENGINE) $(LDLIBS)
 
-$(BUILD)/test_kernels: src/test_kernels.c src/quants.c src/kernels.c src/gguf.c $(HDRS_NATIVE) | $(BUILD)
-	$(CC) $(CFLAGS) -o $@ src/test_kernels.c src/quants.c src/kernels.c src/gguf.c $(LDLIBS)
+$(BUILD)/test_cpu_dispatch: src/test_cpu_dispatch.c $(SRCS_ENGINE) $(HDRS_NATIVE) | $(BUILD)
+	$(CC) $(CFLAGS) -o $@ src/test_cpu_dispatch.c $(SRCS_ENGINE) $(LDLIBS)
+
+$(BUILD)/test_kernels: src/test_kernels.c $(SRCS_KERNELS) src/gguf.c $(HDRS_NATIVE) | $(BUILD)
+	$(CC) $(CFLAGS) -o $@ src/test_kernels.c $(SRCS_KERNELS) src/gguf.c $(LDLIBS)
 
 $(BUILD)/embeddinggemma: $(SRCS_SERVER) $(SRCS_ENGINE) $(SRCS_SERVICE) $(HDRS_NATIVE) | $(BUILD)
 	$(CC) $(CFLAGS) -o $@ $(SRCS_SERVER) $(SRCS_ENGINE) $(SRCS_SERVICE) $(LDLIBS)
@@ -374,8 +385,8 @@ $(BUILD)/test_batch_metal: src/test_batch.c $(SRCS_ENGINE) $(METAL_ENGINE_OBJS)
 	$(CC) $(CFLAGS) -DEI_ENABLE_METAL -o $@ src/test_batch.c $(SRCS_ENGINE) \
 		$(METAL_ENGINE_OBJS) $(LDLIBS) $(METALLIB_LDFLAGS) $(METAL_FRAMEWORKS)
 
-$(BUILD)/perf_kernels: perf/harness/bench_kernels.c src/quants.c src/kernels.c src/gguf.c $(HDRS_NATIVE) | $(BUILD)
-	$(CC) $(CFLAGS) -Isrc -o $@ perf/harness/bench_kernels.c src/quants.c src/kernels.c src/gguf.c $(LDLIBS)
+$(BUILD)/perf_kernels: perf/harness/bench_kernels.c $(SRCS_KERNELS) src/gguf.c $(HDRS_NATIVE) | $(BUILD)
+	$(CC) $(CFLAGS) -Isrc -o $@ perf/harness/bench_kernels.c $(SRCS_KERNELS) src/gguf.c $(LDLIBS)
 
 $(BUILD)/perf_engine: perf/harness/bench_engine.c $(SRCS_ENGINE) $(HDRS_NATIVE) | $(BUILD)
 	$(CC) $(CFLAGS) -Isrc -o $@ perf/harness/bench_engine.c $(SRCS_ENGINE) $(LDLIBS)
@@ -433,13 +444,15 @@ $(BUILD)/xpu:
 	mkdir -p $(BUILD)/xpu
 
 .PHONY: test test-unit test-defaults check test-http test-http-metal test-http-cuda test-http-rocm test-http-xpu test-metal test-cuda test-rocm test-xpu perf perf-engine perf-engine-metal perf-engine-cuda perf-engine-rocm perf-engine-xpu perf-concurrency perf-concurrency-cuda perf-concurrency-rocm perf-concurrency-xpu perf-dimensions perf-batch perf-tokenization perf-servers perf-compare-llamacpp metal metal-kernels cuda rocm xpu check-scripts clean clean-cpu clean-metal clean-cuda clean-rocm clean-xpu release-darwin release-linux-cpu release-linux-cuda release-linux-rocm release-linux-xpu release-checksums release-verify release-ready release-info help
-test: $(BUILD)/embeddinggemma $(BUILD)/test_gguf $(BUILD)/test_tokenizer $(BUILD)/test_kernels $(BUILD)/test_embed $(BUILD)/test_batch $(BUILD)/test_inference_service $(BUILD)/test_response_cache $(BUILD)/test_float_format
+test: $(BUILD)/embeddinggemma $(BUILD)/test_gguf $(BUILD)/test_tokenizer $(BUILD)/test_kernels $(BUILD)/test_embed $(BUILD)/test_batch $(BUILD)/test_cpu_dispatch $(BUILD)/test_inference_service $(BUILD)/test_response_cache $(BUILD)/test_float_format
 	python3 testdata/test_model_manifest.py --binary ./$(BUILD)/test_gguf \
 		--model $(MODEL) --manifest testdata/model-manifest.json
 	./$(BUILD)/test_tokenizer $(MODEL) testdata/goldens-tokens.json
 	./$(BUILD)/test_kernels
 	./$(BUILD)/test_embed $(MODEL) testdata/goldens-llamacpp.json
+	EI_CPU_ISA=baseline ./$(BUILD)/test_embed $(MODEL) testdata/goldens-llamacpp.json
 	./$(BUILD)/test_batch $(MODEL) cpu
+	./$(BUILD)/test_cpu_dispatch $(MODEL) testdata/test-strings.json
 	./$(BUILD)/test_inference_service
 	./$(BUILD)/test_response_cache
 	./$(BUILD)/test_float_format
@@ -736,3 +749,14 @@ ROUTE ?= TENSOR=1,FLASH=1
 .PHONY: bench-gemma2-metal
 bench-gemma2-metal: $(BUILD)/bench_gemma2_metal
 	./$(BUILD)/bench_gemma2_metal $(GEMMA2_MODEL) "$(ROUTE)" $(CASES)
+# Process fixture contains fictional bytes, never a model or network request.
+$(BUILD)/test_downloader_fixture: src/test_downloader_fixture.c | $(BUILD)
+	$(CC) $(CFLAGS) -o $@ $< $(LDLIBS)
+
+.PHONY: test-windows-download
+$(BUILD)/test_windows_paths: src/test_windows_paths.c src/windows_compat.h src/common.h | $(BUILD)
+	$(CC) $(CFLAGS) -o $@ $< $(LDLIBS)
+
+test-windows-download: $(BUILD)/embeddinggemma $(BUILD)/test_downloader_fixture $(BUILD)/test_windows_paths
+	./$(BUILD)/test_windows_paths
+	python3 testdata/test_windows_download.py --binary ./$(BUILD)/embeddinggemma.exe --downloader ./$(BUILD)/test_downloader_fixture.exe

PORT_SPEC.md

@@ -144,7 +144,7 @@ llama.cpp deep-read) before implementing.
 | target | status | implementation/build |
 |---|---|---|
 | macOS ARM64 CPU | implemented, parity tested | C11 + NEON dot-product path |
-| x86 CPU | implemented, parity tested | scalar, SSSE3, and AVX2 paths |
+| x86 CPU | implemented, parity tested | scalar, SSSE3, and runtime-dispatched AVX2 paths |
 | macOS Metal | implemented, parity tested | Objective-C host + standalone metallib; short-shape residual/next-norm fusion; FP16 K/V at long sequence lengths with FP32 accumulation |
 | NVIDIA CUDA | implemented, parity tested | Q4 x Q8 DP4A latency path; direct-FP16 combined-QKV epilogue; Flash, online, banded-SWA, and tensor-core attention; native packed-Q4 MMA diagnostics; CUDA graph replay |
 | ROCm/HIP | implemented, parity tested | portable CDNA fat binary; paired packed-Q4 wave and MFMA projection kernels; exact singleton V-only attention; batched hipBLAS GQA; FP16/FP32 score routing; fused residual/RMS and pooling epilogues |

README.md

@@ -292,6 +292,33 @@ Published executables are deliberately small. Release `v0.3.1` contains:
 | Linux x86_64 | ROCm | 1.2 MiB |
 | Linux x86_64 | XPU SYCL | 2.1 MiB |
 
+### Fork CI binaries
+
+This fork's `CI` workflow uploads checksummed CPU executable artifacts for
+Linux x86_64, Linux ARM64, macOS ARM64, and Windows x86_64 after unit, model,
+and HTTP conformance checks pass. Download artifacts from the workflow run;
+these are CI builds, not upstream release assets. Existing CUDA, ROCm, XPU
+and Metal release targets remain available through the upstream build process.
+
+CPU builds use the compiler's baseline architecture, without `-march=native`:
+x86_64 uses SSE2 and ARM64 uses NEON. On x86_64 the same executable switches to
+AVX2/FMA/F16C kernels at startup when the CPU supports them (Haswell, Zen, and
+newer); set `EI_CPU_ISA=baseline` to force the SSE2 route. Linux artifacts are built on Ubuntu 24.04
+and require a compatible glibc (2.39 or later); macOS ARM64 is tested on the macOS 15 runner and targets macOS 14 or
+later. Windows uses a native MinGW UCRT64 executable with statically linked
+winpthreads and compiler runtime, tested on Windows Server 2022. It requires
+Windows' Universal C Runtime; older Windows versions are not CI-tested.
+No Windows GPU backend or PowerShell installer is provided by this change.
+
+On Windows, run `embeddinggemma-windows-x86_64-cpu.exe`. The default model
+cache is `%LOCALAPPDATA%/embeddinggemma.c/`, falling back to
+`%USERPROFILE%/.cache/embeddinggemma.c/`; `XDG_CACHE_HOME`, `EI_MODEL_PATH`, and
+`--model` retain their existing precedence. Automatic download uses native
+`curl.exe` or `wget.exe` on PATH and supports paths containing spaces.
+Build locally from an MSYS2 UCRT64 shell with `make check`,
+`make test-windows-download`, and `make test` after supplying the public model.
+The model's Gemma license is separate from this repository's MIT license.
+
 The 278 MB Q4_0 model is downloaded separately from Hugging Face and is not
 distributed by this project.
 

perf/optimization_status.md

@@ -3927,3 +3927,51 @@ dispatched 4 simdgroups for an 8-simdgroup kernel (NaN output); it now uses
 - Cross-request image packing (under 10% at 432x432, 3.5% at 768x768).
 - Hardware image decode (PNG decode 1.7-4.8 ms of a 21-56 ms miss).
 - Flash softmax fast exp and rescale skip (above).
+## 2026-10-08: x86-64 Runtime AVX2 Dispatch
+
+Status: candidate (fork PR; upstream proposal pending).
+
+Current implementation: portable x86-64 CPU binaries are built at the compiler
+baseline (SSE2), so the existing compile-time `__AVX2__` kernels never ran in
+release or CI artifacts. Worse, the Q4_0 x Q8_0 dot products had no SSE2 path
+at all, so shipped binaries ran the projections in scalar C
+(`ei_quants_kernel_variant() == "cpu-scalar"`).
+
+Current public route: `src/kernels_avx2.c` holds the AVX2 kernels behind
+per-function `target("avx2,fma,f16c")` attributes. `kernels.c` and `quants.c`
+call them through `EI_DISPATCH_AVX2` after a one-time CPUID + XGETBV check
+(AVX2, FMA, F16C, OS YMM state). `EI_CPU_ISA=auto|baseline|avx2` overrides it.
+`scripts/check-avx2-isolation.py` fails CI if VEX code appears outside the
+`ei_avx2_*` functions. ARM64 NEON object code is byte-identical to before.
+
+Correctness: `test_cpu_dispatch` embeds the test strings plus 150- and
+600-token inputs and a packed batch under both routes in one process, gated at
+cosine >= 0.999 and max abs diff <= 1e-2 at D=768/512/256/128 (the golden
+gate; a last-bit summation difference can flip a Q8_0 activation rounding and
+cascade, so the routes are not bit-identical on the real model). Against the
+llama.cpp goldens on the real model, AVX2 min cosine is 0.999861 and SSE2 is
+0.999871.
+`test_kernels` runs every kernel test under both routes and checks that Q8_0
+quantization is bit-identical and Q4_0 rows/dual/batch agree within 3e-7.
+
+Experiments (sandbox Intel Xeon @ 2.80 GHz, 4 vCPU, GCC 13.3, synthetic
+random-weight GGUF with the production shapes, 150-token chunk, best of 3):
+
+- Moving the existing AVX2 kernels behind dispatch: SSE2 1,326 ms vs AVX2
+  535 ms (2.5x).
+- Callgrind on the AVX2 route showed 56% of instructions in the software
+  `ei_fp16_to_fp32` block-scale conversion. Hardware F16C (`_cvtsh_ss`, exact)
+  in the AVX2 kernels: 387 ms (3.3x vs SSE2), output bit-identical.
+- FMA accumulation in the Q4_0 dots: q4gemv 25-27 vs 20-24 GFLOPS with
+  separate mul+add. Retained.
+- `EI_CPU_MULTIROW_MIN_TOKENS=1` or `64` (four-token batch kernel) at 150
+  tokens: 461-465 ms vs 387 ms. Rejected; the 512 boundary stays.
+
+Decision: retain runtime dispatch, F16C, and FMA.
+
+Open questions: the remaining gap to llama.cpp at 150 tokens is structural,
+not ISA width. The CPU engine still runs per-token GEMV below 512 tokens and
+re-streams all ~57 MB of layer weights for every token. llama.cpp's x86
+prefill uses tiled/repacked Q4_0 GEMM. A register-blocked Q4_0 x Q8_0 GEMM
+(several rows x several tokens per tile) is the next CPU lever; AVX-VNNI would
+come after that.