diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59b5e2e..94b8d60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,21 +4,128 @@ on: push: branches: [main] pull_request: + workflow_dispatch: permissions: contents: read jobs: portable: - name: Portable checks (${{ matrix.os }}) + name: CPU binary (${{ matrix.platform }}) runs-on: ${{ matrix.os }} + timeout-minutes: 30 strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-14] + include: + - os: ubuntu-24.04 + platform: linux-x86_64 + - os: ubuntu-24.04-arm + platform: linux-arm64 + - os: macos-15 + platform: darwin-arm64 + - os: windows-2022 + platform: windows-x86_64 + defaults: + run: + shell: bash steps: - - uses: actions/checkout@v4 - - name: Run model-free checks - run: make check - - name: Build CPU server - run: make + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - name: Set up native Windows compiler + if: runner.os == 'Windows' + uses: msys2/setup-msys2@ec48f7c5447b3140e2b088413ae3a55687bccb6e # v2 + with: + msystem: UCRT64 + update: true + install: make mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-python + - name: Run model-free checks and build + if: runner.os != 'Windows' + run: make check && make + - name: Run native Windows checks and build + if: runner.os == 'Windows' + shell: msys2 {0} + run: make check && make && make test-windows-download + - name: Download public upstream test model + run: | + mkdir -p model + curl -fL --retry 3 -o model/embeddinggemma-300M-qat-Q4_0.gguf 'https://huggingface.co/ggml-org/embeddinggemma-300M-qat-q4_0-GGUF/resolve/8dd0ca2a66a8f14470acb0e2a71f801afbc5fb73/embeddinggemma-300M-qat-Q4_0.gguf' + python -c 'import hashlib,pathlib; p=pathlib.Path("model/embeddinggemma-300M-qat-Q4_0.gguf"); assert hashlib.sha256(p.read_bytes()).hexdigest()=="50d28e22432a148f6f8a86eab3700f92add5d1f54baf7790675a2a4dadbccf26"' + - name: Run embedding and HTTP conformance checks + if: runner.os != 'Windows' + run: make test + - name: Run native Windows embedding and HTTP checks + if: runner.os == 'Windows' + shell: msys2 {0} + run: make test + - name: Verify actual executable format and architecture + env: + PLATFORM: ${{ matrix.platform }} + run: | + suffix='' + if [ "$RUNNER_OS" = Windows ]; then suffix='.exe'; fi + python scripts/check-binary-platform.py "$PLATFORM" "build/embeddinggemma${suffix}" + - name: Verify AVX2 code stays in runtime-dispatched kernels + if: matrix.platform == 'linux-x86_64' + run: python3 scripts/check-avx2-isolation.py build/embeddinggemma + - name: Verify Windows AVX2 code stays in runtime-dispatched kernels + if: runner.os == 'Windows' + shell: msys2 {0} + run: python scripts/check-avx2-isolation.py build/embeddinggemma.exe + - name: Inspect Windows runtime dependencies + if: runner.os == 'Windows' + shell: msys2 {0} + run: | + objdump -f build/embeddinggemma.exe + objdump -p build/embeddinggemma.exe | grep 'DLL Name' + if objdump -p build/embeddinggemma.exe | grep -Ei 'DLL Name:.*(msys-|cygwin|libwinpthread|libgcc|libstdc)'; then + echo 'unexpected non-system runtime DLL dependency' >&2 + exit 1 + fi + - name: Inspect Unix executable + if: runner.os != 'Windows' + run: uname -m && file build/embeddinggemma && cc --version + - name: Strip Windows executable + if: runner.os == 'Windows' + shell: msys2 {0} + run: strip build/embeddinggemma.exe + - name: Strip Unix executable + if: runner.os != 'Windows' + run: strip build/embeddinggemma + - name: Preserve Windows runtime notices and toolchain provenance + if: runner.os == 'Windows' + shell: msys2 {0} + run: | + mkdir -p dist/licenses + packages=(mingw-w64-ucrt-x86_64-crt mingw-w64-ucrt-x86_64-winpthreads mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-libgcc) + pacman -Q "${packages[@]}" > dist/toolchain.txt + gcc --version >> dist/toolchain.txt + while IFS= read -r notice; do + if [ -f "$notice" ]; then + relative="${notice#*/share/licenses/}" + mkdir -p "dist/licenses/$(dirname "$relative")" + cp "$notice" "dist/licenses/$relative" + fi + done < <(pacman -Qql "${packages[@]}" | grep '/share/licenses/') + test -n "$(find dist/licenses -type f -print -quit)" + - name: Preserve Unix toolchain provenance + if: runner.os != 'Windows' + run: mkdir -p dist && cc --version > dist/toolchain.txt + - name: Stage executable and checksum + env: + PLATFORM: ${{ matrix.platform }} + run: | + mkdir -p dist + suffix='' + if [ "$RUNNER_OS" = Windows ]; then suffix='.exe'; fi + name="embeddinggemma-${PLATFORM}-cpu${suffix}" + cp "build/embeddinggemma${suffix}" "dist/$name" + python -c 'import hashlib,pathlib; p=next(pathlib.Path("dist").glob("embeddinggemma-*")); p.with_name(p.name+".sha256").write_text(hashlib.sha256(p.read_bytes()).hexdigest()+" "+p.name+"\n")' + cp LICENSE dist/LICENSE + git rev-parse HEAD > dist/SOURCE_REVISION + - name: Upload verified executable + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: embeddinggemma-${{ matrix.platform }}-cpu + path: dist/* + if-no-files-found: error + retention-days: 30 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0c78ef3..afdf377 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,8 +87,10 @@ make ``` The output is `build/embeddinggemma`. The CPU route includes scalar C, ARM64 -NEON, AVX2, and SSSE3 kernels and selects the best available implementation at -runtime. +NEON, SSE2/SSSE3, and AVX2 kernels. NEON and SSE2/SSSE3 follow the compiler +target; on x86-64 the AVX2 kernels (`src/kernels_avx2.c`, with per-function +`target("avx2,fma,f16c")` attributes) are selected at runtime when the CPU and +OS support AVX2, FMA, and F16C, so one baseline binary runs on any x86-64 CPU. Build the self-contained Metal server: @@ -231,6 +233,9 @@ Diagnostic controls: - `EI_THREADS` (1..1024; the server's `--threads` sets it): override the CPU thread count. The default is every core up to 32. +- `EI_CPU_ISA=auto|baseline|avx2`: x86-64 kernel family. `baseline` forces the + SSE2 route on an AVX2 host; `avx2` fails at startup when unsupported. + `test_cpu_dispatch` compares the two routes in one process. - `EI_CPU_SHORT_THREADS`: override the short-projection worker width. - `EI_CPU_DUAL_PROJECTION`: control paired projection accumulation. - `EI_CPU_FUSED_RMS_QUANT=0`: materialize normalized activations before Q8. diff --git a/Makefile b/Makefile index 470f246..ce848f3 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,13 @@ CC ?= cc CXX ?= c++ CFLAGS ?= -std=c11 -O2 -Wall -Wextra -Werror -g LDLIBS ?= -lm -pthread +ifneq ($(findstring MINGW,$(shell uname -s)),) +CFLAGS += -D_WIN32_WINNT=0x0601 -D_CRT_RAND_S -D__USE_MINGW_ANSI_STDIO=1 +# Win64 GCC does not realign the stack for 32-byte AVX spills (GCC bug 54412); +# assembling aligned vector moves as unaligned keeps the AVX2 kernels safe. +CFLAGS += -Wa,-muse-unaligned-vector-move +LDLIBS += -lws2_32 -static +endif NVCC ?= nvcc CUDA_HOME ?= /usr/local/cuda NVCCFLAGS ?= -std=c++17 -O3 --use_fast_math -lineinfo @@ -99,7 +106,8 @@ HDRS_NATIVE := $(wildcard src/*.h src/vendor/*.h) SRCS_CORE := src/gguf.c SRCS_MODEL := $(SRCS_CORE) src/model.c SRCS_TOKENIZER := $(SRCS_MODEL) src/tokenizer.c -SRCS_ENGINE := $(SRCS_TOKENIZER) src/quants.c src/kernels.c src/parallel.c src/engine.c src/gemma2.c src/gemma2_fast.c src/gemma2_fast_x86.c src/media.c src/response_cache.c +SRCS_KERNELS := src/quants.c src/kernels.c src/kernels_avx2.c +SRCS_ENGINE := $(SRCS_TOKENIZER) $(SRCS_KERNELS) src/parallel.c src/engine.c src/gemma2.c src/gemma2_fast.c src/gemma2_fast_x86.c src/media.c src/response_cache.c SRCS_SERVICE := src/inference_service.c SRCS_SERVER := src/server.c src/float_format.c @@ -184,8 +192,11 @@ $(BUILD)/test_embed: src/test_embed.c $(SRCS_ENGINE) $(HDRS_NATIVE) | $(BUILD) $(BUILD)/test_batch: src/test_batch.c $(SRCS_ENGINE) $(HDRS_NATIVE) | $(BUILD) $(CC) $(CFLAGS) -o $@ src/test_batch.c $(SRCS_ENGINE) $(LDLIBS) -$(BUILD)/test_kernels: src/test_kernels.c src/quants.c src/kernels.c src/gguf.c $(HDRS_NATIVE) | $(BUILD) - $(CC) $(CFLAGS) -o $@ src/test_kernels.c src/quants.c src/kernels.c src/gguf.c $(LDLIBS) +$(BUILD)/test_cpu_dispatch: src/test_cpu_dispatch.c $(SRCS_ENGINE) $(HDRS_NATIVE) | $(BUILD) + $(CC) $(CFLAGS) -o $@ src/test_cpu_dispatch.c $(SRCS_ENGINE) $(LDLIBS) + +$(BUILD)/test_kernels: src/test_kernels.c $(SRCS_KERNELS) src/gguf.c $(HDRS_NATIVE) | $(BUILD) + $(CC) $(CFLAGS) -o $@ src/test_kernels.c $(SRCS_KERNELS) src/gguf.c $(LDLIBS) $(BUILD)/embeddinggemma: $(SRCS_SERVER) $(SRCS_ENGINE) $(SRCS_SERVICE) $(HDRS_NATIVE) | $(BUILD) $(CC) $(CFLAGS) -o $@ $(SRCS_SERVER) $(SRCS_ENGINE) $(SRCS_SERVICE) $(LDLIBS) @@ -374,8 +385,8 @@ $(BUILD)/test_batch_metal: src/test_batch.c $(SRCS_ENGINE) $(METAL_ENGINE_OBJS) $(CC) $(CFLAGS) -DEI_ENABLE_METAL -o $@ src/test_batch.c $(SRCS_ENGINE) \ $(METAL_ENGINE_OBJS) $(LDLIBS) $(METALLIB_LDFLAGS) $(METAL_FRAMEWORKS) -$(BUILD)/perf_kernels: perf/harness/bench_kernels.c src/quants.c src/kernels.c src/gguf.c $(HDRS_NATIVE) | $(BUILD) - $(CC) $(CFLAGS) -Isrc -o $@ perf/harness/bench_kernels.c src/quants.c src/kernels.c src/gguf.c $(LDLIBS) +$(BUILD)/perf_kernels: perf/harness/bench_kernels.c $(SRCS_KERNELS) src/gguf.c $(HDRS_NATIVE) | $(BUILD) + $(CC) $(CFLAGS) -Isrc -o $@ perf/harness/bench_kernels.c $(SRCS_KERNELS) src/gguf.c $(LDLIBS) $(BUILD)/perf_engine: perf/harness/bench_engine.c $(SRCS_ENGINE) $(HDRS_NATIVE) | $(BUILD) $(CC) $(CFLAGS) -Isrc -o $@ perf/harness/bench_engine.c $(SRCS_ENGINE) $(LDLIBS) @@ -433,13 +444,15 @@ $(BUILD)/xpu: mkdir -p $(BUILD)/xpu .PHONY: test test-unit test-defaults check test-http test-http-metal test-http-cuda test-http-rocm test-http-xpu test-metal test-cuda test-rocm test-xpu perf perf-engine perf-engine-metal perf-engine-cuda perf-engine-rocm perf-engine-xpu perf-concurrency perf-concurrency-cuda perf-concurrency-rocm perf-concurrency-xpu perf-dimensions perf-batch perf-tokenization perf-servers perf-compare-llamacpp metal metal-kernels cuda rocm xpu check-scripts clean clean-cpu clean-metal clean-cuda clean-rocm clean-xpu release-darwin release-linux-cpu release-linux-cuda release-linux-rocm release-linux-xpu release-checksums release-verify release-ready release-info help -test: $(BUILD)/embeddinggemma $(BUILD)/test_gguf $(BUILD)/test_tokenizer $(BUILD)/test_kernels $(BUILD)/test_embed $(BUILD)/test_batch $(BUILD)/test_inference_service $(BUILD)/test_response_cache $(BUILD)/test_float_format +test: $(BUILD)/embeddinggemma $(BUILD)/test_gguf $(BUILD)/test_tokenizer $(BUILD)/test_kernels $(BUILD)/test_embed $(BUILD)/test_batch $(BUILD)/test_cpu_dispatch $(BUILD)/test_inference_service $(BUILD)/test_response_cache $(BUILD)/test_float_format python3 testdata/test_model_manifest.py --binary ./$(BUILD)/test_gguf \ --model $(MODEL) --manifest testdata/model-manifest.json ./$(BUILD)/test_tokenizer $(MODEL) testdata/goldens-tokens.json ./$(BUILD)/test_kernels ./$(BUILD)/test_embed $(MODEL) testdata/goldens-llamacpp.json + EI_CPU_ISA=baseline ./$(BUILD)/test_embed $(MODEL) testdata/goldens-llamacpp.json ./$(BUILD)/test_batch $(MODEL) cpu + ./$(BUILD)/test_cpu_dispatch $(MODEL) testdata/test-strings.json ./$(BUILD)/test_inference_service ./$(BUILD)/test_response_cache ./$(BUILD)/test_float_format @@ -736,3 +749,14 @@ ROUTE ?= TENSOR=1,FLASH=1 .PHONY: bench-gemma2-metal bench-gemma2-metal: $(BUILD)/bench_gemma2_metal ./$(BUILD)/bench_gemma2_metal $(GEMMA2_MODEL) "$(ROUTE)" $(CASES) +# Process fixture contains fictional bytes, never a model or network request. +$(BUILD)/test_downloader_fixture: src/test_downloader_fixture.c | $(BUILD) + $(CC) $(CFLAGS) -o $@ $< $(LDLIBS) + +.PHONY: test-windows-download +$(BUILD)/test_windows_paths: src/test_windows_paths.c src/windows_compat.h src/common.h | $(BUILD) + $(CC) $(CFLAGS) -o $@ $< $(LDLIBS) + +test-windows-download: $(BUILD)/embeddinggemma $(BUILD)/test_downloader_fixture $(BUILD)/test_windows_paths + ./$(BUILD)/test_windows_paths + python3 testdata/test_windows_download.py --binary ./$(BUILD)/embeddinggemma.exe --downloader ./$(BUILD)/test_downloader_fixture.exe diff --git a/PORT_SPEC.md b/PORT_SPEC.md index 9e6c331..2d342b2 100644 --- a/PORT_SPEC.md +++ b/PORT_SPEC.md @@ -144,7 +144,7 @@ llama.cpp deep-read) before implementing. | target | status | implementation/build | |---|---|---| | macOS ARM64 CPU | implemented, parity tested | C11 + NEON dot-product path | -| x86 CPU | implemented, parity tested | scalar, SSSE3, and AVX2 paths | +| x86 CPU | implemented, parity tested | scalar, SSSE3, and runtime-dispatched AVX2 paths | | macOS Metal | implemented, parity tested | Objective-C host + standalone metallib; short-shape residual/next-norm fusion; FP16 K/V at long sequence lengths with FP32 accumulation | | NVIDIA CUDA | implemented, parity tested | Q4 x Q8 DP4A latency path; direct-FP16 combined-QKV epilogue; Flash, online, banded-SWA, and tensor-core attention; native packed-Q4 MMA diagnostics; CUDA graph replay | | ROCm/HIP | implemented, parity tested | portable CDNA fat binary; paired packed-Q4 wave and MFMA projection kernels; exact singleton V-only attention; batched hipBLAS GQA; FP16/FP32 score routing; fused residual/RMS and pooling epilogues | diff --git a/README.md b/README.md index 80ece24..525c102 100644 --- a/README.md +++ b/README.md @@ -292,6 +292,33 @@ Published executables are deliberately small. Release `v0.3.1` contains: | Linux x86_64 | ROCm | 1.2 MiB | | Linux x86_64 | XPU SYCL | 2.1 MiB | +### Fork CI binaries + +This fork's `CI` workflow uploads checksummed CPU executable artifacts for +Linux x86_64, Linux ARM64, macOS ARM64, and Windows x86_64 after unit, model, +and HTTP conformance checks pass. Download artifacts from the workflow run; +these are CI builds, not upstream release assets. Existing CUDA, ROCm, XPU +and Metal release targets remain available through the upstream build process. + +CPU builds use the compiler's baseline architecture, without `-march=native`: +x86_64 uses SSE2 and ARM64 uses NEON. On x86_64 the same executable switches to +AVX2/FMA/F16C kernels at startup when the CPU supports them (Haswell, Zen, and +newer); set `EI_CPU_ISA=baseline` to force the SSE2 route. Linux artifacts are built on Ubuntu 24.04 +and require a compatible glibc (2.39 or later); macOS ARM64 is tested on the macOS 15 runner and targets macOS 14 or +later. Windows uses a native MinGW UCRT64 executable with statically linked +winpthreads and compiler runtime, tested on Windows Server 2022. It requires +Windows' Universal C Runtime; older Windows versions are not CI-tested. +No Windows GPU backend or PowerShell installer is provided by this change. + +On Windows, run `embeddinggemma-windows-x86_64-cpu.exe`. The default model +cache is `%LOCALAPPDATA%/embeddinggemma.c/`, falling back to +`%USERPROFILE%/.cache/embeddinggemma.c/`; `XDG_CACHE_HOME`, `EI_MODEL_PATH`, and +`--model` retain their existing precedence. Automatic download uses native +`curl.exe` or `wget.exe` on PATH and supports paths containing spaces. +Build locally from an MSYS2 UCRT64 shell with `make check`, +`make test-windows-download`, and `make test` after supplying the public model. +The model's Gemma license is separate from this repository's MIT license. + The 278 MB Q4_0 model is downloaded separately from Hugging Face and is not distributed by this project. diff --git a/perf/optimization_status.md b/perf/optimization_status.md index a0a0d76..aa8e629 100644 --- a/perf/optimization_status.md +++ b/perf/optimization_status.md @@ -3927,3 +3927,51 @@ dispatched 4 simdgroups for an 8-simdgroup kernel (NaN output); it now uses - Cross-request image packing (under 10% at 432x432, 3.5% at 768x768). - Hardware image decode (PNG decode 1.7-4.8 ms of a 21-56 ms miss). - Flash softmax fast exp and rescale skip (above). +## 2026-10-08: x86-64 Runtime AVX2 Dispatch + +Status: candidate (fork PR; upstream proposal pending). + +Current implementation: portable x86-64 CPU binaries are built at the compiler +baseline (SSE2), so the existing compile-time `__AVX2__` kernels never ran in +release or CI artifacts. Worse, the Q4_0 x Q8_0 dot products had no SSE2 path +at all, so shipped binaries ran the projections in scalar C +(`ei_quants_kernel_variant() == "cpu-scalar"`). + +Current public route: `src/kernels_avx2.c` holds the AVX2 kernels behind +per-function `target("avx2,fma,f16c")` attributes. `kernels.c` and `quants.c` +call them through `EI_DISPATCH_AVX2` after a one-time CPUID + XGETBV check +(AVX2, FMA, F16C, OS YMM state). `EI_CPU_ISA=auto|baseline|avx2` overrides it. +`scripts/check-avx2-isolation.py` fails CI if VEX code appears outside the +`ei_avx2_*` functions. ARM64 NEON object code is byte-identical to before. + +Correctness: `test_cpu_dispatch` embeds the test strings plus 150- and +600-token inputs and a packed batch under both routes in one process, gated at +cosine >= 0.999 and max abs diff <= 1e-2 at D=768/512/256/128 (the golden +gate; a last-bit summation difference can flip a Q8_0 activation rounding and +cascade, so the routes are not bit-identical on the real model). Against the +llama.cpp goldens on the real model, AVX2 min cosine is 0.999861 and SSE2 is +0.999871. +`test_kernels` runs every kernel test under both routes and checks that Q8_0 +quantization is bit-identical and Q4_0 rows/dual/batch agree within 3e-7. + +Experiments (sandbox Intel Xeon @ 2.80 GHz, 4 vCPU, GCC 13.3, synthetic +random-weight GGUF with the production shapes, 150-token chunk, best of 3): + +- Moving the existing AVX2 kernels behind dispatch: SSE2 1,326 ms vs AVX2 + 535 ms (2.5x). +- Callgrind on the AVX2 route showed 56% of instructions in the software + `ei_fp16_to_fp32` block-scale conversion. Hardware F16C (`_cvtsh_ss`, exact) + in the AVX2 kernels: 387 ms (3.3x vs SSE2), output bit-identical. +- FMA accumulation in the Q4_0 dots: q4gemv 25-27 vs 20-24 GFLOPS with + separate mul+add. Retained. +- `EI_CPU_MULTIROW_MIN_TOKENS=1` or `64` (four-token batch kernel) at 150 + tokens: 461-465 ms vs 387 ms. Rejected; the 512 boundary stays. + +Decision: retain runtime dispatch, F16C, and FMA. + +Open questions: the remaining gap to llama.cpp at 150 tokens is structural, +not ISA width. The CPU engine still runs per-token GEMV below 512 tokens and +re-streams all ~57 MB of layer weights for every token. llama.cpp's x86 +prefill uses tiled/repacked Q4_0 GEMM. A register-blocked Q4_0 x Q8_0 GEMM +(several rows x several tokens per tile) is the next CPU lever; AVX-VNNI would +come after that. diff --git a/scripts/check-avx2-isolation.py b/scripts/check-avx2-isolation.py new file mode 100755 index 0000000..7f88281 --- /dev/null +++ b/scripts/check-avx2-isolation.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Verify that a portable x86-64 executable keeps AVX code inside the +runtime-dispatched kernels: ei_avx2_* (kernels_avx2.c) and the EmbeddingGemma 2 +g2x_* / q8_tile_* kernels (gemma2_fast_x86.c, behind __builtin_cpu_supports). + +Every other function must run on an SSE2-only CPU, so it must not contain a +VEX-encoded instruction (mnemonics starting with "v") or touch a ymm/zmm +register. Fails if the dispatched kernels are missing or AVX leaks elsewhere. +""" +import argparse +import re +import subprocess +import sys + +FUNCTION = re.compile(r"^[0-9a-f]+ <(?P[^>]+)>:$") +DISPATCHED_PREFIXES = ("ei_avx2_", "g2x_", "q8_tile_") +INSTRUCTION = re.compile(r"^\s*[0-9a-f]+:\s+(?P[a-z][a-z0-9.]*)\s*(?P.*)$") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary") + parser.add_argument("--objdump", default="objdump") + args = parser.parse_args() + + listing = subprocess.run( + [args.objdump, "-d", "--no-show-raw-insn", args.binary], + check=True, capture_output=True, text=True, + ).stdout + + function = None + dispatched = set() + leaks = {} + for line in listing.splitlines(): + match = FUNCTION.match(line) + if match: + function = match.group("name") + continue + match = INSTRUCTION.match(line) + if not match or function is None: + continue + mnemonic = match.group("mnemonic") + operands = match.group("operands") + if not (mnemonic.startswith("v") or "%ymm" in operands or "%zmm" in operands): + continue + if function.startswith("ei_avx2_"): + dispatched.add(function) + elif function.startswith(DISPATCHED_PREFIXES): + pass + else: + leaks.setdefault(function, f"{mnemonic} {operands}".strip()) + + if leaks: + for name, example in sorted(leaks.items()): + print(f"AVX instruction outside dispatched kernels: {name}: {example}", + file=sys.stderr) + return 1 + if not dispatched: + print("no ei_avx2_* kernels found; runtime dispatch is missing", file=sys.stderr) + return 1 + print(f"AVX code isolated to {len(dispatched)} dispatched ei_avx2_* kernels") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-binary-platform.py b/scripts/check-binary-platform.py new file mode 100644 index 0000000..4f46d7c --- /dev/null +++ b/scripts/check-binary-platform.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Reject a binary whose native format or architecture disagrees with its label.""" +from pathlib import Path +import struct +import sys + +platform, filename = sys.argv[1:] +data = Path(filename).read_bytes() +if platform.startswith("linux-"): + assert data[:6] == b"\x7fELF\x02\x01", "expected a little-endian ELF64 executable" + machine = struct.unpack_from(" diff --git a/src/gemma2_fast.c b/src/gemma2_fast.c index f69362a..db4163f 100644 --- a/src/gemma2_fast.c +++ b/src/gemma2_fast.c @@ -83,7 +83,9 @@ struct ei_gemma2_fast { workspace ws; }; static ei_gemma2_fast default_ctx = {PTHREAD_MUTEX_INITIALIZER, {0}}; +#if (defined(__aarch64__) && defined(__APPLE__)) || defined(__x86_64__) static pthread_mutex_t weight_cache_lock = PTHREAD_MUTEX_INITIALIZER; +#endif static void reserve(workspace *ws, size_t n) { if (ws->tokens >= n) diff --git a/src/gguf.c b/src/gguf.c index 69c2314..2940fcf 100644 --- a/src/gguf.c +++ b/src/gguf.c @@ -1,7 +1,11 @@ #include "gguf.h" #include +#ifdef _WIN32 +#include "windows_compat.h" +#else #include +#endif #include #include @@ -119,6 +123,21 @@ static uint64_t tensor_bytes(const ei_tensor *t, const char *namebuf) { void ei_gguf_open(ei_gguf *g, const char *path) { memset(g, 0, sizeof *g); +#ifdef _WIN32 + HANDLE file = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file == INVALID_HANDLE_VALUE) ei_die("%s: cannot open", path); + LARGE_INTEGER size; + if (!GetFileSizeEx(file, &size) || size.QuadPart <= 0 || + (uint64_t)size.QuadPart > SIZE_MAX) ei_die("%s: cannot stat", path); + g->map_len = (size_t)size.QuadPart; + HANDLE mapping = CreateFileMappingA(file, NULL, PAGE_READONLY, 0, 0, NULL); + CloseHandle(file); + if (!mapping) ei_die("%s: cannot create mapping", path); + g->map = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0); + CloseHandle(mapping); + if (!g->map) ei_die("%s: cannot map file", path); +#else int fd = open(path, O_RDONLY); if (fd < 0) ei_die("%s: cannot open", path); struct stat st; @@ -128,6 +147,8 @@ void ei_gguf_open(ei_gguf *g, const char *path) { close(fd); if (g->map == MAP_FAILED) ei_die("%s: mmap failed", path); +#endif + cur c = { g->map, g->map + g->map_len, path }; if (take_u32(&c) != GGUF_MAGIC) ei_die("%s: not a GGUF file", path); @@ -203,7 +224,11 @@ void ei_gguf_close(ei_gguf *g) { } free(g->kv); free(g->tensors); +#ifdef _WIN32 + if (g->map) UnmapViewOfFile(g->map); +#else if (g->map) munmap((void *)g->map, g->map_len); +#endif memset(g, 0, sizeof *g); } diff --git a/src/inference_service.c b/src/inference_service.c index 90a0c3f..5cba6a4 100644 --- a/src/inference_service.c +++ b/src/inference_service.c @@ -9,6 +9,7 @@ #include #include #include +#include "windows_compat.h" typedef enum { EI_ENTRY_PENDING, @@ -847,7 +848,7 @@ static void write_persistent_cache_locked(ei_inference_service *service) { fwrite(&count, sizeof count, 1, file) == 1; } if (fclose(file) != 0) ok = false; - if (ok && rename(tmp, path) == 0) { + if (ok && ei_replace_file(tmp, path) == 0) { fprintf(stderr, "persisted %llu cached embeddings to %s\n", (unsigned long long)count, path); } else { diff --git a/src/kernels.c b/src/kernels.c index 0732269..7594120 100644 --- a/src/kernels.c +++ b/src/kernels.c @@ -1,18 +1,23 @@ #include "kernels.h" +#include "kernels_avx2.h" #include #if defined(__ARM_NEON) && defined(__aarch64__) #include -#elif defined(__AVX2__) || defined(__SSE2__) -#include +#elif defined(__SSE2__) +#include #endif +/* On x86-64, EI_DISPATCH_AVX2 hands off to kernels_avx2.c at runtime; the + * inline paths below are the NEON, SSE2 baseline, and scalar routes. */ + const char *ei_cpu_kernel_variant(void) { +#if defined(EI_X86_DISPATCH) + if (ei_cpu_avx2_active()) return "cpu-avx2"; +#endif #if defined(__ARM_NEON) && defined(__aarch64__) return "cpu-neon"; -#elif defined(__AVX2__) - return "cpu-avx2"; #elif defined(__SSSE3__) return "cpu-ssse3"; #elif defined(__SSE2__) @@ -22,16 +27,7 @@ const char *ei_cpu_kernel_variant(void) { #endif } -#if defined(__AVX2__) -static inline float ei_hsum_f32x8(__m256 v) { - __m128 sum = _mm_add_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps(v, 1)); - sum = _mm_add_ps(sum, _mm_movehl_ps(sum, sum)); - sum = _mm_add_ss(sum, _mm_shuffle_ps(sum, sum, 1)); - return _mm_cvtss_f32(sum); -} -#endif - -#if defined(__SSE2__) && !defined(__AVX2__) +#if defined(__SSE2__) static inline float ei_hsum_f32x4(__m128 v) { v = _mm_add_ps(v, _mm_movehl_ps(v, v)); v = _mm_add_ss(v, _mm_shuffle_ps(v, v, 1)); @@ -40,6 +36,7 @@ static inline float ei_hsum_f32x4(__m128 v) { #endif static float ei_sum_squares(const float *x, int32_t n) { + EI_DISPATCH_AVX2(return ei_avx2_sum_squares(x, n)); #if defined(__ARM_NEON) && defined(__aarch64__) float32x4_t sum0 = vdupq_n_f32(0.0f); float32x4_t sum1 = vdupq_n_f32(0.0f); @@ -59,26 +56,6 @@ static float ei_sum_squares(const float *x, int32_t n) { float sum = vaddvq_f32(vaddq_f32(vaddq_f32(sum0, sum1), vaddq_f32(sum2, sum3))); for (; i < n; i++) sum += x[i] * x[i]; return sum; -#elif defined(__AVX2__) - __m256 sum0 = _mm256_setzero_ps(); - __m256 sum1 = _mm256_setzero_ps(); - __m256 sum2 = _mm256_setzero_ps(); - __m256 sum3 = _mm256_setzero_ps(); - int32_t i = 0; - for (; i + 31 < n; i += 32) { - __m256 x0 = _mm256_loadu_ps(x + i); - __m256 x1 = _mm256_loadu_ps(x + i + 8); - __m256 x2 = _mm256_loadu_ps(x + i + 16); - __m256 x3 = _mm256_loadu_ps(x + i + 24); - sum0 = _mm256_add_ps(sum0, _mm256_mul_ps(x0, x0)); - sum1 = _mm256_add_ps(sum1, _mm256_mul_ps(x1, x1)); - sum2 = _mm256_add_ps(sum2, _mm256_mul_ps(x2, x2)); - sum3 = _mm256_add_ps(sum3, _mm256_mul_ps(x3, x3)); - } - float sum = ei_hsum_f32x8(_mm256_add_ps(_mm256_add_ps(sum0, sum1), - _mm256_add_ps(sum2, sum3))); - for (; i < n; i++) sum += x[i] * x[i]; - return sum; #elif defined(__SSE2__) __m128 sum0 = _mm_setzero_ps(); __m128 sum1 = _mm_setzero_ps(); @@ -101,6 +78,7 @@ static float ei_sum_squares(const float *x, int32_t n) { static void ei_norm_scale(const float *x, const float *w, int32_t n, float scale, float *out) { + EI_DISPATCH_AVX2(ei_avx2_norm_scale(x, w, n, scale, out); return); #if defined(__ARM_NEON) && defined(__aarch64__) const float32x4_t scale4 = vdupq_n_f32(scale); int32_t i = 0; @@ -111,16 +89,6 @@ static void ei_norm_scale(const float *x, const float *w, int32_t n, vst1q_f32(out + i + 12, vmulq_f32(vmulq_f32(vld1q_f32(x + i + 12), vld1q_f32(w + i + 12)), scale4)); } for (; i < n; i++) out[i] = x[i] * scale * w[i]; -#elif defined(__AVX2__) - const __m256 scale8 = _mm256_set1_ps(scale); - int32_t i = 0; - for (; i + 15 < n; i += 16) { - _mm256_storeu_ps(out + i, _mm256_mul_ps(_mm256_mul_ps(_mm256_loadu_ps(x + i), - _mm256_loadu_ps(w + i)), scale8)); - _mm256_storeu_ps(out + i + 8, _mm256_mul_ps(_mm256_mul_ps(_mm256_loadu_ps(x + i + 8), - _mm256_loadu_ps(w + i + 8)), scale8)); - } - for (; i < n; i++) out[i] = x[i] * scale * w[i]; #elif defined(__SSE2__) const __m128 scale4 = _mm_set1_ps(scale); int32_t i = 0; @@ -163,6 +131,7 @@ void ei_rms_norm_inplace(float *x, const float *w, int32_t n, float eps) { void ei_rms_norm_residual_inplace(float *residual, const float *x, const float *w, int32_t n, float eps) { const float scale = 1.0f / sqrtf(ei_sum_squares(x, n) / (float)n + eps); + EI_DISPATCH_AVX2(ei_avx2_norm_residual(residual, x, w, n, scale); return); #if defined(__ARM_NEON) && defined(__aarch64__) const float32x4_t scale4 = vdupq_n_f32(scale); int32_t i = 0; @@ -173,15 +142,6 @@ void ei_rms_norm_residual_inplace(float *residual, const float *x, const float * vst1q_f32(residual + i + 4, vaddq_f32(vld1q_f32(residual + i + 4), y1)); } for (; i < n; i++) residual[i] += x[i] * scale * w[i]; -#elif defined(__AVX2__) - const __m256 scale8 = _mm256_set1_ps(scale); - int32_t i = 0; - for (; i + 7 < n; i += 8) { - __m256 y = _mm256_mul_ps(_mm256_mul_ps(_mm256_loadu_ps(x + i), - _mm256_loadu_ps(w + i)), scale8); - _mm256_storeu_ps(residual + i, _mm256_add_ps(_mm256_loadu_ps(residual + i), y)); - } - for (; i < n; i++) residual[i] += x[i] * scale * w[i]; #else for (int32_t i = 0; i < n; i++) residual[i] += x[i] * scale * w[i]; #endif @@ -203,6 +163,7 @@ void ei_rope_neox_inplace(float *x, int32_t n_heads, int32_t pos, float base) { } static void ei_vec_scale_inplace(float *x, float scale, int32_t n) { + EI_DISPATCH_AVX2(ei_avx2_vec_scale_inplace(x, scale, n); return); #if defined(__ARM_NEON) && defined(__aarch64__) const float32x4_t scale4 = vdupq_n_f32(scale); int32_t i = 0; @@ -211,11 +172,6 @@ static void ei_vec_scale_inplace(float *x, float scale, int32_t n) { vst1q_f32(x + i + 4, vmulq_f32(vld1q_f32(x + i + 4), scale4)); } for (; i < n; i++) x[i] *= scale; -#elif defined(__AVX2__) - const __m256 scale8 = _mm256_set1_ps(scale); - int32_t i = 0; - for (; i + 7 < n; i += 8) _mm256_storeu_ps(x + i, _mm256_mul_ps(_mm256_loadu_ps(x + i), scale8)); - for (; i < n; i++) x[i] *= scale; #else for (int32_t i = 0; i < n; i++) x[i] *= scale; #endif @@ -264,6 +220,7 @@ void ei_qk_norm_rope_qk_inplace(float *q, float *k, const float *q_weight, } static float ei_dot_f32(const float *a, const float *b, int32_t n) { + EI_DISPATCH_AVX2(return ei_avx2_dot_f32(a, b, n)); #if defined(__ARM_NEON) && defined(__aarch64__) float32x4_t sum0 = vdupq_n_f32(0.0f); float32x4_t sum1 = vdupq_n_f32(0.0f); @@ -275,17 +232,6 @@ static float ei_dot_f32(const float *a, const float *b, int32_t n) { float sum = vaddvq_f32(vaddq_f32(sum0, sum1)); for (; i < n; i++) sum += a[i] * b[i]; return sum; -#elif defined(__AVX2__) - __m256 sum0 = _mm256_setzero_ps(); - __m256 sum1 = _mm256_setzero_ps(); - int32_t i = 0; - for (; i + 15 < n; i += 16) { - sum0 = _mm256_add_ps(sum0, _mm256_mul_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i))); - sum1 = _mm256_add_ps(sum1, _mm256_mul_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8))); - } - float sum = ei_hsum_f32x8(_mm256_add_ps(sum0, sum1)); - for (; i < n; i++) sum += a[i] * b[i]; - return sum; #elif defined(__SSE2__) __m128 sum = _mm_setzero_ps(); int32_t i = 0; @@ -301,6 +247,7 @@ static float ei_dot_f32(const float *a, const float *b, int32_t n) { } static void ei_axpy(float *dst, const float *src, float scale, int32_t n) { + EI_DISPATCH_AVX2(ei_avx2_axpy(dst, src, scale, n); return); #if defined(__ARM_NEON) && defined(__aarch64__) int32_t i = 0; for (; i + 7 < n; i += 8) { @@ -308,14 +255,6 @@ static void ei_axpy(float *dst, const float *src, float scale, int32_t n) { vst1q_f32(dst + i + 4, vfmaq_n_f32(vld1q_f32(dst + i + 4), vld1q_f32(src + i + 4), scale)); } for (; i < n; i++) dst[i] += scale * src[i]; -#elif defined(__AVX2__) - const __m256 scale8 = _mm256_set1_ps(scale); - int32_t i = 0; - for (; i + 7 < n; i += 8) { - __m256 add = _mm256_mul_ps(_mm256_loadu_ps(src + i), scale8); - _mm256_storeu_ps(dst + i, _mm256_add_ps(_mm256_loadu_ps(dst + i), add)); - } - for (; i < n; i++) dst[i] += scale * src[i]; #else for (int32_t i = 0; i < n; i++) dst[i] += scale * src[i]; #endif diff --git a/src/kernels_avx2.c b/src/kernels_avx2.c new file mode 100644 index 0000000..8184fbb --- /dev/null +++ b/src/kernels_avx2.c @@ -0,0 +1,306 @@ +/* AVX2/FMA/F16C CPU kernels and x86-64 runtime ISA selection. + * + * This is the only translation unit with AVX code. Every kernel below carries + * EI_AVX2_TARGET rather than relying on -mavx2, so the Makefile keeps one + * baseline CFLAGS for every file and nothing outside these functions (headers, + * static inline helpers, initializers) is compiled for AVX2. Callers in + * kernels.c and quants.c reach these only after ei_cpu_avx2_active(). */ +#include "kernels_avx2.h" + +#include + +#if defined(EI_X86_DISPATCH) +#include +#include + +_Atomic int ei_cpu_isa_state = -1; + +/* AVX2, FMA and F16C in CPUID, and YMM state enabled by the OS (XCR0). */ +static bool ei_cpu_detect_avx2(void) { + unsigned eax, ebx, ecx, edx; + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx)) return false; + const unsigned fma = 1u << 12, osxsave = 1u << 27, avx = 1u << 28, f16c = 1u << 29; + if ((ecx & (fma | osxsave | avx | f16c)) != (fma | osxsave | avx | f16c)) return false; + unsigned xcr0_lo, xcr0_hi; + __asm__ volatile("xgetbv" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); + if ((xcr0_lo & 6u) != 6u) return false; /* XMM and YMM state */ + if (!__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx)) return false; + return (ebx & (1u << 5)) != 0; /* AVX2 */ +} + +bool ei_cpu_avx2_supported(void) { + return ei_cpu_detect_avx2(); +} + +static int ei_cpu_isa_from_name(const char *isa) { + if (!isa || !*isa || strcmp(isa, "auto") == 0) return ei_cpu_detect_avx2() ? 1 : 0; + if (strcmp(isa, "baseline") == 0 || strcmp(isa, "sse2") == 0) return 0; + if (strcmp(isa, "avx2") == 0) return ei_cpu_detect_avx2() ? 1 : -1; + return -1; +} + +int ei_cpu_resolve_isa(void) { + const char *env = getenv("EI_CPU_ISA"); + int state = ei_cpu_isa_from_name(env); + if (state < 0) { + ei_die("EI_CPU_ISA=%s is not supported here (use auto, baseline, or avx2 " + "on an AVX2+FMA+F16C CPU)", env); + } + atomic_store_explicit(&ei_cpu_isa_state, state, memory_order_relaxed); + return state; +} + +bool ei_cpu_set_isa(const char *isa) { + int state = ei_cpu_isa_from_name(isa); + if (state < 0) return false; + atomic_store_explicit(&ei_cpu_isa_state, state, memory_order_relaxed); + return true; +} + +#define EI_AVX2_TARGET __attribute__((target("avx2,fma,f16c"))) + +/* Hardware half -> float; exact, so it matches ei_fp16_to_fp32 bit for bit. */ +EI_AVX2_TARGET static inline float ei_avx2_fp16(ei_fp16 h) { + return _cvtsh_ss(h); +} + +EI_AVX2_TARGET static inline float ei_avx2_hsum_f32x8(__m256 v) { + __m128 sum = _mm_add_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps(v, 1)); + sum = _mm_add_ps(sum, _mm_movehl_ps(sum, sum)); + sum = _mm_add_ss(sum, _mm_shuffle_ps(sum, sum, 1)); + return _mm_cvtss_f32(sum); +} + +EI_AVX2_TARGET float ei_avx2_sum_squares(const float *x, int32_t n) { + __m256 sum0 = _mm256_setzero_ps(); + __m256 sum1 = _mm256_setzero_ps(); + __m256 sum2 = _mm256_setzero_ps(); + __m256 sum3 = _mm256_setzero_ps(); + int32_t i = 0; + for (; i + 31 < n; i += 32) { + __m256 x0 = _mm256_loadu_ps(x + i); + __m256 x1 = _mm256_loadu_ps(x + i + 8); + __m256 x2 = _mm256_loadu_ps(x + i + 16); + __m256 x3 = _mm256_loadu_ps(x + i + 24); + sum0 = _mm256_add_ps(sum0, _mm256_mul_ps(x0, x0)); + sum1 = _mm256_add_ps(sum1, _mm256_mul_ps(x1, x1)); + sum2 = _mm256_add_ps(sum2, _mm256_mul_ps(x2, x2)); + sum3 = _mm256_add_ps(sum3, _mm256_mul_ps(x3, x3)); + } + float sum = ei_avx2_hsum_f32x8(_mm256_add_ps(_mm256_add_ps(sum0, sum1), + _mm256_add_ps(sum2, sum3))); + for (; i < n; i++) sum += x[i] * x[i]; + return sum; +} + +EI_AVX2_TARGET void ei_avx2_norm_scale(const float *x, const float *w, int32_t n, + float scale, float *out) { + const __m256 scale8 = _mm256_set1_ps(scale); + int32_t i = 0; + for (; i + 15 < n; i += 16) { + _mm256_storeu_ps(out + i, _mm256_mul_ps(_mm256_mul_ps(_mm256_loadu_ps(x + i), + _mm256_loadu_ps(w + i)), scale8)); + _mm256_storeu_ps(out + i + 8, _mm256_mul_ps(_mm256_mul_ps(_mm256_loadu_ps(x + i + 8), + _mm256_loadu_ps(w + i + 8)), scale8)); + } + for (; i < n; i++) out[i] = x[i] * scale * w[i]; +} + +EI_AVX2_TARGET void ei_avx2_norm_residual(float *residual, const float *x, const float *w, + int32_t n, float scale) { + const __m256 scale8 = _mm256_set1_ps(scale); + int32_t i = 0; + for (; i + 7 < n; i += 8) { + __m256 y = _mm256_mul_ps(_mm256_mul_ps(_mm256_loadu_ps(x + i), + _mm256_loadu_ps(w + i)), scale8); + _mm256_storeu_ps(residual + i, _mm256_add_ps(_mm256_loadu_ps(residual + i), y)); + } + for (; i < n; i++) residual[i] += x[i] * scale * w[i]; +} + +EI_AVX2_TARGET void ei_avx2_vec_scale_inplace(float *x, float scale, int32_t n) { + const __m256 scale8 = _mm256_set1_ps(scale); + int32_t i = 0; + for (; i + 7 < n; i += 8) _mm256_storeu_ps(x + i, _mm256_mul_ps(_mm256_loadu_ps(x + i), scale8)); + for (; i < n; i++) x[i] *= scale; +} + +EI_AVX2_TARGET float ei_avx2_dot_f32(const float *a, const float *b, int32_t n) { + __m256 sum0 = _mm256_setzero_ps(); + __m256 sum1 = _mm256_setzero_ps(); + int32_t i = 0; + for (; i + 15 < n; i += 16) { + sum0 = _mm256_add_ps(sum0, _mm256_mul_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i))); + sum1 = _mm256_add_ps(sum1, _mm256_mul_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8))); + } + float sum = ei_avx2_hsum_f32x8(_mm256_add_ps(sum0, sum1)); + for (; i < n; i++) sum += a[i] * b[i]; + return sum; +} + +EI_AVX2_TARGET void ei_avx2_axpy(float *dst, const float *src, float scale, int32_t n) { + const __m256 scale8 = _mm256_set1_ps(scale); + int32_t i = 0; + for (; i + 7 < n; i += 8) { + __m256 add = _mm256_mul_ps(_mm256_loadu_ps(src + i), scale8); + _mm256_storeu_ps(dst + i, _mm256_add_ps(_mm256_loadu_ps(dst + i), add)); + } + for (; i < n; i++) dst[i] += scale * src[i]; +} + +EI_AVX2_TARGET void ei_avx2_dequantize_row_q8_0(const ei_block_q8_0 *blocks, + uint64_t row_blocks, float scale, + float *out) { + for (uint64_t b = 0; b < row_blocks; b++) { + const __m256 d8 = _mm256_set1_ps(ei_avx2_fp16(blocks[b].d) * scale); + for (int j = 0; j < EI_QK; j += 8) { + const __m128i q8 = _mm_loadl_epi64((const __m128i *)(blocks[b].qs + j)); + const __m256i q32 = _mm256_cvtepi8_epi32(q8); + _mm256_storeu_ps(out + b * EI_QK + (uint64_t)j, + _mm256_mul_ps(_mm256_cvtepi32_ps(q32), d8)); + } + } +} + +EI_AVX2_TARGET void ei_avx2_quantize_row_q8_0(const float *x, ei_block_q8_0 *out, + int32_t nb) { + const __m256 abs_mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffff)); + const __m256 half = _mm256_set1_ps(0.5f); + const __m256 sign = _mm256_set1_ps(-0.0f); + const __m256i order = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); + for (int32_t b = 0; b < nb; b++) { + const float *xb = x + b * EI_QK; + __m256 maxv = _mm256_setzero_ps(); + for (int j = 0; j < EI_QK; j += 8) { + maxv = _mm256_max_ps(maxv, _mm256_and_ps(_mm256_loadu_ps(xb + j), abs_mask)); + } + __m128 max4 = _mm_max_ps(_mm256_castps256_ps128(maxv), _mm256_extractf128_ps(maxv, 1)); + max4 = _mm_max_ps(max4, _mm_movehl_ps(max4, max4)); + max4 = _mm_max_ss(max4, _mm_shuffle_ps(max4, max4, 1)); + const float amax = _mm_cvtss_f32(max4); + const float d = amax / 127.0f; + out[b].d = ei_fp32_to_fp16(d); + if (amax == 0.0f) { + memset(out[b].qs, 0, sizeof out[b].qs); + continue; + } + const __m256 scale = _mm256_set1_ps(1.0f / d); + __m256i quants[4]; + for (int j = 0; j < 4; j++) { + __m256 values = _mm256_mul_ps(_mm256_loadu_ps(xb + j * 8), scale); + __m256 signed_half = _mm256_or_ps(_mm256_and_ps(values, sign), half); + quants[j] = _mm256_cvttps_epi32(_mm256_add_ps(values, signed_half)); + } + __m256i q01 = _mm256_packs_epi32(quants[0], quants[1]); + __m256i q23 = _mm256_packs_epi32(quants[2], quants[3]); + __m256i q8 = _mm256_packs_epi16(q01, q23); + _mm256_storeu_si256((__m256i *)out[b].qs, _mm256_permutevar8x32_epi32(q8, order)); + } +} + +/* Signed 4-bit weights (low nibbles in lanes 0-15, high nibbles in 16-31) for + * one Q4_0 block, matching the Q8_0 activation layout. */ +EI_AVX2_TARGET static inline __m256i ei_avx2_unpack_q4_0(const ei_block_q4_0 *block) { + const __m128i packed = _mm_loadu_si128((const __m128i *)block->qs); + const __m256i q4 = _mm256_inserti128_si256( + _mm256_castsi128_si256(packed), _mm_srli_epi16(packed, 4), 1); + return _mm256_sub_epi8(_mm256_and_si256(q4, _mm256_set1_epi8(0x0F)), + _mm256_set1_epi8(8)); +} + +/* Eight int32 partial sums of the 32 int8 products, as floats. */ +EI_AVX2_TARGET static inline __m256 ei_avx2_dot_i8x32(__m256i x, __m256i y) { + const __m256i dot16 = _mm256_maddubs_epi16(_mm256_sign_epi8(x, x), + _mm256_sign_epi8(y, x)); + return _mm256_cvtepi32_ps(_mm256_madd_epi16(dot16, _mm256_set1_epi16(1))); +} + +EI_AVX2_TARGET float ei_avx2_vec_dot_q4_0_q8_0(const ei_block_q4_0 *x, + const ei_block_q8_0 *y, int32_t nb) { + __m256 sum = _mm256_setzero_ps(); + for (int32_t b = 0; b < nb; b++) { + const __m256i yq = _mm256_loadu_si256((const __m256i *)y[b].qs); + const __m256 dot = ei_avx2_dot_i8x32(ei_avx2_unpack_q4_0(x + b), yq); + const __m256 scale = _mm256_set1_ps(ei_avx2_fp16(x[b].d) * + ei_avx2_fp16(y[b].d)); + sum = _mm256_fmadd_ps(dot, scale, sum); + } + return ei_avx2_hsum_f32x8(sum); +} + +EI_AVX2_TARGET void ei_avx2_vec_dot_q4_0_q8_0_batch4(const ei_block_q4_0 *weights, + const ei_block_q8_0 *inputs, + int32_t input_stride, int32_t count, + int32_t nb, float out[4]) { + __m256 sums[4] = { + _mm256_setzero_ps(), _mm256_setzero_ps(), + _mm256_setzero_ps(), _mm256_setzero_ps(), + }; + for (int32_t block = 0; block < nb; block++) { + const __m256i weight_values = ei_avx2_unpack_q4_0(weights + block); + const float weight_scale = ei_avx2_fp16(weights[block].d); + for (int32_t input = 0; input < count; input++) { + const ei_block_q8_0 *activation = inputs + input * input_stride + block; + const __m256i values = _mm256_loadu_si256((const __m256i *)activation->qs); + const __m256 scale = _mm256_set1_ps( + weight_scale * ei_avx2_fp16(activation->d)); + sums[input] = _mm256_fmadd_ps(ei_avx2_dot_i8x32(weight_values, values), + scale, sums[input]); + } + } + for (int32_t input = 0; input < count; input++) out[input] = ei_avx2_hsum_f32x8(sums[input]); +} + +EI_AVX2_TARGET void ei_avx2_vec_dot_q4_0_q8_0_dual(const ei_block_q4_0 *x0, + const ei_block_q4_0 *x1, + const ei_block_q8_0 *y, int32_t nb, + float *out0, float *out1) { + __m256 sum0 = _mm256_setzero_ps(); + __m256 sum1 = _mm256_setzero_ps(); + for (int32_t b = 0; b < nb; b++) { + const __m256i yq = _mm256_loadu_si256((const __m256i *)y[b].qs); + const float y_scale = ei_avx2_fp16(y[b].d); + sum0 = _mm256_fmadd_ps(ei_avx2_dot_i8x32(ei_avx2_unpack_q4_0(x0 + b), yq), + _mm256_set1_ps(ei_avx2_fp16(x0[b].d) * y_scale), sum0); + sum1 = _mm256_fmadd_ps(ei_avx2_dot_i8x32(ei_avx2_unpack_q4_0(x1 + b), yq), + _mm256_set1_ps(ei_avx2_fp16(x1[b].d) * y_scale), sum1); + } + *out0 = ei_avx2_hsum_f32x8(sum0); + *out1 = ei_avx2_hsum_f32x8(sum1); +} + +EI_AVX2_TARGET void ei_avx2_vec_dot_q4_0_q8_0_triple(const ei_block_q4_0 *x0, + const ei_block_q4_0 *x1, + const ei_block_q4_0 *x2, + const ei_block_q8_0 *y, int32_t nb, + float *out0, float *out1, + float *out2) { + __m256 sum0 = _mm256_setzero_ps(); + __m256 sum1 = _mm256_setzero_ps(); + __m256 sum2 = _mm256_setzero_ps(); + for (int32_t b = 0; b < nb; b++) { + const __m256i yq = _mm256_loadu_si256((const __m256i *)y[b].qs); + const float y_scale = ei_avx2_fp16(y[b].d); + sum0 = _mm256_fmadd_ps(ei_avx2_dot_i8x32(ei_avx2_unpack_q4_0(x0 + b), yq), + _mm256_set1_ps(ei_avx2_fp16(x0[b].d) * y_scale), sum0); + sum1 = _mm256_fmadd_ps(ei_avx2_dot_i8x32(ei_avx2_unpack_q4_0(x1 + b), yq), + _mm256_set1_ps(ei_avx2_fp16(x1[b].d) * y_scale), sum1); + sum2 = _mm256_fmadd_ps(ei_avx2_dot_i8x32(ei_avx2_unpack_q4_0(x2 + b), yq), + _mm256_set1_ps(ei_avx2_fp16(x2[b].d) * y_scale), sum2); + } + *out0 = ei_avx2_hsum_f32x8(sum0); + *out1 = ei_avx2_hsum_f32x8(sum1); + *out2 = ei_avx2_hsum_f32x8(sum2); +} + +#else /* !EI_X86_DISPATCH: NEON and other targets keep compile-time kernels. */ + +bool ei_cpu_avx2_supported(void) { + return false; +} + +bool ei_cpu_set_isa(const char *isa) { + return !isa || !*isa || strcmp(isa, "auto") == 0 || strcmp(isa, "baseline") == 0; +} + +#endif diff --git a/src/kernels_avx2.h b/src/kernels_avx2.h new file mode 100644 index 0000000..ee59f9b --- /dev/null +++ b/src/kernels_avx2.h @@ -0,0 +1,70 @@ +/* x86-64 runtime CPU dispatch for the AVX2/FMA/F16C CPU kernels. + * + * Portable x86-64 builds use the compiler baseline (SSE2). The AVX2 kernels + * live in kernels_avx2.c, where each function carries + * __attribute__((target("avx2,fma,f16c"))), so no other code is compiled with + * AVX instructions. Callers reach them only through ei_cpu_avx2_active(). */ +#ifndef EI_KERNELS_AVX2_H +#define EI_KERNELS_AVX2_H + +#include "quants.h" + +#include + +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) +#define EI_X86_DISPATCH 1 + +/* -1 until first use, then 0 (baseline) or 1 (AVX2 + FMA + F16C). */ +extern _Atomic int ei_cpu_isa_state; +int ei_cpu_resolve_isa(void); + +/* True when the CPU and OS support AVX2, FMA and F16C, unless EI_CPU_ISA=baseline + * (or a test) forced the SSE2 route. A relaxed load: cheap per kernel call. */ +static inline bool ei_cpu_avx2_active(void) { + int state = atomic_load_explicit(&ei_cpu_isa_state, memory_order_relaxed); + if (state < 0) state = ei_cpu_resolve_isa(); + return state == 1; +} + +float ei_avx2_sum_squares(const float *x, int32_t n); +void ei_avx2_norm_scale(const float *x, const float *w, int32_t n, + float scale, float *out); +void ei_avx2_norm_residual(float *residual, const float *x, const float *w, + int32_t n, float scale); +void ei_avx2_vec_scale_inplace(float *x, float scale, int32_t n); +float ei_avx2_dot_f32(const float *a, const float *b, int32_t n); +void ei_avx2_axpy(float *dst, const float *src, float scale, int32_t n); + +void ei_avx2_dequantize_row_q8_0(const ei_block_q8_0 *blocks, uint64_t row_blocks, + float scale, float *out); +void ei_avx2_quantize_row_q8_0(const float *x, ei_block_q8_0 *out, int32_t nb); +float ei_avx2_vec_dot_q4_0_q8_0(const ei_block_q4_0 *x, const ei_block_q8_0 *y, + int32_t nb); +void ei_avx2_vec_dot_q4_0_q8_0_batch4(const ei_block_q4_0 *weights, + const ei_block_q8_0 *inputs, + int32_t input_stride, int32_t count, + int32_t nb, float out[4]); +void ei_avx2_vec_dot_q4_0_q8_0_dual(const ei_block_q4_0 *x0, const ei_block_q4_0 *x1, + const ei_block_q8_0 *y, int32_t nb, + float *out0, float *out1); +void ei_avx2_vec_dot_q4_0_q8_0_triple(const ei_block_q4_0 *x0, const ei_block_q4_0 *x1, + const ei_block_q4_0 *x2, const ei_block_q8_0 *y, + int32_t nb, float *out0, float *out1, + float *out2); + +/* Runs `call` (which must return) when the AVX2 route is active. */ +#define EI_DISPATCH_AVX2(call) do { if (ei_cpu_avx2_active()) { call; } } while (0) +#else +#define EI_DISPATCH_AVX2(call) do { } while (0) +#endif + +/* Select the x86 kernel family: "auto" (default), "baseline" ("sse2"), or + * "avx2". Returns false for an unknown name or AVX2 on a CPU without it. + * Non-x86 builds accept only "auto" and "baseline". On x86-64 the EI_CPU_ISA + * environment variable sets the initial choice. Not for use while kernels run + * on other threads. */ +bool ei_cpu_set_isa(const char *isa); +/* True when this process can run the AVX2 route (x86-64 with AVX2, FMA and F16C). */ +bool ei_cpu_avx2_supported(void); + +#endif /* EI_KERNELS_AVX2_H */ diff --git a/src/parallel.c b/src/parallel.c index 50d0c56..7fc2f21 100644 --- a/src/parallel.c +++ b/src/parallel.c @@ -11,6 +11,7 @@ #include #include #include +#include "windows_compat.h" typedef struct { struct ei_thread_pool *pool; @@ -139,7 +140,13 @@ static void *worker_main(void *opaque) { } int32_t ei_cpu_count(void) { +#ifdef _WIN32 + SYSTEM_INFO info; + GetSystemInfo(&info); + long available = (long)info.dwNumberOfProcessors; +#else long available = sysconf(_SC_NPROCESSORS_ONLN); +#endif return available < 1 ? 1 : available > 4096 ? 4096 : (int32_t)available; } diff --git a/src/quants.c b/src/quants.c index 7716c5a..2fcb97f 100644 --- a/src/quants.c +++ b/src/quants.c @@ -1,4 +1,5 @@ #include "quants.h" +#include "kernels_avx2.h" #include @@ -6,10 +7,13 @@ #include #endif -#if defined(__AVX2__) || defined(__SSSE3__) -#include +#if defined(__SSSE3__) +#include #endif +/* On x86-64, EI_DISPATCH_AVX2 hands off to kernels_avx2.c at runtime; the + * inline paths below are the NEON, SSSE3, and scalar routes. */ + ei_fp16 ei_fp32_to_fp16(float f) { uint32_t x; memcpy(&x, &f, sizeof x); @@ -37,14 +41,15 @@ ei_fp16 ei_fp32_to_fp16(float f) { } const char *ei_quants_kernel_variant(void) { +#if defined(EI_X86_DISPATCH) + if (ei_cpu_avx2_active()) return "cpu-avx2"; +#endif #if defined(__ARM_NEON) && defined(__aarch64__) #if defined(__ARM_FEATURE_DOTPROD) return "cpu-neon-dotprod"; #else return "cpu-neon"; #endif -#elif defined(__AVX2__) - return "cpu-avx2"; #elif defined(__SSSE3__) return "cpu-ssse3"; #else @@ -63,6 +68,7 @@ void ei_dequantize_row_q8_0_scaled(const ei_tensor *t, int32_t row, float scale, uint64_t row_blocks = t->ne[0] / EI_QK; const ei_block_q8_0 *blocks = (const ei_block_q8_0 *)((const uint8_t *)t->data + (uint64_t)row * ei_tensor_row_bytes(t)); + EI_DISPATCH_AVX2(ei_avx2_dequantize_row_q8_0(blocks, row_blocks, scale, out); return); for (uint64_t b = 0; b < row_blocks; b++) { float d = ei_fp16_to_fp32(blocks[b].d) * scale; #if defined(__ARM_NEON) && defined(__aarch64__) @@ -74,14 +80,6 @@ void ei_dequantize_row_q8_0_scaled(const ei_tensor *t, int32_t row, float scale, vst1q_f32(out + b * EI_QK + (uint64_t)j, vmulq_f32(q0, d4)); vst1q_f32(out + b * EI_QK + (uint64_t)j + 4, vmulq_f32(q1, d4)); } -#elif defined(__AVX2__) - const __m256 d8 = _mm256_set1_ps(d); - for (int j = 0; j < EI_QK; j += 8) { - const __m128i q8 = _mm_loadl_epi64((const __m128i *)(blocks[b].qs + j)); - const __m256i q32 = _mm256_cvtepi8_epi32(q8); - _mm256_storeu_ps(out + b * EI_QK + (uint64_t)j, - _mm256_mul_ps(_mm256_cvtepi32_ps(q32), d8)); - } #else for (int j = 0; j < EI_QK; j++) { out[b * EI_QK + (uint64_t)j] = d * (float)blocks[b].qs[j]; @@ -98,6 +96,7 @@ void ei_quantize_row_q8_0(const float *x, ei_block_q8_0 *out, int32_t n) { if (n % EI_QK != 0) ei_die("q8_0 quantize length %d is not a multiple of 32", n); int32_t nb = n / EI_QK; + EI_DISPATCH_AVX2(ei_avx2_quantize_row_q8_0(x, out, nb); return); for (int32_t b = 0; b < nb; b++) { const float *xb = x + b * EI_QK; #if defined(__ARM_NEON) && defined(__aarch64__) @@ -108,16 +107,6 @@ void ei_quantize_row_q8_0(const float *x, ei_block_q8_0 *out, int32_t n) { max1 = vmaxq_f32(max1, vabsq_f32(vld1q_f32(xb + j + 4))); } float amax = vmaxvq_f32(vmaxq_f32(max0, max1)); -#elif defined(__AVX2__) - const __m256 abs_mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffff)); - __m256 maxv = _mm256_setzero_ps(); - for (int j = 0; j < EI_QK; j += 8) { - maxv = _mm256_max_ps(maxv, _mm256_and_ps(_mm256_loadu_ps(xb + j), abs_mask)); - } - __m128 max4 = _mm_max_ps(_mm256_castps256_ps128(maxv), _mm256_extractf128_ps(maxv, 1)); - max4 = _mm_max_ps(max4, _mm_movehl_ps(max4, max4)); - max4 = _mm_max_ss(max4, _mm_shuffle_ps(max4, max4, 1)); - float amax = _mm_cvtss_f32(max4); #else float amax = 0.0f; for (int j = 0; j < EI_QK; j++) { @@ -148,23 +137,6 @@ void ei_quantize_row_q8_0(const float *x, ei_block_q8_0 *out, int32_t n) { const int16x8_t q67 = vcombine_s16(vqmovn_s32(q6), vqmovn_s32(q7)); vst1q_s8(out[b].qs, vcombine_s8(vqmovn_s16(q01), vqmovn_s16(q23))); vst1q_s8(out[b].qs + 16, vcombine_s8(vqmovn_s16(q45), vqmovn_s16(q67))); -#elif defined(__AVX2__) - const __m256 scale = _mm256_set1_ps(id); - const __m256 half = _mm256_set1_ps(0.5f); - const __m256 sign = _mm256_set1_ps(-0.0f); - __m256 values[4]; - __m256i quants[4]; - for (int j = 0; j < 4; j++) { - values[j] = _mm256_mul_ps(_mm256_loadu_ps(xb + j * 8), scale); - __m256 signed_half = _mm256_or_ps(_mm256_and_ps(values[j], sign), half); - quants[j] = _mm256_cvttps_epi32(_mm256_add_ps(values[j], signed_half)); - } - __m256i q01 = _mm256_packs_epi32(quants[0], quants[1]); - __m256i q23 = _mm256_packs_epi32(quants[2], quants[3]); - __m256i q8 = _mm256_packs_epi16(q01, q23); - const __m256i order = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); - _mm256_storeu_si256((__m256i *)out[b].qs, - _mm256_permutevar8x32_epi32(q8, order)); #else for (int j = 0; j < EI_QK; j++) { int q = (int)roundf(xb[j] * id); @@ -210,56 +182,20 @@ static float ei_vec_dot_q4_0_q8_0_neon(const ei_block_q4_0 *x, } #endif -#if defined(__AVX2__) || defined(__SSSE3__) +#if defined(__SSSE3__) static inline float ei_hsum_f32x4(__m128 v) { v = _mm_add_ps(v, _mm_movehl_ps(v, v)); v = _mm_add_ss(v, _mm_shuffle_ps(v, v, 1)); return _mm_cvtss_f32(v); } -#if !defined(__AVX2__) static inline __m128i ei_mul_sum_i8_pairs_sse(__m128i x, __m128i y) { const __m128i ax = _mm_sign_epi8(x, x); const __m128i sy = _mm_sign_epi8(y, x); const __m128i dot16 = _mm_maddubs_epi16(ax, sy); return _mm_madd_epi16(dot16, _mm_set1_epi16(1)); } -#endif -#if defined(__AVX2__) -static inline float ei_hsum_f32x8(__m256 v) { - return ei_hsum_f32x4(_mm_add_ps(_mm256_castps256_ps128(v), - _mm256_extractf128_ps(v, 1))); -} - -static float ei_vec_dot_q4_0_q8_0_avx2(const ei_block_q4_0 *x, - const ei_block_q8_0 *y, - int32_t nb) { - const __m256i low_mask = _mm256_set1_epi8(0x0F); - const __m256i off = _mm256_set1_epi8(8); - const __m256i ones = _mm256_set1_epi16(1); - __m256 sum = _mm256_setzero_ps(); - - for (int32_t b = 0; b < nb; b++) { - const __m128i q4_128 = _mm_loadu_si128((const __m128i *)x[b].qs); - const __m256i q4 = _mm256_inserti128_si256( - _mm256_castsi128_si256(q4_128), _mm_srli_epi16(q4_128, 4), 1); - const __m256i xq = _mm256_sub_epi8(_mm256_and_si256(q4, low_mask), off); - const __m256i yq = _mm256_loadu_si256((const __m256i *)y[b].qs); - const __m256i ax = _mm256_sign_epi8(xq, xq); - const __m256i sy = _mm256_sign_epi8(yq, xq); - const __m256i dot16 = _mm256_maddubs_epi16(ax, sy); - const __m256 dot = _mm256_cvtepi32_ps(_mm256_madd_epi16(dot16, ones)); - const __m256 scale = _mm256_set1_ps(ei_fp16_to_fp32(x[b].d) * - ei_fp16_to_fp32(y[b].d)); - sum = _mm256_add_ps(sum, _mm256_mul_ps(dot, scale)); - } - - return ei_hsum_f32x8(sum); -} -#endif - -#if !defined(__AVX2__) static float ei_vec_dot_q4_0_q8_0_ssse3(const ei_block_q4_0 *x, const ei_block_q8_0 *y, int32_t nb) { @@ -283,15 +219,13 @@ static float ei_vec_dot_q4_0_q8_0_ssse3(const ei_block_q4_0 *x, return ei_hsum_f32x4(sum); } #endif -#endif float ei_vec_dot_q4_0_q8_0(const ei_block_q4_0 *x, const ei_block_q8_0 *y, int32_t n) { if (n % EI_QK != 0) ei_die("q4_0 dot length %d is not a multiple of 32", n); int32_t nb = n / EI_QK; + EI_DISPATCH_AVX2(return ei_avx2_vec_dot_q4_0_q8_0(x, y, nb)); #if defined(__ARM_NEON) && defined(__aarch64__) return ei_vec_dot_q4_0_q8_0_neon(x, y, nb); -#elif defined(__AVX2__) - return ei_vec_dot_q4_0_q8_0_avx2(x, y, nb); #elif defined(__SSSE3__) return ei_vec_dot_q4_0_q8_0_ssse3(x, y, nb); #else @@ -314,6 +248,8 @@ static void ei_vec_dot_q4_0_q8_0_batch4(const ei_block_q4_0 *weights, const ei_block_q8_0 *inputs, int32_t input_stride, int32_t count, int32_t n, float out[4]) { + EI_DISPATCH_AVX2(ei_avx2_vec_dot_q4_0_q8_0_batch4(weights, inputs, input_stride, + count, n / EI_QK, out); return); #if defined(__ARM_NEON) && defined(__aarch64__) int32_t nb = n / EI_QK; const uint8x16_t low_mask = vdupq_n_u8(0x0Fu); @@ -340,37 +276,6 @@ static void ei_vec_dot_q4_0_q8_0_batch4(const ei_block_q4_0 *weights, } } for (int32_t input = 0; input < count; input++) out[input] = vaddvq_f32(sums[input]); -#elif defined(__AVX2__) - int32_t nb = n / EI_QK; - const __m256i low_mask = _mm256_set1_epi8(0x0F); - const __m256i off = _mm256_set1_epi8(8); - const __m256i ones = _mm256_set1_epi16(1); - __m256 sums[4] = { - _mm256_setzero_ps(), _mm256_setzero_ps(), - _mm256_setzero_ps(), _mm256_setzero_ps(), - }; - for (int32_t block = 0; block < nb; block++) { - const __m128i packed = _mm_loadu_si128((const __m128i *)weights[block].qs); - const __m256i q4 = _mm256_inserti128_si256( - _mm256_castsi128_si256(packed), _mm_srli_epi16(packed, 4), 1); - const __m256i weight_values = _mm256_sub_epi8( - _mm256_and_si256(q4, low_mask), off); - const float weight_scale = ei_fp16_to_fp32(weights[block].d); - for (int32_t input = 0; input < count; input++) { - const ei_block_q8_0 *activation = inputs + input * input_stride + block; - const __m256i values = _mm256_loadu_si256( - (const __m256i *)activation->qs); - const __m256i dot16 = _mm256_maddubs_epi16( - _mm256_sign_epi8(weight_values, weight_values), - _mm256_sign_epi8(values, weight_values)); - const __m256 dot = _mm256_cvtepi32_ps( - _mm256_madd_epi16(dot16, ones)); - const __m256 scale = _mm256_set1_ps( - weight_scale * ei_fp16_to_fp32(activation->d)); - sums[input] = _mm256_add_ps(sums[input], _mm256_mul_ps(dot, scale)); - } - } - for (int32_t input = 0; input < count; input++) out[input] = ei_hsum_f32x8(sums[input]); #else for (int32_t input = 0; input < count; input++) { out[input] = ei_vec_dot_q4_0_q8_0( @@ -383,6 +288,7 @@ void ei_vec_dot_q4_0_q8_0_dual(const ei_block_q4_0 *x0, const ei_block_q4_0 *x1, const ei_block_q8_0 *y, int32_t n, float *out0, float *out1) { if (n % EI_QK != 0) ei_die("dual q4_0 dot length %d is not a multiple of 32", n); + EI_DISPATCH_AVX2(ei_avx2_vec_dot_q4_0_q8_0_dual(x0, x1, y, n / EI_QK, out0, out1); return); #if defined(__ARM_NEON) && defined(__aarch64__) int32_t nb = n / EI_QK; const uint8x16_t low_mask = vdupq_n_u8(0x0Fu); @@ -414,33 +320,6 @@ void ei_vec_dot_q4_0_q8_0_dual(const ei_block_q4_0 *x0, const ei_block_q4_0 *x1, } *out0 = vaddvq_f32(sum0); *out1 = vaddvq_f32(sum1); -#elif defined(__AVX2__) - int32_t nb = n / EI_QK; - const __m256i low_mask = _mm256_set1_epi8(0x0F); - const __m256i off = _mm256_set1_epi8(8); - const __m256i ones = _mm256_set1_epi16(1); - __m256 sum0 = _mm256_setzero_ps(); - __m256 sum1 = _mm256_setzero_ps(); - for (int32_t b = 0; b < nb; b++) { - const __m256i yq = _mm256_loadu_si256((const __m256i *)y[b].qs); - const float y_scale = ei_fp16_to_fp32(y[b].d); - const ei_block_q4_0 *blocks[2] = { x0 + b, x1 + b }; - __m256 *sums[2] = { &sum0, &sum1 }; - for (int i = 0; i < 2; i++) { - const __m128i packed = _mm_loadu_si128((const __m128i *)blocks[i]->qs); - const __m256i q4 = _mm256_inserti128_si256( - _mm256_castsi128_si256(packed), _mm_srli_epi16(packed, 4), 1); - const __m256i xq = _mm256_sub_epi8(_mm256_and_si256(q4, low_mask), off); - const __m256i dot16 = _mm256_maddubs_epi16( - _mm256_sign_epi8(xq, xq), _mm256_sign_epi8(yq, xq)); - const __m256 dot = _mm256_cvtepi32_ps(_mm256_madd_epi16(dot16, ones)); - const __m256 scale = _mm256_set1_ps( - ei_fp16_to_fp32(blocks[i]->d) * y_scale); - *sums[i] = _mm256_add_ps(*sums[i], _mm256_mul_ps(dot, scale)); - } - } - *out0 = ei_hsum_f32x8(sum0); - *out1 = ei_hsum_f32x8(sum1); #else *out0 = ei_vec_dot_q4_0_q8_0(x0, y, n); *out1 = ei_vec_dot_q4_0_q8_0(x1, y, n); @@ -451,6 +330,8 @@ void ei_vec_dot_q4_0_q8_0_triple(const ei_block_q4_0 *x0, const ei_block_q4_0 *x const ei_block_q4_0 *x2, const ei_block_q8_0 *y, int32_t n, float *out0, float *out1, float *out2) { if (n % EI_QK != 0) ei_die("triple q4_0 dot length %d is not a multiple of 32", n); + EI_DISPATCH_AVX2(ei_avx2_vec_dot_q4_0_q8_0_triple(x0, x1, x2, y, n / EI_QK, + out0, out1, out2); return); #if defined(__ARM_NEON) && defined(__aarch64__) int32_t nb = n / EI_QK; const uint8x16_t low_mask = vdupq_n_u8(0x0Fu); @@ -478,34 +359,6 @@ void ei_vec_dot_q4_0_q8_0_triple(const ei_block_q4_0 *x0, const ei_block_q4_0 *x *out0 = vaddvq_f32(sums[0]); *out1 = vaddvq_f32(sums[1]); *out2 = vaddvq_f32(sums[2]); -#elif defined(__AVX2__) - int32_t nb = n / EI_QK; - const __m256i low_mask = _mm256_set1_epi8(0x0F); - const __m256i off = _mm256_set1_epi8(8); - const __m256i ones = _mm256_set1_epi16(1); - __m256 sums[3] = { - _mm256_setzero_ps(), _mm256_setzero_ps(), _mm256_setzero_ps(), - }; - for (int32_t b = 0; b < nb; b++) { - const __m256i yq = _mm256_loadu_si256((const __m256i *)y[b].qs); - const float y_scale = ei_fp16_to_fp32(y[b].d); - const ei_block_q4_0 *blocks[3] = { x0 + b, x1 + b, x2 + b }; - for (int i = 0; i < 3; i++) { - const __m128i packed = _mm_loadu_si128((const __m128i *)blocks[i]->qs); - const __m256i q4 = _mm256_inserti128_si256( - _mm256_castsi128_si256(packed), _mm_srli_epi16(packed, 4), 1); - const __m256i xq = _mm256_sub_epi8(_mm256_and_si256(q4, low_mask), off); - const __m256i dot16 = _mm256_maddubs_epi16( - _mm256_sign_epi8(xq, xq), _mm256_sign_epi8(yq, xq)); - const __m256 dot = _mm256_cvtepi32_ps(_mm256_madd_epi16(dot16, ones)); - const __m256 scale = _mm256_set1_ps( - ei_fp16_to_fp32(blocks[i]->d) * y_scale); - sums[i] = _mm256_add_ps(sums[i], _mm256_mul_ps(dot, scale)); - } - } - *out0 = ei_hsum_f32x8(sums[0]); - *out1 = ei_hsum_f32x8(sums[1]); - *out2 = ei_hsum_f32x8(sums[2]); #else *out0 = ei_vec_dot_q4_0_q8_0(x0, y, n); *out1 = ei_vec_dot_q4_0_q8_0(x1, y, n); diff --git a/src/server.c b/src/server.c index 32a6cd8..8c2ae30 100644 --- a/src/server.c +++ b/src/server.c @@ -17,23 +17,52 @@ #include "gemma2_cuda.h" #endif -#include #include #include #include #include +#include +#include +#include +#include +#include +#include "windows_compat.h" +#ifdef _WIN32 +#include +#include +#include +typedef SOCKET ei_socket; +#define EI_INVALID_SOCKET INVALID_SOCKET +#define ei_socket_close closesocket +#else +#include #include #include #include -#include -#include -#include #include #include -#include #include #include -#include +typedef int ei_socket; +#define EI_INVALID_SOCKET (-1) +#define ei_socket_close close +#endif + +static int ei_socket_errno(void) { +#ifdef _WIN32 + int error = WSAGetLastError(); + if (error == WSAEINTR) return EINTR; + if (error == WSAETIMEDOUT || error == WSAEWOULDBLOCK) return EAGAIN; + return error; +#else + return errno; +#endif +} + +static int ei_setsockopt(ei_socket fd, int level, int name, + const void *value, int length) { + return setsockopt(fd, level, name, (const char *)value, length); +} static pthread_mutex_t inference_mutex = PTHREAD_MUTEX_INITIALIZER; // GPU media contexts (vision/audio towers and the mixed-input text forward): @@ -248,7 +277,9 @@ static bool media_audio_submit(void *ctx, const float *mel, int frames, float ** #define MAX_BODY_BYTES (16u * 1024u * 1024u) #define MAX_HEADER_BYTES (64u * 1024u) +#ifndef _WIN32 extern char **environ; +#endif #if defined(EI_ENABLE_ROCM) #define DEFAULT_INFERENCE_BACKEND "rocm" @@ -739,8 +770,24 @@ static bool parse_required_model(const char *body, sv_string *model, return true; } -static void send_header_and_body(int fd, const char *hdr, size_t hdr_len, +static void send_header_and_body(ei_socket fd, const char *hdr, size_t hdr_len, const char *body, size_t body_len) { +#ifdef _WIN32 + const char *parts[] = {hdr, body}; + size_t lengths[] = {hdr_len, body_len}; + for (int i = 0; i < 2; i++) { + while (lengths[i]) { + int chunk = lengths[i] > INT_MAX ? INT_MAX : (int)lengths[i]; + int sent = send(fd, parts[i], chunk, 0); + if (sent <= 0) { + if (sent < 0 && ei_socket_errno() == EINTR) continue; + return; + } + parts[i] += sent; + lengths[i] -= (size_t)sent; + } + } +#else struct iovec iov[2] = { {(void *)(uintptr_t)hdr, hdr_len}, {(void *)(uintptr_t)body, body_len}, @@ -748,8 +795,8 @@ static void send_header_and_body(int fd, const char *hdr, size_t hdr_len, int index = 0; while (index < 2) { ssize_t sent = writev(fd, iov + index, 2 - index); - if (sent < 0) { - if (errno == EINTR) continue; + if (sent <= 0) { + if (ei_socket_errno() == EINTR) continue; return; } size_t remaining = (size_t)sent; @@ -762,9 +809,10 @@ static void send_header_and_body(int fd, const char *hdr, size_t hdr_len, iov[index].iov_len -= remaining; } } +#endif } -static void http_response_raw(int fd, int status, const char *reason, +static void http_response_raw(ei_socket fd, int status, const char *reason, const char *content_type, const char *body, size_t body_len, bool keep_alive) { char hdr[512]; @@ -781,20 +829,20 @@ static void http_response_raw(int fd, int status, const char *reason, send_header_and_body(fd, hdr, (size_t)n, body, body_len); } -static void http_response_typed(int fd, int status, const char *reason, +static void http_response_typed(ei_socket fd, int status, const char *reason, const char *content_type, const char *body, bool keep_alive) { http_response_raw(fd, status, reason, content_type, body, strlen(body), keep_alive); } -static void http_response(int fd, int status, const char *reason, +static void http_response(ei_socket fd, int status, const char *reason, const char *body, bool keep_alive) { http_response_typed(fd, status, reason, "application/json; charset=utf-8", body, keep_alive); } -static void http_error(int fd, int status, const char *reason, const char *msg, +static void http_error(ei_socket fd, int status, const char *reason, const char *msg, bool keep_alive) { sbuf b = {0}; sbuf_append_z(&b, "{\"error\":"); @@ -806,7 +854,7 @@ static void http_error(int fd, int status, const char *reason, const char *msg, free(b.data); } -static void openai_http_error(int fd, int status, const char *reason, +static void openai_http_error(ei_socket fd, int status, const char *reason, const char *msg, const char *param, bool keep_alive) { sbuf b = {0}; @@ -822,7 +870,7 @@ static void openai_http_error(int fd, int status, const char *reason, free(b.data); } -static void embedding_http_error(int fd, embedding_api api, int status, +static void embedding_http_error(ei_socket fd, embedding_api api, int status, const char *reason, const char *msg, const char *param, bool keep_alive) { if (api == EMBEDDING_API_OPENAI) { @@ -903,7 +951,7 @@ static char *find_header_end(char *buf, size_t n) { } typedef struct { - int fd; + ei_socket fd; sbuf buffered; } http_connection; @@ -923,8 +971,8 @@ static http_read_result read_request(http_connection *connection, while (!hdr_end) { ssize_t n = recv(connection->fd, tmp, sizeof tmp, 0); if (n < 0) { - if (errno == EINTR) continue; - if ((errno == EAGAIN || errno == EWOULDBLOCK) && + if (ei_socket_errno() == EINTR) continue; + if ((ei_socket_errno() == EAGAIN || ei_socket_errno() == EWOULDBLOCK) && connection->buffered.n == 0) { return HTTP_READ_CLOSED; } @@ -981,7 +1029,7 @@ static http_read_result read_request(http_connection *connection, while (connection->buffered.n - body_start < body_len) { ssize_t n = recv(connection->fd, tmp, sizeof tmp, 0); if (n < 0) { - if (errno == EINTR) continue; + if (ei_socket_errno() == EINTR) continue; snprintf(err, err_len, "failed to read request body"); free(headers); return HTTP_READ_ERROR; @@ -1010,7 +1058,7 @@ static http_read_result read_request(http_connection *connection, return HTTP_READ_OK; } -static void handle_tags(int fd, const server_opts *opts, bool keep_alive) { +static void handle_tags(ei_socket fd, const server_opts *opts, bool keep_alive) { (void)opts; sbuf b = {0}; sbuf_append_z(&b, "{\"models\":[{\"name\":"); @@ -1025,7 +1073,7 @@ static void handle_tags(int fd, const server_opts *opts, bool keep_alive) { static ei_response_cache *media_cache; #include "http_media.h" -static void handle_embed(int fd, ei_inference_service *service, +static void handle_embed(ei_socket fd, ei_inference_service *service, const server_opts *opts, const char *body, size_t body_len, bool keep_alive, ei_response_cache *response_cache, @@ -1303,6 +1351,10 @@ static char *path_join(const char *a, const char *b) { static char *dirname_copy(const char *path) { const char *slash = strrchr(path, '/'); +#ifdef _WIN32 + const char *backslash = strrchr(path, '\\'); + if (backslash && (!slash || backslash > slash)) slash = backslash; +#endif if (!slash) return dup_z("."); if (slash == path) return dup_z("/"); return dup_range(path, (size_t)(slash - path)); @@ -1318,8 +1370,17 @@ static char *resolve_model_path(const server_opts *opts) { if (cache_home && *cache_home) { root = dup_z(cache_home); } else { +#ifdef _WIN32 + const char *local = getenv("LOCALAPPDATA"); + if (local && *local) root = dup_z(local); + else { + const char *home = getenv("USERPROFILE"); + root = path_join(home && *home ? home : ".", ".cache"); + } +#else const char *home = getenv("HOME"); root = path_join(home && *home ? home : ".", ".cache"); +#endif } char *directory = path_join(root, "embeddinggemma.c"); char *model = path_join(directory, opts->profile->file); @@ -1345,19 +1406,37 @@ static void ensure_dir(const char *path) { return; } if (errno != ENOENT) ei_die("cannot stat %s: %s", path, strerror(errno)); - if (mkdir(path, 0755) != 0 && errno != EEXIST) { +#ifdef _WIN32 + if (_mkdir(path) +#else + if (mkdir(path, 0755) +#endif + != 0 && errno != EEXIST) { ei_die("cannot create directory %s: %s", path, strerror(errno)); } } static void mkdir_p(const char *path) { char *tmp = dup_z(path); +#ifdef _WIN32 + for (char *p = tmp; *p; p++) if (*p == '\\') *p = '/'; +#endif size_t n = strlen(tmp); if (n == 0) { free(tmp); return; } - for (size_t i = 1; i < n; i++) { + size_t start = 1; +#ifdef _WIN32 + start = ei_windows_directory_start(tmp); + if (start == SIZE_MAX) ei_die("invalid UNC directory: %s", path); + if (tmp[0] == '/' && tmp[1] == '/' && start >= n) { + /* Existing share roots are managed by SMB, never created here. */ + free(tmp); + return; + } +#endif + for (size_t i = start; i < n; i++) { if (tmp[i] == '/') { tmp[i] = '\0'; ensure_dir(tmp); @@ -1376,6 +1455,39 @@ typedef enum { static downloader_result run_downloader(const char *program, char *const arguments[]) { +#ifdef _WIN32 + /* The Windows CRT joins argv without quoting. Quote each argument using + * its command-line rules, preserving spaces and trailing backslashes. */ + size_t count = 0; + while (arguments[count]) count++; + char **quoted = ei_xcalloc(count + 1, sizeof(*quoted)); + for (size_t i = 0; i < count; i++) { + const char *arg = arguments[i]; + char *out = quoted[i] = ei_xmalloc(strlen(arg) * 2 + 3); + *out++ = '"'; + size_t slashes = 0; + for (;;) { + char ch = *arg++; + if (ch == '\\') { slashes++; continue; } + size_t emit = (ch == '"' || ch == '\0') ? slashes * 2 : slashes; + while (emit--) *out++ = '\\'; + slashes = 0; + if (ch == '\0') break; + if (ch == '"') *out++ = '\\'; + *out++ = ch; + } + *out++ = '"'; + *out = '\0'; + } + intptr_t status = _spawnvp(_P_WAIT, program, (const char *const *)quoted); + int spawn_errno = errno; + for (size_t i = 0; i < count; i++) free(quoted[i]); + free(quoted); + errno = spawn_errno; + if (status == -1 && errno == ENOENT) return DOWNLOADER_UNAVAILABLE; + if (status == 0) return DOWNLOADER_SUCCEEDED; + fprintf(stderr, "%s failed (status %lld)\n", program, (long long)status); +#else pid_t child; int rc = posix_spawnp(&child, program, NULL, NULL, arguments, environ); if (rc == ENOENT) return DOWNLOADER_UNAVAILABLE; @@ -1402,6 +1514,7 @@ static downloader_result run_downloader(const char *program, fprintf(stderr, "%s terminated by signal %d\n", program, WTERMSIG(status)); } +#endif return DOWNLOADER_FAILED; } @@ -1497,7 +1610,7 @@ static uint64_t model_fingerprint(const char *path) { if (!file) return 0; unsigned char buffer[65536]; if (fseek(file, 0, SEEK_END) == 0) { - long size = ftell(file); + int64_t size = (int64_t)ftell(file); for (int i = 0; i < 8; i++) { hash ^= (uint64_t)((size >> (i * 8)) & 0xff); hash *= 1099511628211ull; @@ -1544,7 +1657,11 @@ static void usage(const char *argv0) { " [--cuda-devices all|I,J,...] (embeddinggemma-2 on several GPUs)\n" "default listen: 0.0.0.0:%d\n" "default model: $XDG_CACHE_HOME/embeddinggemma.c/%s\n" +#ifdef _WIN32 + " or $LOCALAPPDATA/embeddinggemma.c/%s\n", +#else " or $HOME/.cache/embeddinggemma.c/%s\n", +#endif argv0, EMBEDDINGGEMMA_DEFAULT_PORT, ei_profiles[0].file, ei_profiles[0].file); } @@ -1721,7 +1838,7 @@ static bool parse_args(int argc, char **argv, server_opts *opts) { } typedef struct { - int *fds; + ei_socket *fds; size_t capacity; size_t head; size_t count; @@ -1753,7 +1870,7 @@ static void socket_queue_init(socket_queue *queue, size_t capacity) { } } -static bool socket_queue_try_push(socket_queue *queue, int fd) { +static bool socket_queue_try_push(socket_queue *queue, ei_socket fd) { pthread_mutex_lock(&queue->mutex); if (queue->count == queue->capacity) { pthread_mutex_unlock(&queue->mutex); @@ -1767,10 +1884,10 @@ static bool socket_queue_try_push(socket_queue *queue, int fd) { return true; } -static int socket_queue_pop(socket_queue *queue) { +static ei_socket socket_queue_pop(socket_queue *queue) { pthread_mutex_lock(&queue->mutex); while (queue->count == 0) pthread_cond_wait(&queue->ready, &queue->mutex); - int fd = queue->fds[queue->head]; + ei_socket fd = queue->fds[queue->head]; queue->head = (queue->head + 1) % queue->capacity; queue->count--; pthread_mutex_unlock(&queue->mutex); @@ -1801,14 +1918,18 @@ static void keepalive_limiter_release(keepalive_limiter *limiter) { static void *server_worker_main(void *opaque) { server_worker *worker = opaque; for (;;) { - int fd = socket_queue_pop(worker->queue); + ei_socket fd = socket_queue_pop(worker->queue); http_connection connection = { .fd = fd }; +#ifdef _WIN32 + DWORD receive_timeout = worker->opts->keepalive_timeout_ms; +#else struct timeval receive_timeout = { .tv_sec = worker->opts->keepalive_timeout_ms / 1000u, .tv_usec = (suseconds_t)( worker->opts->keepalive_timeout_ms % 1000u) * 1000, }; - setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &receive_timeout, +#endif + ei_setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &receive_timeout, sizeof receive_timeout); bool keepalive_slot = worker->opts->keepalive_max_requests > 1 && keepalive_limiter_try_acquire(worker->keepalive); @@ -1822,7 +1943,7 @@ static void *server_worker_main(void *opaque) { } if (keepalive_slot) keepalive_limiter_release(worker->keepalive); free(connection.buffered.data); - close(fd); + ei_socket_close(fd); } return NULL; } @@ -1850,7 +1971,12 @@ static bool execute_engine_batch_slot(void *opaque, size_t slot, const int32_t * } int main(int argc, char **argv) { +#ifdef _WIN32 + WSADATA wsa; + if (WSAStartup(MAKEWORD(2, 2), &wsa) != 0) ei_die("Winsock initialization failed"); +#else signal(SIGPIPE, SIG_IGN); +#endif server_opts opts; if (!parse_args(argc, argv, &opts)) { @@ -2074,10 +2200,10 @@ int main(int argc, char **argv) { } } - int s = socket(AF_INET, SOCK_STREAM, 0); - if (s < 0) ei_die("socket failed: %s", strerror(errno)); + ei_socket s = socket(AF_INET, SOCK_STREAM, 0); + if (s == EI_INVALID_SOCKET) ei_die("socket failed: %s", strerror(errno)); int yes = 1; - setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof yes); + ei_setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof yes); struct sockaddr_in addr; memset(&addr, 0, sizeof addr); @@ -2105,6 +2231,10 @@ int main(int argc, char **argv) { /* Graceful shutdown so the persistent cache is flushed. sigaction without * SA_RESTART makes accept() return EINTR when a stop signal arrives. */ if (opts.persistent_cache_path) { +#ifdef _WIN32 + signal(SIGTERM, handle_stop_signal); + signal(SIGINT, handle_stop_signal); +#else struct sigaction action; memset(&action, 0, sizeof action); action.sa_handler = handle_stop_signal; @@ -2112,22 +2242,35 @@ int main(int argc, char **argv) { action.sa_flags = 0; sigaction(SIGTERM, &action, NULL); sigaction(SIGINT, &action, NULL); +#endif } while (!g_stop_requested) { - int c = accept(s, NULL, NULL); - if (c >= 0) { +#ifdef _WIN32 + /* CRT signal handlers do not interrupt Winsock accept. Polling keeps + * graceful cache publication reachable after a stop signal. */ + fd_set ready; + FD_ZERO(&ready); + FD_SET(s, &ready); + struct timeval timeout = {0, 250000}; + int selected = select(0, &ready, NULL, NULL, &timeout); + if (g_stop_requested) break; + if (selected == 0) continue; + if (selected == SOCKET_ERROR) ei_die("select failed: %d", WSAGetLastError()); +#endif + ei_socket c = accept(s, NULL, NULL); + if (c != EI_INVALID_SOCKET) { int nodelay = 1; - setsockopt(c, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof nodelay); + ei_setsockopt(c, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof nodelay); } - if (c < 0) { - if (errno == EINTR) continue; + if (c == EI_INVALID_SOCKET) { + if (ei_socket_errno() == EINTR) continue; fprintf(stderr, "accept failed: %s\n", strerror(errno)); continue; } if (!socket_queue_try_push(&connection_queue, c)) { http_error(c, 503, "Service Unavailable", "server request queue is full", false); - close(c); + ei_socket_close(c); } } if (opts.persistent_cache_path) { diff --git a/src/test_cpu_dispatch.c b/src/test_cpu_dispatch.c new file mode 100644 index 0000000..33e49b0 --- /dev/null +++ b/src/test_cpu_dispatch.c @@ -0,0 +1,244 @@ +/* x86-64 runtime dispatch parity: embeds the same inputs through the forced + * SSE2 baseline and the AVX2/FMA/F16C kernels in one process and compares them at + * every Matryoshka dimension. Also times one ~150-token chunk per route. */ +#define _POSIX_C_SOURCE 200809L + +#include "engine.h" +#include "kernels_avx2.h" + +#include +#include +#include + +/* The routes differ only in float summation order and FMA, but a last-bit + * difference can flip a Q8_0 activation rounding and cascade through 24 + * layers, so the gate matches the llama.cpp golden gate rather than + * bit-equality. Real-model results sit near cosine 0.9999. */ +#define PARITY_MIN_COSINE 0.999 +#define PARITY_MAX_ABS_DIFF 1e-2 + +static const int32_t dimensions[] = { EI_N_EMBD, 512, 256, 128 }; +#define N_DIMS ((int)(sizeof dimensions / sizeof dimensions[0])) + +typedef struct { + char **items; + size_t n; +} string_list; + +static char *read_file(const char *path) { + FILE *f = fopen(path, "rb"); + if (!f) ei_die("%s: cannot open", path); + if (fseek(f, 0, SEEK_END) != 0) ei_die("%s: seek failed", path); + long n = ftell(f); + if (n < 0 || fseek(f, 0, SEEK_SET) != 0) ei_die("%s: tell failed", path); + char *data = ei_xmalloc((size_t)n + 1u); + if (fread(data, 1, (size_t)n, f) != (size_t)n) ei_die("%s: read failed", path); + fclose(f); + data[n] = '\0'; + return data; +} + +/* A flat JSON array of strings with simple escapes (testdata/test-strings.json). */ +static string_list parse_strings(const char *path) { + char *data = read_file(path); + string_list list = { 0 }; + const char *p = strchr(data, '['); + if (!p) ei_die("%s: expected a JSON array", path); + for (p++; *p && *p != ']'; p++) { + if (*p != '"') continue; + char *s = ei_xmalloc(strlen(p) + 1u); + size_t len = 0; + for (p++; *p != '"'; p++) { + if (!*p) ei_die("%s: unterminated string", path); + char c = *p; + if (c == '\\') { + c = *++p; + if (c == 'n') c = '\n'; + else if (c == 't') c = '\t'; + else if (c != '"' && c != '\\' && c != '/') ei_die("%s: unsupported escape", path); + } + s[len++] = c; + } + s[len] = '\0'; + list.items = ei_xrealloc(list.items, sizeof(char *) * (list.n + 1u)); + list.items[list.n++] = s; + } + free(data); + return list; +} + +/* An input of exactly n_tokens ids, cycling through the tokenized samples. */ +static ei_tokens repeated_tokens(const ei_tokens *samples, size_t n_samples, size_t n_tokens) { + ei_tokens out = { ei_xmalloc(sizeof(int32_t) * n_tokens), n_tokens }; + size_t len = 0; + for (size_t i = 0; len < n_tokens; i = (i + 1) % n_samples) { + for (size_t j = 0; j < samples[i].n && len < n_tokens; j++) { + out.ids[len++] = samples[i].ids[j]; + } + } + return out; +} + +static void embed_as(ei_engine *e, const char *isa, const ei_tokens *tokens, + float out[EI_N_EMBD]) { + if (!ei_cpu_set_isa(isa)) ei_die("cannot select CPU ISA %s", isa); + char err[256]; + if (!ei_engine_embed_tokens(e, tokens->ids, tokens->n, out, err, sizeof err)) { + ei_die("embedding failed under %s: %s", isa, err); + } +} + +static void embed_batch_as(ei_engine *e, const char *isa, const ei_tokens *tokens, + size_t n, float *out) { + if (!ei_cpu_set_isa(isa)) ei_die("cannot select CPU ISA %s", isa); + size_t total = 0; + for (size_t i = 0; i < n; i++) total += tokens[i].n; + int32_t *ids = ei_xmalloc(sizeof(int32_t) * total); + size_t *offsets = ei_xmalloc(sizeof(size_t) * (n + 1u)); + offsets[0] = 0; + for (size_t i = 0; i < n; i++) { + memcpy(ids + offsets[i], tokens[i].ids, sizeof(int32_t) * tokens[i].n); + offsets[i + 1] = offsets[i] + tokens[i].n; + } + char err[256]; + if (!ei_engine_embed_tokens_batch(e, ids, offsets, n, out, err, sizeof err)) { + ei_die("batch embedding failed under %s: %s", isa, err); + } + free(ids); + free(offsets); +} + +static double max_abs[N_DIMS]; +static double min_cos[N_DIMS]; +static int failures; + +static void compare(const char *label, const float *baseline, const float *avx2) { + for (int d = 0; d < N_DIMS; d++) { + float a[EI_N_EMBD], b[EI_N_EMBD]; + memcpy(a, baseline, sizeof a); + memcpy(b, avx2, sizeof b); + if (!ei_embedding_normalize_prefix(a, dimensions[d]) || + !ei_embedding_normalize_prefix(b, dimensions[d])) { + ei_die("bad dimension %d", dimensions[d]); + } + double dot = 0.0, na = 0.0, nb = 0.0, diff = 0.0; + for (int32_t i = 0; i < dimensions[d]; i++) { + dot += (double)a[i] * b[i]; + na += (double)a[i] * a[i]; + nb += (double)b[i] * b[i]; + double delta = fabs((double)a[i] - b[i]); + if (!(delta <= diff)) diff = delta; /* also catches NaN */ + } + double cosine = dot / sqrt(na * nb); + if (!(cosine >= min_cos[d])) min_cos[d] = cosine; + if (!(diff <= max_abs[d])) max_abs[d] = diff; + if (!(cosine >= PARITY_MIN_COSINE) || !(diff <= PARITY_MAX_ABS_DIFF)) { + printf("FAIL %s D=%d: SSE2 vs AVX2 cosine %.9f, max abs diff %.3g\n", + label, dimensions[d], cosine, diff); + failures++; + } + } +} + +static double now_seconds(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + ei_die("clock_gettime failed: %s", strerror(errno)); + } + return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9; +} + +static double time_chunk(ei_engine *e, const char *isa, const ei_tokens *tokens, int reps) { + float out[EI_N_EMBD]; + embed_as(e, isa, tokens, out); /* warm-up */ + double best = INFINITY; + for (int r = 0; r < reps; r++) { + double start = now_seconds(); + embed_as(e, isa, tokens, out); + double elapsed = now_seconds() - start; + if (elapsed < best) best = elapsed; + } + return best; +} + +int main(int argc, char **argv) { + if (argc != 3) ei_die("usage: %s ", argv[0]); + if (!ei_cpu_avx2_supported()) { + printf("cpu dispatch parity: skipped (no x86-64 AVX2+FMA+F16C route on this host)\n"); + return 0; + } + + ei_engine engine; + ei_engine_load_backend(&engine, argv[1], "cpu"); + string_list texts = parse_strings(argv[2]); + /* The samples, then 150 tokens (one typical chunk) and 600 tokens (past + * the 512-token sliding window). */ + const size_t n_samples = texts.n, n_inputs = texts.n + 2u; + ei_tokens *tokens = ei_xmalloc(sizeof(ei_tokens) * n_inputs); + for (size_t i = 0; i < n_samples; i++) { + ei_tokenize_spm(&engine.tokenizer, texts.items[i], strlen(texts.items[i]), + true, false, &tokens[i]); + } + tokens[n_samples] = repeated_tokens(tokens, n_samples, 150u); + tokens[n_samples + 1u] = repeated_tokens(tokens, n_samples, 600u); + + for (int d = 0; d < N_DIMS; d++) { + max_abs[d] = 0.0; + min_cos[d] = 1.0; + } + + for (size_t i = 0; i < n_inputs; i++) { + float baseline[EI_N_EMBD], avx2[EI_N_EMBD]; + embed_as(&engine, "baseline", &tokens[i], baseline); + embed_as(&engine, "avx2", &tokens[i], avx2); + char label[64]; + snprintf(label, sizeof label, "input %zu (%zu tokens)", i, tokens[i].n); + compare(label, baseline, avx2); + } + + /* The batched route uses the multi-input Q4_0 kernel. */ + float *batch_baseline = ei_xmalloc(sizeof(float) * EI_N_EMBD * n_samples); + float *batch_avx2 = ei_xmalloc(sizeof(float) * EI_N_EMBD * n_samples); + embed_batch_as(&engine, "baseline", tokens, n_samples, batch_baseline); + embed_batch_as(&engine, "avx2", tokens, n_samples, batch_avx2); + for (size_t i = 0; i < n_samples; i++) { + char label[64]; + snprintf(label, sizeof label, "batch item %zu", i); + compare(label, batch_baseline + i * EI_N_EMBD, batch_avx2 + i * EI_N_EMBD); + } + + for (int d = 0; d < N_DIMS; d++) { + printf("cpu dispatch parity D=%d: min cosine %.9f, max abs diff %.3g\n", + dimensions[d], min_cos[d], max_abs[d]); + } + if (failures) { + ei_die("cpu dispatch parity: %d comparisons outside cosine >= %.3f, " + "max abs <= %.0e", failures, PARITY_MIN_COSINE, PARITY_MAX_ABS_DIFF); + } + printf("cpu dispatch parity: %zu inputs + %zu batched, SSE2 vs AVX2 within " + "cosine >= %.3f and max abs <= %.0e\n", + n_inputs, n_samples, PARITY_MIN_COSINE, PARITY_MAX_ABS_DIFF); + + const ei_tokens *chunk = &tokens[n_samples]; + size_t chunk_tokens = chunk->n; + double sse2 = time_chunk(&engine, "baseline", chunk, 3); + double avx2 = time_chunk(&engine, "avx2", chunk, 3); + printf("cpu dispatch timing: %zu-token chunk, %d threads: SSE2 %.1f ms " + "(%.0f tok/s), AVX2 %.1f ms (%.0f tok/s), speedup %.2fx\n", + chunk_tokens, ei_engine_threads(&engine), sse2 * 1e3, + (double)chunk_tokens / sse2, avx2 * 1e3, (double)chunk_tokens / avx2, + sse2 / avx2); + + for (size_t i = 0; i < n_samples; i++) { + ei_tokens_free(&tokens[i]); + free(texts.items[i]); + } + free(tokens[n_samples].ids); + free(tokens[n_samples + 1u].ids); + free(tokens); + free(texts.items); + free(batch_baseline); + free(batch_avx2); + ei_engine_free(&engine); + return 0; +} diff --git a/src/test_downloader_fixture.c b/src/test_downloader_fixture.c new file mode 100644 index 0000000..1aec7e3 --- /dev/null +++ b/src/test_downloader_fixture.c @@ -0,0 +1,16 @@ +/* Fictional downloader used only by the isolated process-spawning test. */ +#include +#include +#include +int main(int argc, char **argv) { + if (getenv("EI_TEST_CURL_FAIL") && strstr(argv[0], "curl")) return 7; + for (int i = 1; i + 1 < argc; i++) { + if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "-O") == 0) { + FILE *out = fopen(argv[i + 1], "wb"); + if (!out) return 2; + if (fwrite("fictional", 1, 9, out) != 9) return 3; + return fclose(out) == 0 ? 0 : 4; + } + } + return 5; +} diff --git a/src/test_gemma2.c b/src/test_gemma2.c index 7ec4ff5..9530822 100644 --- a/src/test_gemma2.c +++ b/src/test_gemma2.c @@ -7,6 +7,24 @@ #if defined(EI_ENABLE_CUDA) #include "gemma2_cuda.h" #endif +#ifdef _WIN32 +#include "windows_compat.h" +#include +// The CRT argv is in the ANSI code page; the goldens are UTF-8 text. +static char **utf8_argv(void) { + int n = 0; + wchar_t **wide = CommandLineToArgvW(GetCommandLineW(), &n); + if (!wide) ei_die("CommandLineToArgvW failed"); + char **out = ei_xcalloc((size_t)n + 1, sizeof *out); + for (int i = 0; i < n; i++) { + int bytes = WideCharToMultiByte(CP_UTF8, 0, wide[i], -1, NULL, 0, NULL, NULL); + out[i] = ei_xmalloc((size_t)bytes); + WideCharToMultiByte(CP_UTF8, 0, wide[i], -1, out[i], bytes, NULL, NULL); + } + LocalFree(wide); + return out; +} +#endif static void vector(const float *x, size_t n) { putchar('['); @@ -15,6 +33,9 @@ static void vector(const float *x, size_t n) { putchar(']'); } int main(int argc, char **argv) { +#ifdef _WIN32 + argv = utf8_argv(); +#endif if (argc < 4) ei_die("usage: %s text.gguf projector.gguf text|vision|audio [text]", argv[0]); ei_engine e; diff --git a/src/test_inference_service.c b/src/test_inference_service.c index 70e0f14..9ad81ac 100644 --- a/src/test_inference_service.c +++ b/src/test_inference_service.c @@ -3,6 +3,13 @@ #include "inference_service.h" #include +#ifdef _WIN32 +static int setenv(const char *name, const char *value, int overwrite) { + if (!overwrite && getenv(name)) return 0; + return _putenv_s(name, value); +} +static int unsetenv(const char *name) { return _putenv_s(name, ""); } +#endif #include #include diff --git a/src/test_kernels.c b/src/test_kernels.c index fca1d70..b1dd258 100644 --- a/src/test_kernels.c +++ b/src/test_kernels.c @@ -1,4 +1,5 @@ #include "kernels.h" +#include "kernels_avx2.h" #include "quants.h" #include @@ -254,12 +255,59 @@ static void test_matryoshka_dimensions(void) { } } +/* Runs the quantized kernels under the forced SSE2 baseline and under AVX2 on + * the same inputs. Integer dot products are exact, so only the float + * accumulation order (and FMA) separates the two routes. */ +static void test_isa_parity(void) { + enum { K = EI_N_FF, ROWS = 64, INPUTS = 6, BLOCKS = K / EI_QK }; + static ei_block_q4_0 weights[ROWS * BLOCKS]; + static float x[INPUTS * K]; + static ei_block_q8_0 xq[2][INPUTS * BLOCKS]; + static float rows[2][ROWS], dual[2][2 * ROWS], batch[2][INPUTS * ROWS]; + static float dequant[2][K]; + for (int i = 0; i < ROWS * BLOCKS; i++) { + weights[i].d = ei_fp32_to_fp16(0.002f + 0.0001f * (float)(i % 97)); + for (int j = 0; j < 16; j++) weights[i].qs[j] = (uint8_t)(next_f32() * 127.0f + 128.0f); + } + fill(x, INPUTS * K); + ei_tensor w = { .type = EI_T_Q4_0, .ne = { K, ROWS, 1, 1 }, .data = weights }; + ei_tensor xt = { .type = EI_T_Q8_0, .ne = { K, INPUTS, 1, 1 } }; + const char *isas[2] = { "baseline", "avx2" }; + for (int r = 0; r < 2; r++) { + if (!ei_cpu_set_isa(isas[r])) ei_die("cannot select %s", isas[r]); + for (int t = 0; t < INPUTS; t++) { + ei_quantize_row_q8_0(x + t * K, xq[r] + t * BLOCKS, K); + } + ei_matmul_q4_0_q8_0_rows3(&w, xq[0], rows[r], 0, ROWS); + ei_matmul_q4_0_q8_0_dual_rows(&w, &w, xq[0], dual[r], dual[r] + ROWS, 0, ROWS); + ei_matmul_q4_0_q8_0_batch_rows(&w, xq[0], INPUTS, batch[r], 0, ROWS); + xt.data = xq[0]; + ei_dequantize_row_q8_0_scaled(&xt, 1, 0.5f, dequant[r]); + } + if (!ei_cpu_set_isa("auto")) ei_die("cannot restore auto ISA"); + if (memcmp(xq[0], xq[1], sizeof xq[0]) != 0) ei_die("q8_0 quantization differs by ISA"); + printf("q8 quantize ISA parity: identical\n"); + require_close("q4 rows ISA parity", rows[1], rows[0], ROWS, 2e-5f); + require_close("q4 dual ISA parity", dual[1], dual[0], 2 * ROWS, 2e-5f); + require_close("q4 batch ISA parity", batch[1], batch[0], INPUTS * ROWS, 2e-5f); + require_close("q8 dequant ISA parity", dequant[1], dequant[0], K, 0.0f); +} + int main(void) { - test_q4_triple(); - test_attention(); - test_norm_residual(); - test_qk_norm_rope(); - test_mean_pool(); - test_matryoshka_dimensions(); + const char *isas[] = { "baseline", "avx2" }; + for (int i = 0; i < 2; i++) { + if (!ei_cpu_set_isa(isas[i])) { + printf("CPU kernels: %s route unavailable on this host\n", isas[i]); + continue; + } + printf("CPU kernels: %s / %s\n", ei_cpu_kernel_variant(), ei_quants_kernel_variant()); + test_q4_triple(); + test_attention(); + test_norm_residual(); + test_qk_norm_rope(); + test_mean_pool(); + test_matryoshka_dimensions(); + } + if (ei_cpu_avx2_supported()) test_isa_parity(); return 0; } diff --git a/src/test_windows_paths.c b/src/test_windows_paths.c new file mode 100644 index 0000000..7d9b46a --- /dev/null +++ b/src/test_windows_paths.c @@ -0,0 +1,30 @@ +/* Synthetic UNC roots need no SMB service or access to a real network share. */ +#include "common.h" +#include "windows_compat.h" +#include +int main(void) { + assert(ei_windows_directory_start("//fictional-server/share/cache") == + strlen("//fictional-server/share") + 1); + assert(ei_windows_directory_start("//fictional-server/share") == + strlen("//fictional-server/share") + 1); + assert(ei_windows_directory_start("//fictional-server/") == SIZE_MAX); + assert(ei_windows_directory_start("//fictional-server//cache") == SIZE_MAX); + assert(ei_windows_directory_start("C:/fictional directory/cache") == 3); + assert(ei_windows_directory_start("fictional/cache") == 1); + char first[] = "build/windows-file-a.XXXXXX"; + char second[] = "build/windows-file-b.XXXXXX"; + int a = mkstemp(first), b = mkstemp(second); + assert(a >= 0 && b >= 0 && strcmp(first, second) != 0); + const char bytes[] = {'\r', '\n', 26, 0}; + assert(_write(a, bytes, sizeof bytes) == (int)sizeof bytes); + assert(_close(a) == 0 && _close(b) == 0); + assert(ei_replace_file(first, second) == 0); + FILE *file = fopen(second, "rb"); + assert(file); + char actual[sizeof bytes]; + assert(fread(actual, 1, sizeof actual, file) == sizeof actual); + assert(memcmp(actual, bytes, sizeof bytes) == 0); + assert(fclose(file) == 0 && _unlink(second) == 0); + puts("Windows UNC roots, binary temporary files and cache replacement: passed"); + return 0; +} diff --git a/src/windows_compat.h b/src/windows_compat.h new file mode 100644 index 0000000..8f3b977 --- /dev/null +++ b/src/windows_compat.h @@ -0,0 +1,69 @@ +/* Native Windows file helpers; MinGW supplies C11 and winpthreads. */ +#ifndef EI_WINDOWS_COMPAT_H +#define EI_WINDOWS_COMPAT_H +#ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* A network server is not a directory: preserve the complete UNC share root + * before mkdir_p visits any separator. Input separators are already '/'. */ +static inline size_t ei_windows_directory_start(const char *path) { + if (path[0] == '/' && path[1] == '/') { + const char *server_end = strchr(path + 2, '/'); + if (!server_end || server_end == path + 2 || !server_end[1]) return SIZE_MAX; + const char *share_end = strchr(server_end + 1, '/'); + if (share_end == server_end + 1) return SIZE_MAX; + return share_end ? (size_t)(share_end - path) + 1 : strlen(path) + 1; + } + return strlen(path) >= 3 && path[1] == ':' && path[2] == '/' ? 3 : 1; +} + +/* Exclusive creation preserves mkstemp's security and binary file semantics. */ +static inline int ei_mkstemp(char *pattern) { + size_t n = strlen(pattern); + if (n < 6 || strcmp(pattern + n - 6, "XXXXXX") != 0) { + errno = EINVAL; + return -1; + } + for (unsigned attempt = 0; attempt < 256; attempt++) { + unsigned int random; + if (rand_s(&random) != 0) { errno = EIO; return -1; } + static const char alphabet[] = "0123456789abcdefghijklmnopqrstuvwxyz"; + for (size_t i = n - 6; i < n; i++) { + pattern[i] = alphabet[random % 36]; + random /= 36; + } + int fd = _open(pattern, _O_CREAT | _O_EXCL | _O_RDWR | _O_BINARY, + _S_IREAD | _S_IWRITE); + if (fd >= 0 || errno != EEXIST) return fd; + } + errno = EEXIST; + return -1; +} +#define mkstemp ei_mkstemp +/* The server maps --threads and --cuda-devices onto environment variables. */ +static inline int ei_setenv(const char *name, const char *value, int overwrite) { + if (!overwrite && getenv(name)) return 0; + return _putenv_s(name, value) == 0 ? 0 : -1; +} +#define setenv ei_setenv +/* Windows rename does not replace a destination; cache publication must. */ +static inline int ei_replace_file(const char *from, const char *to) { + if (MoveFileExA(from, to, MOVEFILE_REPLACE_EXISTING)) return 0; + errno = EIO; + return -1; +} +#else +#define ei_replace_file rename +#endif +#endif diff --git a/testdata/test_windows_download.py b/testdata/test_windows_download.py new file mode 100644 index 0000000..a15a581 --- /dev/null +++ b/testdata/test_windows_download.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Test native Windows download spawning without network or household data.""" +import argparse +import os +from pathlib import Path +import shutil +import subprocess +import tempfile + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--binary", required=True) + parser.add_argument("--downloader", required=True) + args = parser.parse_args() + binary = str(Path(args.binary).resolve()) + downloader = Path(args.downloader).resolve() + with tempfile.TemporaryDirectory(prefix="embeddinggemma fictional ") as temp: + root = Path(temp) + for mode in ("curl", "fallback", "wget", "missing", "profile", "explicit"): + directory = root / mode + directory.mkdir() + tools = directory / "tools with spaces" + tools.mkdir() + if mode not in ("wget", "missing"): + shutil.copyfile(downloader, tools / "curl.exe") + if mode in ("fallback", "wget"): + shutil.copyfile(downloader, tools / "wget.exe") + env = os.environ.copy() + for key in ("XDG_CACHE_HOME", "EI_MODEL_PATH", "EI_TEST_CURL_FAIL", "HOME"): + env.pop(key, None) + env.update(PATH=str(tools), LOCALAPPDATA=str(directory), USERPROFILE=str(directory)) + command = [binary, "--profile", "embeddinggemma-300m"] + expected = directory / "embeddinggemma.c" / "embeddinggemma-300M-qat-Q4_0.gguf" + if mode == "fallback": + env["EI_TEST_CURL_FAIL"] = "1" + if mode == "profile": + env.pop("LOCALAPPDATA") + expected = directory / ".cache" / "embeddinggemma.c" / expected.name + if mode == "explicit": + expected = directory / "explicit path" / "model.gguf" + command += ["--model", str(expected)] + result = subprocess.run(command, env=env, capture_output=True, timeout=20) + assert result.returncode != 0, result.stderr + if mode == "missing": + assert b"neither curl nor wget is available" in result.stderr, result.stderr + assert not expected.exists() + else: + assert expected.read_bytes() == b"fictional", result.stderr + assert b"not a GGUF file" in result.stderr, result.stderr + assert not list(directory.rglob("*.download.*")) + print("Windows download paths, quoting, fallback and cleanup: passed") + + +if __name__ == "__main__": + main()