From 5f6c6e8cf453923495a9de0db634ed848e0c1012 Mon Sep 17 00:00:00 2001 From: Jetha Chan Date: Sun, 4 Oct 2026 12:02:44 +0900 Subject: [PATCH 1/9] Build and test native Windows CPU binaries alongside Linux and macOS --- .github/workflows/ci.yml | 66 +++++++++++++-- Makefile | 4 + src/gguf.c | 25 ++++++ src/inference_service.c | 4 +- src/parallel.c | 7 ++ src/server.c | 169 ++++++++++++++++++++++++++++++--------- src/windows_compat.h | 49 ++++++++++++ 7 files changed, 276 insertions(+), 48 deletions(-) create mode 100644 src/windows_compat.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59b5e2e..515f07a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,21 +4,73 @@ on: push: branches: [main] pull_request: + workflow_dispatch: permissions: contents: read jobs: portable: - name: Portable checks (${{ matrix.os }}) + name: CPU binary (${{ matrix.platform }}) runs-on: ${{ matrix.os }} + timeout-minutes: 30 strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-14] + include: + - os: ubuntu-24.04 + platform: linux-x86_64 + - os: ubuntu-24.04-arm + platform: linux-arm64 + - os: macos-14 + platform: darwin-arm64 + - os: windows-2022 + platform: windows-x86_64 + defaults: + run: + shell: bash steps: - - uses: actions/checkout@v4 - - name: Run model-free checks - run: make check - - name: Build CPU server - run: make + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - name: Set up native Windows compiler + if: runner.os == 'Windows' + uses: msys2/setup-msys2@ec48f7c5447b3140e2b088413ae3a55687bccb6e # v2 + with: + msystem: UCRT64 + update: true + install: make mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-python + - name: Run model-free checks and build + if: runner.os != 'Windows' + run: make check && make + - name: Run native Windows checks and build + if: runner.os == 'Windows' + shell: msys2 {0} + run: make check && make + - name: Download public upstream test model + run: | + mkdir -p model + curl -fL --retry 3 -o model/embeddinggemma-300M-qat-Q4_0.gguf 'https://huggingface.co/ggml-org/embeddinggemma-300M-qat-q4_0-GGUF/resolve/main/embeddinggemma-300M-qat-Q4_0.gguf' + - name: Run embedding and HTTP conformance checks + if: runner.os != 'Windows' + run: make test + - name: Run native Windows embedding and HTTP checks + if: runner.os == 'Windows' + shell: msys2 {0} + run: make test + - name: Stage executable and checksum + env: + PLATFORM: ${{ matrix.platform }} + run: | + mkdir -p dist + suffix='' + if [ "$RUNNER_OS" = Windows ]; then suffix='.exe'; fi + name="embeddinggemma-${PLATFORM}-cpu${suffix}" + cp "build/embeddinggemma${suffix}" "dist/$name" + python -c 'import hashlib,pathlib; p=next(pathlib.Path("dist").glob("embeddinggemma-*")); p.with_name(p.name+".sha256").write_text(hashlib.sha256(p.read_bytes()).hexdigest()+" "+p.name+"\n")' + cp LICENSE dist/LICENSE + - name: Upload verified executable + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: embeddinggemma-${{ matrix.platform }}-cpu + path: dist/* + if-no-files-found: error + retention-days: 30 diff --git a/Makefile b/Makefile index 470f246..04f414e 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,10 @@ CC ?= cc CXX ?= c++ CFLAGS ?= -std=c11 -O2 -Wall -Wextra -Werror -g LDLIBS ?= -lm -pthread +ifneq ($(findstring MINGW,$(shell uname -s)),) +CFLAGS += -D_WIN32_WINNT=0x0601 -D_CRT_RAND_S -D__USE_MINGW_ANSI_STDIO=1 +LDLIBS += -lws2_32 -static +endif NVCC ?= nvcc CUDA_HOME ?= /usr/local/cuda NVCCFLAGS ?= -std=c++17 -O3 --use_fast_math -lineinfo diff --git a/src/gguf.c b/src/gguf.c index 69c2314..2940fcf 100644 --- a/src/gguf.c +++ b/src/gguf.c @@ -1,7 +1,11 @@ #include "gguf.h" #include +#ifdef _WIN32 +#include "windows_compat.h" +#else #include +#endif #include #include @@ -119,6 +123,21 @@ static uint64_t tensor_bytes(const ei_tensor *t, const char *namebuf) { void ei_gguf_open(ei_gguf *g, const char *path) { memset(g, 0, sizeof *g); +#ifdef _WIN32 + HANDLE file = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file == INVALID_HANDLE_VALUE) ei_die("%s: cannot open", path); + LARGE_INTEGER size; + if (!GetFileSizeEx(file, &size) || size.QuadPart <= 0 || + (uint64_t)size.QuadPart > SIZE_MAX) ei_die("%s: cannot stat", path); + g->map_len = (size_t)size.QuadPart; + HANDLE mapping = CreateFileMappingA(file, NULL, PAGE_READONLY, 0, 0, NULL); + CloseHandle(file); + if (!mapping) ei_die("%s: cannot create mapping", path); + g->map = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0); + CloseHandle(mapping); + if (!g->map) ei_die("%s: cannot map file", path); +#else int fd = open(path, O_RDONLY); if (fd < 0) ei_die("%s: cannot open", path); struct stat st; @@ -128,6 +147,8 @@ void ei_gguf_open(ei_gguf *g, const char *path) { close(fd); if (g->map == MAP_FAILED) ei_die("%s: mmap failed", path); +#endif + cur c = { g->map, g->map + g->map_len, path }; if (take_u32(&c) != GGUF_MAGIC) ei_die("%s: not a GGUF file", path); @@ -203,7 +224,11 @@ void ei_gguf_close(ei_gguf *g) { } free(g->kv); free(g->tensors); +#ifdef _WIN32 + if (g->map) UnmapViewOfFile(g->map); +#else if (g->map) munmap((void *)g->map, g->map_len); +#endif memset(g, 0, sizeof *g); } diff --git a/src/inference_service.c b/src/inference_service.c index 90a0c3f..a173170 100644 --- a/src/inference_service.c +++ b/src/inference_service.c @@ -9,6 +9,8 @@ #include #include #include +#include "windows_compat.h" +#include "windows_compat.h" typedef enum { EI_ENTRY_PENDING, @@ -847,7 +849,7 @@ static void write_persistent_cache_locked(ei_inference_service *service) { fwrite(&count, sizeof count, 1, file) == 1; } if (fclose(file) != 0) ok = false; - if (ok && rename(tmp, path) == 0) { + if (ok && ei_replace_file(tmp, path) == 0) { fprintf(stderr, "persisted %llu cached embeddings to %s\n", (unsigned long long)count, path); } else { diff --git a/src/parallel.c b/src/parallel.c index 50d0c56..7fc2f21 100644 --- a/src/parallel.c +++ b/src/parallel.c @@ -11,6 +11,7 @@ #include #include #include +#include "windows_compat.h" typedef struct { struct ei_thread_pool *pool; @@ -139,7 +140,13 @@ static void *worker_main(void *opaque) { } int32_t ei_cpu_count(void) { +#ifdef _WIN32 + SYSTEM_INFO info; + GetSystemInfo(&info); + long available = (long)info.dwNumberOfProcessors; +#else long available = sysconf(_SC_NPROCESSORS_ONLN); +#endif return available < 1 ? 1 : available > 4096 ? 4096 : (int32_t)available; } diff --git a/src/server.c b/src/server.c index 32a6cd8..0580c02 100644 --- a/src/server.c +++ b/src/server.c @@ -17,23 +17,52 @@ #include "gemma2_cuda.h" #endif -#include #include #include #include #include +#include +#include +#include +#include +#include +#include "windows_compat.h" +#ifdef _WIN32 +#include +#include +#include +typedef SOCKET ei_socket; +#define EI_INVALID_SOCKET INVALID_SOCKET +#define ei_socket_close closesocket +#else +#include #include #include #include -#include -#include -#include #include #include -#include #include #include -#include +typedef int ei_socket; +#define EI_INVALID_SOCKET (-1) +#define ei_socket_close close +#endif + +static int ei_socket_errno(void) { +#ifdef _WIN32 + int error = WSAGetLastError(); + if (error == WSAEINTR) return EINTR; + if (error == WSAETIMEDOUT || error == WSAEWOULDBLOCK) return EAGAIN; + return error; +#else + return errno; +#endif +} + +static int ei_setsockopt(ei_socket fd, int level, int name, + const void *value, int length) { + return setsockopt(fd, level, name, (const char *)value, length); +} static pthread_mutex_t inference_mutex = PTHREAD_MUTEX_INITIALIZER; // GPU media contexts (vision/audio towers and the mixed-input text forward): @@ -739,8 +768,24 @@ static bool parse_required_model(const char *body, sv_string *model, return true; } -static void send_header_and_body(int fd, const char *hdr, size_t hdr_len, +static void send_header_and_body(ei_socket fd, const char *hdr, size_t hdr_len, const char *body, size_t body_len) { +#ifdef _WIN32 + const char *parts[] = {hdr, body}; + size_t lengths[] = {hdr_len, body_len}; + for (int i = 0; i < 2; i++) { + while (lengths[i]) { + int chunk = lengths[i] > INT_MAX ? INT_MAX : (int)lengths[i]; + int sent = send(fd, parts[i], chunk, 0); + if (sent <= 0) { + if (sent < 0 && ei_socket_errno() == EINTR) continue; + return; + } + parts[i] += sent; + lengths[i] -= (size_t)sent; + } + } +#else struct iovec iov[2] = { {(void *)(uintptr_t)hdr, hdr_len}, {(void *)(uintptr_t)body, body_len}, @@ -748,8 +793,8 @@ static void send_header_and_body(int fd, const char *hdr, size_t hdr_len, int index = 0; while (index < 2) { ssize_t sent = writev(fd, iov + index, 2 - index); - if (sent < 0) { - if (errno == EINTR) continue; + if (sent <= 0) { + if (ei_socket_errno() == EINTR) continue; return; } size_t remaining = (size_t)sent; @@ -762,9 +807,10 @@ static void send_header_and_body(int fd, const char *hdr, size_t hdr_len, iov[index].iov_len -= remaining; } } +#endif } -static void http_response_raw(int fd, int status, const char *reason, +static void http_response_raw(ei_socket fd, int status, const char *reason, const char *content_type, const char *body, size_t body_len, bool keep_alive) { char hdr[512]; @@ -781,20 +827,20 @@ static void http_response_raw(int fd, int status, const char *reason, send_header_and_body(fd, hdr, (size_t)n, body, body_len); } -static void http_response_typed(int fd, int status, const char *reason, +static void http_response_typed(ei_socket fd, int status, const char *reason, const char *content_type, const char *body, bool keep_alive) { http_response_raw(fd, status, reason, content_type, body, strlen(body), keep_alive); } -static void http_response(int fd, int status, const char *reason, +static void http_response(ei_socket fd, int status, const char *reason, const char *body, bool keep_alive) { http_response_typed(fd, status, reason, "application/json; charset=utf-8", body, keep_alive); } -static void http_error(int fd, int status, const char *reason, const char *msg, +static void http_error(ei_socket fd, int status, const char *reason, const char *msg, bool keep_alive) { sbuf b = {0}; sbuf_append_z(&b, "{\"error\":"); @@ -806,7 +852,7 @@ static void http_error(int fd, int status, const char *reason, const char *msg, free(b.data); } -static void openai_http_error(int fd, int status, const char *reason, +static void openai_http_error(ei_socket fd, int status, const char *reason, const char *msg, const char *param, bool keep_alive) { sbuf b = {0}; @@ -822,7 +868,7 @@ static void openai_http_error(int fd, int status, const char *reason, free(b.data); } -static void embedding_http_error(int fd, embedding_api api, int status, +static void embedding_http_error(ei_socket fd, embedding_api api, int status, const char *reason, const char *msg, const char *param, bool keep_alive) { if (api == EMBEDDING_API_OPENAI) { @@ -903,7 +949,7 @@ static char *find_header_end(char *buf, size_t n) { } typedef struct { - int fd; + ei_socket fd; sbuf buffered; } http_connection; @@ -923,8 +969,8 @@ static http_read_result read_request(http_connection *connection, while (!hdr_end) { ssize_t n = recv(connection->fd, tmp, sizeof tmp, 0); if (n < 0) { - if (errno == EINTR) continue; - if ((errno == EAGAIN || errno == EWOULDBLOCK) && + if (ei_socket_errno() == EINTR) continue; + if ((ei_socket_errno() == EAGAIN || ei_socket_errno() == EWOULDBLOCK) && connection->buffered.n == 0) { return HTTP_READ_CLOSED; } @@ -981,7 +1027,7 @@ static http_read_result read_request(http_connection *connection, while (connection->buffered.n - body_start < body_len) { ssize_t n = recv(connection->fd, tmp, sizeof tmp, 0); if (n < 0) { - if (errno == EINTR) continue; + if (ei_socket_errno() == EINTR) continue; snprintf(err, err_len, "failed to read request body"); free(headers); return HTTP_READ_ERROR; @@ -1010,7 +1056,7 @@ static http_read_result read_request(http_connection *connection, return HTTP_READ_OK; } -static void handle_tags(int fd, const server_opts *opts, bool keep_alive) { +static void handle_tags(ei_socket fd, const server_opts *opts, bool keep_alive) { (void)opts; sbuf b = {0}; sbuf_append_z(&b, "{\"models\":[{\"name\":"); @@ -1025,7 +1071,7 @@ static void handle_tags(int fd, const server_opts *opts, bool keep_alive) { static ei_response_cache *media_cache; #include "http_media.h" -static void handle_embed(int fd, ei_inference_service *service, +static void handle_embed(ei_socket fd, ei_inference_service *service, const server_opts *opts, const char *body, size_t body_len, bool keep_alive, ei_response_cache *response_cache, @@ -1303,6 +1349,10 @@ static char *path_join(const char *a, const char *b) { static char *dirname_copy(const char *path) { const char *slash = strrchr(path, '/'); +#ifdef _WIN32 + const char *backslash = strrchr(path, '\\'); + if (backslash && (!slash || backslash > slash)) slash = backslash; +#endif if (!slash) return dup_z("."); if (slash == path) return dup_z("/"); return dup_range(path, (size_t)(slash - path)); @@ -1318,8 +1368,17 @@ static char *resolve_model_path(const server_opts *opts) { if (cache_home && *cache_home) { root = dup_z(cache_home); } else { +#ifdef _WIN32 + const char *local = getenv("LOCALAPPDATA"); + if (local && *local) root = dup_z(local); + else { + const char *home = getenv("USERPROFILE"); + root = path_join(home && *home ? home : ".", ".cache"); + } +#else const char *home = getenv("HOME"); root = path_join(home && *home ? home : ".", ".cache"); +#endif } char *directory = path_join(root, "embeddinggemma.c"); char *model = path_join(directory, opts->profile->file); @@ -1345,20 +1404,28 @@ static void ensure_dir(const char *path) { return; } if (errno != ENOENT) ei_die("cannot stat %s: %s", path, strerror(errno)); - if (mkdir(path, 0755) != 0 && errno != EEXIST) { +#ifdef _WIN32 + if (_mkdir(path) +#else + if (mkdir(path, 0755) +#endif + != 0 && errno != EEXIST) { ei_die("cannot create directory %s: %s", path, strerror(errno)); } } static void mkdir_p(const char *path) { char *tmp = dup_z(path); +#ifdef _WIN32 + for (char *p = tmp; *p; p++) if (*p == '\\') *p = '/'; +#endif size_t n = strlen(tmp); if (n == 0) { free(tmp); return; } for (size_t i = 1; i < n; i++) { - if (tmp[i] == '/') { + if (tmp[i] == '/' && !(i == 2 && tmp[1] == ':')) { tmp[i] = '\0'; ensure_dir(tmp); tmp[i] = '/'; @@ -1376,6 +1443,13 @@ typedef enum { static downloader_result run_downloader(const char *program, char *const arguments[]) { +#ifdef _WIN32 + /* _spawnvp handles argument quoting; never invoke a command shell. */ + intptr_t status = _spawnvp(_P_WAIT, program, (const char *const *)arguments); + if (status == -1 && errno == ENOENT) return DOWNLOADER_UNAVAILABLE; + if (status == 0) return DOWNLOADER_SUCCEEDED; + fprintf(stderr, "%s failed (status %lld)\n", program, (long long)status); +#else pid_t child; int rc = posix_spawnp(&child, program, NULL, NULL, arguments, environ); if (rc == ENOENT) return DOWNLOADER_UNAVAILABLE; @@ -1402,6 +1476,7 @@ static downloader_result run_downloader(const char *program, fprintf(stderr, "%s terminated by signal %d\n", program, WTERMSIG(status)); } +#endif return DOWNLOADER_FAILED; } @@ -1497,7 +1572,7 @@ static uint64_t model_fingerprint(const char *path) { if (!file) return 0; unsigned char buffer[65536]; if (fseek(file, 0, SEEK_END) == 0) { - long size = ftell(file); + int64_t size = (int64_t)ftell(file); for (int i = 0; i < 8; i++) { hash ^= (uint64_t)((size >> (i * 8)) & 0xff); hash *= 1099511628211ull; @@ -1721,7 +1796,7 @@ static bool parse_args(int argc, char **argv, server_opts *opts) { } typedef struct { - int *fds; + ei_socket *fds; size_t capacity; size_t head; size_t count; @@ -1753,7 +1828,7 @@ static void socket_queue_init(socket_queue *queue, size_t capacity) { } } -static bool socket_queue_try_push(socket_queue *queue, int fd) { +static bool socket_queue_try_push(socket_queue *queue, ei_socket fd) { pthread_mutex_lock(&queue->mutex); if (queue->count == queue->capacity) { pthread_mutex_unlock(&queue->mutex); @@ -1767,10 +1842,10 @@ static bool socket_queue_try_push(socket_queue *queue, int fd) { return true; } -static int socket_queue_pop(socket_queue *queue) { +static ei_socket socket_queue_pop(socket_queue *queue) { pthread_mutex_lock(&queue->mutex); while (queue->count == 0) pthread_cond_wait(&queue->ready, &queue->mutex); - int fd = queue->fds[queue->head]; + ei_socket fd = queue->fds[queue->head]; queue->head = (queue->head + 1) % queue->capacity; queue->count--; pthread_mutex_unlock(&queue->mutex); @@ -1801,14 +1876,18 @@ static void keepalive_limiter_release(keepalive_limiter *limiter) { static void *server_worker_main(void *opaque) { server_worker *worker = opaque; for (;;) { - int fd = socket_queue_pop(worker->queue); + ei_socket fd = socket_queue_pop(worker->queue); http_connection connection = { .fd = fd }; +#ifdef _WIN32 + DWORD receive_timeout = worker->opts->keepalive_timeout_ms; +#else struct timeval receive_timeout = { .tv_sec = worker->opts->keepalive_timeout_ms / 1000u, .tv_usec = (suseconds_t)( worker->opts->keepalive_timeout_ms % 1000u) * 1000, }; - setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &receive_timeout, +#endif + ei_setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &receive_timeout, sizeof receive_timeout); bool keepalive_slot = worker->opts->keepalive_max_requests > 1 && keepalive_limiter_try_acquire(worker->keepalive); @@ -1822,7 +1901,7 @@ static void *server_worker_main(void *opaque) { } if (keepalive_slot) keepalive_limiter_release(worker->keepalive); free(connection.buffered.data); - close(fd); + ei_socket_close(fd); } return NULL; } @@ -1850,7 +1929,12 @@ static bool execute_engine_batch_slot(void *opaque, size_t slot, const int32_t * } int main(int argc, char **argv) { +#ifdef _WIN32 + WSADATA wsa; + if (WSAStartup(MAKEWORD(2, 2), &wsa) != 0) ei_die("Winsock initialization failed"); +#else signal(SIGPIPE, SIG_IGN); +#endif server_opts opts; if (!parse_args(argc, argv, &opts)) { @@ -2074,10 +2158,10 @@ int main(int argc, char **argv) { } } - int s = socket(AF_INET, SOCK_STREAM, 0); - if (s < 0) ei_die("socket failed: %s", strerror(errno)); + ei_socket s = socket(AF_INET, SOCK_STREAM, 0); + if (s == EI_INVALID_SOCKET) ei_die("socket failed: %s", strerror(errno)); int yes = 1; - setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof yes); + ei_setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof yes); struct sockaddr_in addr; memset(&addr, 0, sizeof addr); @@ -2105,6 +2189,10 @@ int main(int argc, char **argv) { /* Graceful shutdown so the persistent cache is flushed. sigaction without * SA_RESTART makes accept() return EINTR when a stop signal arrives. */ if (opts.persistent_cache_path) { +#ifdef _WIN32 + signal(SIGTERM, handle_stop_signal); + signal(SIGINT, handle_stop_signal); +#else struct sigaction action; memset(&action, 0, sizeof action); action.sa_handler = handle_stop_signal; @@ -2112,22 +2200,23 @@ int main(int argc, char **argv) { action.sa_flags = 0; sigaction(SIGTERM, &action, NULL); sigaction(SIGINT, &action, NULL); +#endif } while (!g_stop_requested) { - int c = accept(s, NULL, NULL); - if (c >= 0) { + ei_socket c = accept(s, NULL, NULL); + if (c != EI_INVALID_SOCKET) { int nodelay = 1; - setsockopt(c, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof nodelay); + ei_setsockopt(c, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof nodelay); } - if (c < 0) { - if (errno == EINTR) continue; + if (c == EI_INVALID_SOCKET) { + if (ei_socket_errno() == EINTR) continue; fprintf(stderr, "accept failed: %s\n", strerror(errno)); continue; } if (!socket_queue_try_push(&connection_queue, c)) { http_error(c, 503, "Service Unavailable", "server request queue is full", false); - close(c); + ei_socket_close(c); } } if (opts.persistent_cache_path) { diff --git a/src/windows_compat.h b/src/windows_compat.h new file mode 100644 index 0000000..40bcb76 --- /dev/null +++ b/src/windows_compat.h @@ -0,0 +1,49 @@ +/* Native Windows file helpers; MinGW supplies C11 and winpthreads. */ +#ifndef EI_WINDOWS_COMPAT_H +#define EI_WINDOWS_COMPAT_H +#ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +/* Exclusive creation preserves mkstemp's security and binary file semantics. */ +static inline int ei_mkstemp(char *pattern) { + size_t n = strlen(pattern); + if (n < 6 || strcmp(pattern + n - 6, "XXXXXX") != 0) { + errno = EINVAL; + return -1; + } + for (unsigned attempt = 0; attempt < 256; attempt++) { + unsigned int random; + if (rand_s(&random) != 0) { errno = EIO; return -1; } + static const char alphabet[] = "0123456789abcdefghijklmnopqrstuvwxyz"; + for (size_t i = n - 6; i < n; i++) { + pattern[i] = alphabet[random % 36]; + random /= 36; + } + int fd = _open(pattern, _O_CREAT | _O_EXCL | _O_RDWR | _O_BINARY, + _S_IREAD | _S_IWRITE); + if (fd >= 0 || errno != EEXIST) return fd; + } + errno = EEXIST; + return -1; +} +#define mkstemp ei_mkstemp +/* Windows rename does not replace a destination; cache publication must. */ +static inline int ei_replace_file(const char *from, const char *to) { + if (MoveFileExA(from, to, MOVEFILE_REPLACE_EXISTING)) return 0; + errno = EIO; + return -1; +} +#else +#define ei_replace_file rename +#endif +#endif From b3a37e86c17894d6a3648b84727b22b3188438c7 Mon Sep 17 00:00:00 2001 From: Jetha Chan Date: Sun, 4 Oct 2026 12:09:34 +0900 Subject: [PATCH 2/9] Validate Windows download paths and native inference with pinned test model --- .github/workflows/ci.yml | 12 +++++++-- Makefile | 7 ++++++ README.md | 25 +++++++++++++++++++ src/common.h | 4 +++ src/server.c | 46 +++++++++++++++++++++++++++++++++-- src/test_downloader_fixture.c | 16 ++++++++++++ src/test_inference_service.c | 7 ++++++ src/windows_compat.h | 1 + 8 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 src/test_downloader_fixture.c diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 515f07a..dbd165b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,11 +44,12 @@ jobs: - name: Run native Windows checks and build if: runner.os == 'Windows' shell: msys2 {0} - run: make check && make + run: make check && make && make test-windows-download - name: Download public upstream test model run: | mkdir -p model - curl -fL --retry 3 -o model/embeddinggemma-300M-qat-Q4_0.gguf 'https://huggingface.co/ggml-org/embeddinggemma-300M-qat-q4_0-GGUF/resolve/main/embeddinggemma-300M-qat-Q4_0.gguf' + curl -fL --retry 3 -o model/embeddinggemma-300M-qat-Q4_0.gguf 'https://huggingface.co/ggml-org/embeddinggemma-300M-qat-q4_0-GGUF/resolve/8dd0ca2a66a8f14470acb0e2a71f801afbc5fb73/embeddinggemma-300M-qat-Q4_0.gguf' + python -c 'import hashlib,pathlib; p=pathlib.Path("model/embeddinggemma-300M-qat-Q4_0.gguf"); assert hashlib.sha256(p.read_bytes()).hexdigest()=="50d28e22432a148f6f8a86eab3700f92add5d1f54baf7790675a2a4dadbccf26"' - name: Run embedding and HTTP conformance checks if: runner.os != 'Windows' run: make test @@ -56,6 +57,13 @@ jobs: if: runner.os == 'Windows' shell: msys2 {0} run: make test + - name: Strip Windows executable + if: runner.os == 'Windows' + shell: msys2 {0} + run: strip build/embeddinggemma.exe + - name: Strip Unix executable + if: runner.os != 'Windows' + run: strip build/embeddinggemma - name: Stage executable and checksum env: PLATFORM: ${{ matrix.platform }} diff --git a/Makefile b/Makefile index 04f414e..47d70dd 100644 --- a/Makefile +++ b/Makefile @@ -740,3 +740,10 @@ ROUTE ?= TENSOR=1,FLASH=1 .PHONY: bench-gemma2-metal bench-gemma2-metal: $(BUILD)/bench_gemma2_metal ./$(BUILD)/bench_gemma2_metal $(GEMMA2_MODEL) "$(ROUTE)" $(CASES) +# Process fixture contains fictional bytes, never a model or network request. +$(BUILD)/test_downloader_fixture: src/test_downloader_fixture.c | $(BUILD) + $(CC) $(CFLAGS) -o $@ $< $(LDLIBS) + +.PHONY: test-windows-download +test-windows-download: $(BUILD)/embeddinggemma $(BUILD)/test_downloader_fixture + python3 testdata/test_windows_download.py --binary ./$(BUILD)/embeddinggemma.exe --downloader ./$(BUILD)/test_downloader_fixture.exe diff --git a/README.md b/README.md index 80ece24..a8d9092 100644 --- a/README.md +++ b/README.md @@ -292,6 +292,31 @@ Published executables are deliberately small. Release `v0.3.1` contains: | Linux x86_64 | ROCm | 1.2 MiB | | Linux x86_64 | XPU SYCL | 2.1 MiB | +### Fork CI binaries + +This fork's `CI` workflow uploads checksummed CPU executable artifacts for +Linux x86_64, Linux ARM64, macOS ARM64, and Windows x86_64 after unit, model, +and HTTP conformance checks pass. Download artifacts from the workflow run; +these are CI builds, not upstream release assets. Existing CUDA, ROCm, XPU +and Metal release targets remain available through the upstream build process. + +CPU builds use the compiler's baseline architecture, without `-march=native`: +x86_64 uses SSE2 and ARM64 uses NEON. Linux artifacts are built on Ubuntu 24.04 +and require a compatible glibc (2.39 or later); macOS ARM64 targets macOS 14 or +later. Windows uses a native MinGW UCRT64 executable with statically linked +winpthreads and compiler runtime, tested on Windows Server 2022. It requires +Windows' Universal C Runtime; older Windows versions are not CI-tested. +No Windows GPU backend or PowerShell installer is provided by this change. + +On Windows, run `embeddinggemma-windows-x86_64-cpu.exe`. The default model +cache is `%LOCALAPPDATA%/embeddinggemma.c/`, falling back to +`%USERPROFILE%/.cache/embeddinggemma.c/`; `XDG_CACHE_HOME`, `EI_MODEL_PATH`, and +`--model` retain their existing precedence. Automatic download uses native +`curl.exe` or `wget.exe` on PATH and supports paths containing spaces. +Build locally from an MSYS2 UCRT64 shell with `make check`, +`make test-windows-download`, and `make test` after supplying the public model. +The model's Gemma license is separate from this repository's MIT license. + The 278 MB Q4_0 model is downloaded separately from Hugging Face and is not distributed by this project. diff --git a/src/common.h b/src/common.h index b5a62ec..4a13fa0 100644 --- a/src/common.h +++ b/src/common.h @@ -39,7 +39,11 @@ static inline float ei_fp16_to_fp32(ei_fp16 h) { return f; } +#ifdef _WIN32 +__attribute__((format(gnu_printf, 1, 2), noreturn)) +#else __attribute__((format(printf, 1, 2), noreturn)) +#endif static inline void ei_die(const char *fmt, ...); #include diff --git a/src/server.c b/src/server.c index 0580c02..3af78da 100644 --- a/src/server.c +++ b/src/server.c @@ -1444,8 +1444,34 @@ typedef enum { static downloader_result run_downloader(const char *program, char *const arguments[]) { #ifdef _WIN32 - /* _spawnvp handles argument quoting; never invoke a command shell. */ - intptr_t status = _spawnvp(_P_WAIT, program, (const char *const *)arguments); + /* The Windows CRT joins argv without quoting. Quote each argument using + * its command-line rules, preserving spaces and trailing backslashes. */ + size_t count = 0; + while (arguments[count]) count++; + char **quoted = ei_xcalloc(count + 1, sizeof(*quoted)); + for (size_t i = 0; i < count; i++) { + const char *arg = arguments[i]; + char *out = quoted[i] = ei_xmalloc(strlen(arg) * 2 + 3); + *out++ = '"'; + size_t slashes = 0; + for (;;) { + char ch = *arg++; + if (ch == '\\') { slashes++; continue; } + size_t emit = (ch == '"' || ch == '\0') ? slashes * 2 : slashes; + while (emit--) *out++ = '\\'; + slashes = 0; + if (ch == '\0') break; + if (ch == '"') *out++ = '\\'; + *out++ = ch; + } + *out++ = '"'; + *out = '\0'; + } + intptr_t status = _spawnvp(_P_WAIT, program, (const char *const *)quoted); + int spawn_errno = errno; + for (size_t i = 0; i < count; i++) free(quoted[i]); + free(quoted); + errno = spawn_errno; if (status == -1 && errno == ENOENT) return DOWNLOADER_UNAVAILABLE; if (status == 0) return DOWNLOADER_SUCCEEDED; fprintf(stderr, "%s failed (status %lld)\n", program, (long long)status); @@ -1619,7 +1645,11 @@ static void usage(const char *argv0) { " [--cuda-devices all|I,J,...] (embeddinggemma-2 on several GPUs)\n" "default listen: 0.0.0.0:%d\n" "default model: $XDG_CACHE_HOME/embeddinggemma.c/%s\n" +#ifdef _WIN32 + " or $LOCALAPPDATA/embeddinggemma.c/%s\n", +#else " or $HOME/.cache/embeddinggemma.c/%s\n", +#endif argv0, EMBEDDINGGEMMA_DEFAULT_PORT, ei_profiles[0].file, ei_profiles[0].file); } @@ -2203,6 +2233,18 @@ int main(int argc, char **argv) { #endif } while (!g_stop_requested) { +#ifdef _WIN32 + /* CRT signal handlers do not interrupt Winsock accept. Polling keeps + * graceful cache publication reachable after a stop signal. */ + fd_set ready; + FD_ZERO(&ready); + FD_SET(s, &ready); + struct timeval timeout = {0, 250000}; + int selected = select(0, &ready, NULL, NULL, &timeout); + if (g_stop_requested) break; + if (selected == 0) continue; + if (selected == SOCKET_ERROR) ei_die("select failed: %d", WSAGetLastError()); +#endif ei_socket c = accept(s, NULL, NULL); if (c != EI_INVALID_SOCKET) { int nodelay = 1; diff --git a/src/test_downloader_fixture.c b/src/test_downloader_fixture.c new file mode 100644 index 0000000..1aec7e3 --- /dev/null +++ b/src/test_downloader_fixture.c @@ -0,0 +1,16 @@ +/* Fictional downloader used only by the isolated process-spawning test. */ +#include +#include +#include +int main(int argc, char **argv) { + if (getenv("EI_TEST_CURL_FAIL") && strstr(argv[0], "curl")) return 7; + for (int i = 1; i + 1 < argc; i++) { + if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "-O") == 0) { + FILE *out = fopen(argv[i + 1], "wb"); + if (!out) return 2; + if (fwrite("fictional", 1, 9, out) != 9) return 3; + return fclose(out) == 0 ? 0 : 4; + } + } + return 5; +} diff --git a/src/test_inference_service.c b/src/test_inference_service.c index 70e0f14..9ad81ac 100644 --- a/src/test_inference_service.c +++ b/src/test_inference_service.c @@ -3,6 +3,13 @@ #include "inference_service.h" #include +#ifdef _WIN32 +static int setenv(const char *name, const char *value, int overwrite) { + if (!overwrite && getenv(name)) return 0; + return _putenv_s(name, value); +} +static int unsetenv(const char *name) { return _putenv_s(name, ""); } +#endif #include #include diff --git a/src/windows_compat.h b/src/windows_compat.h index 40bcb76..aaf6339 100644 --- a/src/windows_compat.h +++ b/src/windows_compat.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include From c1d721dcfcc6be01645fa35ba4e46cbd131847b5 Mon Sep 17 00:00:00 2001 From: Jetha Chan Date: Sun, 4 Oct 2026 12:10:04 +0900 Subject: [PATCH 3/9] Include isolated Windows downloader regression test --- testdata/test_windows_download.py | 56 +++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 testdata/test_windows_download.py diff --git a/testdata/test_windows_download.py b/testdata/test_windows_download.py new file mode 100644 index 0000000..2d46b62 --- /dev/null +++ b/testdata/test_windows_download.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Test native Windows download spawning without network or household data.""" +import argparse +import os +from pathlib import Path +import shutil +import subprocess +import tempfile + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--binary", required=True) + parser.add_argument("--downloader", required=True) + args = parser.parse_args() + binary = str(Path(args.binary).resolve()) + downloader = Path(args.downloader).resolve() + with tempfile.TemporaryDirectory(prefix="embeddinggemma fictional ") as temp: + root = Path(temp) + for mode in ("curl", "fallback", "wget", "missing", "profile", "explicit"): + directory = root / mode + directory.mkdir() + tools = directory / "tools with spaces" + tools.mkdir() + if mode not in ("wget", "missing"): + shutil.copyfile(downloader, tools / "curl.exe") + if mode in ("fallback", "wget"): + shutil.copyfile(downloader, tools / "wget.exe") + env = os.environ.copy() + for key in ("XDG_CACHE_HOME", "EI_MODEL_PATH", "EI_TEST_CURL_FAIL", "HOME"): + env.pop(key, None) + env.update(PATH=str(tools), LOCALAPPDATA=str(directory), USERPROFILE=str(directory)) + command = [binary] + expected = directory / "embeddinggemma.c" / "embeddinggemma-300M-qat-Q4_0.gguf" + if mode == "fallback": + env["EI_TEST_CURL_FAIL"] = "1" + if mode == "profile": + env.pop("LOCALAPPDATA") + expected = directory / ".cache" / "embeddinggemma.c" / expected.name + if mode == "explicit": + expected = directory / "explicit path" / "model.gguf" + command += ["--model", str(expected)] + result = subprocess.run(command, env=env, capture_output=True, timeout=20) + assert result.returncode != 0, result.stderr + if mode == "missing": + assert b"neither curl nor wget is available" in result.stderr, result.stderr + assert not expected.exists() + else: + assert expected.read_bytes() == b"fictional", result.stderr + assert b"not a GGUF file" in result.stderr, result.stderr + assert not list(directory.rglob("*.download.*")) + print("Windows download paths, quoting, fallback and cleanup: passed") + + +if __name__ == "__main__": + main() From 7dad9b2f7427542fc17caf7320869699b18466cc Mon Sep 17 00:00:00 2001 From: Jetha Chan Date: Sun, 4 Oct 2026 12:13:25 +0900 Subject: [PATCH 4/9] Support current MinGW headers and verify native binary architectures --- .github/workflows/ci.yml | 22 +++++++++++++++++++++- README.md | 2 +- scripts/check-binary-platform.py | 26 ++++++++++++++++++++++++++ src/server.c | 2 ++ 4 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 scripts/check-binary-platform.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbd165b..f93d939 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: platform: linux-x86_64 - os: ubuntu-24.04-arm platform: linux-arm64 - - os: macos-14 + - os: macos-15 platform: darwin-arm64 - os: windows-2022 platform: windows-x86_64 @@ -57,6 +57,26 @@ jobs: if: runner.os == 'Windows' shell: msys2 {0} run: make test + - name: Verify actual executable format and architecture + env: + PLATFORM: ${{ matrix.platform }} + run: | + suffix='' + if [ "$RUNNER_OS" = Windows ]; then suffix='.exe'; fi + python scripts/check-binary-platform.py "$PLATFORM" "build/embeddinggemma${suffix}" + - name: Inspect Windows runtime dependencies + if: runner.os == 'Windows' + shell: msys2 {0} + run: | + objdump -f build/embeddinggemma.exe + objdump -p build/embeddinggemma.exe | grep 'DLL Name' + if objdump -p build/embeddinggemma.exe | grep -Ei 'DLL Name:.*(msys-|cygwin|libwinpthread|libgcc|libstdc)'; then + echo 'unexpected non-system runtime DLL dependency' >&2 + exit 1 + fi + - name: Inspect Unix executable + if: runner.os != 'Windows' + run: uname -m && file build/embeddinggemma && cc --version - name: Strip Windows executable if: runner.os == 'Windows' shell: msys2 {0} diff --git a/README.md b/README.md index a8d9092..7f8e42e 100644 --- a/README.md +++ b/README.md @@ -302,7 +302,7 @@ and Metal release targets remain available through the upstream build process. CPU builds use the compiler's baseline architecture, without `-march=native`: x86_64 uses SSE2 and ARM64 uses NEON. Linux artifacts are built on Ubuntu 24.04 -and require a compatible glibc (2.39 or later); macOS ARM64 targets macOS 14 or +and require a compatible glibc (2.39 or later); macOS ARM64 is tested on the macOS 15 runner and targets macOS 14 or later. Windows uses a native MinGW UCRT64 executable with statically linked winpthreads and compiler runtime, tested on Windows Server 2022. It requires Windows' Universal C Runtime; older Windows versions are not CI-tested. diff --git a/scripts/check-binary-platform.py b/scripts/check-binary-platform.py new file mode 100644 index 0000000..4f46d7c --- /dev/null +++ b/scripts/check-binary-platform.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Reject a binary whose native format or architecture disagrees with its label.""" +from pathlib import Path +import struct +import sys + +platform, filename = sys.argv[1:] +data = Path(filename).read_bytes() +if platform.startswith("linux-"): + assert data[:6] == b"\x7fELF\x02\x01", "expected a little-endian ELF64 executable" + machine = struct.unpack_from(" Date: Sun, 4 Oct 2026 12:14:40 +0900 Subject: [PATCH 5/9] Include Windows runtime notices and build provenance with artifacts --- .github/workflows/ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f93d939..1188504 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,6 +84,25 @@ jobs: - name: Strip Unix executable if: runner.os != 'Windows' run: strip build/embeddinggemma + - name: Preserve Windows runtime notices and toolchain provenance + if: runner.os == 'Windows' + shell: msys2 {0} + run: | + mkdir -p dist/licenses + packages=(mingw-w64-ucrt-x86_64-crt mingw-w64-ucrt-x86_64-winpthreads mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-libgcc) + pacman -Q "${packages[@]}" > dist/toolchain.txt + gcc --version >> dist/toolchain.txt + while IFS= read -r notice; do + if [ -f "$notice" ]; then + relative="${notice#*/share/licenses/}" + mkdir -p "dist/licenses/$(dirname "$relative")" + cp "$notice" "dist/licenses/$relative" + fi + done < <(pacman -Qql "${packages[@]}" | grep '/share/licenses/') + test -n "$(find dist/licenses -type f -print -quit)" + - name: Preserve Unix toolchain provenance + if: runner.os != 'Windows' + run: mkdir -p dist && cc --version > dist/toolchain.txt - name: Stage executable and checksum env: PLATFORM: ${{ matrix.platform }} @@ -95,6 +114,7 @@ jobs: cp "build/embeddinggemma${suffix}" "dist/$name" python -c 'import hashlib,pathlib; p=next(pathlib.Path("dist").glob("embeddinggemma-*")); p.with_name(p.name+".sha256").write_text(hashlib.sha256(p.read_bytes()).hexdigest()+" "+p.name+"\n")' cp LICENSE dist/LICENSE + git rev-parse HEAD > dist/SOURCE_REVISION - name: Upload verified executable uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: From 9765f25a2dd51f459f98c24db9f96058f88af079 Mon Sep 17 00:00:00 2001 From: Jetha Chan Date: Sun, 4 Oct 2026 12:21:51 +0900 Subject: [PATCH 6/9] Preserve UNC share roots when creating model cache directories --- Makefile | 6 +++++- src/server.c | 14 ++++++++++++-- src/test_windows_paths.c | 30 ++++++++++++++++++++++++++++++ src/windows_compat.h | 13 +++++++++++++ 4 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 src/test_windows_paths.c diff --git a/Makefile b/Makefile index 47d70dd..5a8f5b6 100644 --- a/Makefile +++ b/Makefile @@ -745,5 +745,9 @@ $(BUILD)/test_downloader_fixture: src/test_downloader_fixture.c | $(BUILD) $(CC) $(CFLAGS) -o $@ $< $(LDLIBS) .PHONY: test-windows-download -test-windows-download: $(BUILD)/embeddinggemma $(BUILD)/test_downloader_fixture +$(BUILD)/test_windows_paths: src/test_windows_paths.c src/windows_compat.h src/common.h | $(BUILD) + $(CC) $(CFLAGS) -o $@ $< $(LDLIBS) + +test-windows-download: $(BUILD)/embeddinggemma $(BUILD)/test_downloader_fixture $(BUILD)/test_windows_paths + ./$(BUILD)/test_windows_paths python3 testdata/test_windows_download.py --binary ./$(BUILD)/embeddinggemma.exe --downloader ./$(BUILD)/test_downloader_fixture.exe diff --git a/src/server.c b/src/server.c index a578c77..8c2ae30 100644 --- a/src/server.c +++ b/src/server.c @@ -1426,8 +1426,18 @@ static void mkdir_p(const char *path) { free(tmp); return; } - for (size_t i = 1; i < n; i++) { - if (tmp[i] == '/' && !(i == 2 && tmp[1] == ':')) { + size_t start = 1; +#ifdef _WIN32 + start = ei_windows_directory_start(tmp); + if (start == SIZE_MAX) ei_die("invalid UNC directory: %s", path); + if (tmp[0] == '/' && tmp[1] == '/' && start >= n) { + /* Existing share roots are managed by SMB, never created here. */ + free(tmp); + return; + } +#endif + for (size_t i = start; i < n; i++) { + if (tmp[i] == '/') { tmp[i] = '\0'; ensure_dir(tmp); tmp[i] = '/'; diff --git a/src/test_windows_paths.c b/src/test_windows_paths.c new file mode 100644 index 0000000..7d9b46a --- /dev/null +++ b/src/test_windows_paths.c @@ -0,0 +1,30 @@ +/* Synthetic UNC roots need no SMB service or access to a real network share. */ +#include "common.h" +#include "windows_compat.h" +#include +int main(void) { + assert(ei_windows_directory_start("//fictional-server/share/cache") == + strlen("//fictional-server/share") + 1); + assert(ei_windows_directory_start("//fictional-server/share") == + strlen("//fictional-server/share") + 1); + assert(ei_windows_directory_start("//fictional-server/") == SIZE_MAX); + assert(ei_windows_directory_start("//fictional-server//cache") == SIZE_MAX); + assert(ei_windows_directory_start("C:/fictional directory/cache") == 3); + assert(ei_windows_directory_start("fictional/cache") == 1); + char first[] = "build/windows-file-a.XXXXXX"; + char second[] = "build/windows-file-b.XXXXXX"; + int a = mkstemp(first), b = mkstemp(second); + assert(a >= 0 && b >= 0 && strcmp(first, second) != 0); + const char bytes[] = {'\r', '\n', 26, 0}; + assert(_write(a, bytes, sizeof bytes) == (int)sizeof bytes); + assert(_close(a) == 0 && _close(b) == 0); + assert(ei_replace_file(first, second) == 0); + FILE *file = fopen(second, "rb"); + assert(file); + char actual[sizeof bytes]; + assert(fread(actual, 1, sizeof actual, file) == sizeof actual); + assert(memcmp(actual, bytes, sizeof bytes) == 0); + assert(fclose(file) == 0 && _unlink(second) == 0); + puts("Windows UNC roots, binary temporary files and cache replacement: passed"); + return 0; +} diff --git a/src/windows_compat.h b/src/windows_compat.h index aaf6339..2ac13e8 100644 --- a/src/windows_compat.h +++ b/src/windows_compat.h @@ -15,6 +15,19 @@ #include #include +/* A network server is not a directory: preserve the complete UNC share root + * before mkdir_p visits any separator. Input separators are already '/'. */ +static inline size_t ei_windows_directory_start(const char *path) { + if (path[0] == '/' && path[1] == '/') { + const char *server_end = strchr(path + 2, '/'); + if (!server_end || server_end == path + 2 || !server_end[1]) return SIZE_MAX; + const char *share_end = strchr(server_end + 1, '/'); + if (share_end == server_end + 1) return SIZE_MAX; + return share_end ? (size_t)(share_end - path) + 1 : strlen(path) + 1; + } + return strlen(path) >= 3 && path[1] == ':' && path[2] == '/' ? 3 : 1; +} + /* Exclusive creation preserves mkstemp's security and binary file semantics. */ static inline int ei_mkstemp(char *pattern) { size_t n = strlen(pattern); From 11b838dd17644a87bed703135e8b4efe5336b116 Mon Sep 17 00:00:00 2001 From: Jetha Chan Date: Fri, 9 Oct 2026 09:08:21 +0900 Subject: [PATCH 7/9] Dispatch AVX2 CPU kernels at runtime on x86-64 (#1) * Dispatch AVX2 CPU kernels at runtime on x86-64 Portable x86-64 builds target the SSE2 baseline, so the compile-time __AVX2__ kernels never ran in shipped binaries and the Q4_0 x Q8_0 dots fell back to scalar C. Move the AVX2 kernels into kernels_avx2.c behind per-function target("avx2,fma,f16c") attributes and call them after a one-time CPUID/XGETBV check. EI_CPU_ISA=auto|baseline|avx2 overrides it. Use hardware F16C for the block-scale conversions (exact, bit-identical) and FMA accumulation in the Q4_0 dots. ARM64 NEON object code is unchanged. Add test_cpu_dispatch (SSE2 vs AVX2 embeddings at 768/512/256/128 dims), run test_kernels under both routes, run the golden test with the baseline route too, and add a CI check that AVX code stays in ei_avx2_* functions. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AAuLpjSTeWf1GysUY4DqwM * Gate SSE2/AVX2 parity at the golden cosine and report every input On the real model a last-bit summation difference can flip a Q8_0 activation rounding and cascade, so the routes reach cosine ~0.99988 rather than bit equality; both match the llama.cpp goldens equally (0.99986 AVX2, 0.99987 SSE2). Collect all comparisons before failing. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AAuLpjSTeWf1GysUY4DqwM --------- Co-authored-by: Claude --- .github/workflows/ci.yml | 7 + CONTRIBUTING.md | 9 +- Makefile | 21 ++- PORT_SPEC.md | 2 +- README.md | 4 +- perf/optimization_status.md | 48 +++++ scripts/check-avx2-isolation.py | 62 +++++++ src/kernels.c | 93 ++-------- src/kernels_avx2.c | 306 ++++++++++++++++++++++++++++++++ src/kernels_avx2.h | 70 ++++++++ src/quants.c | 183 ++----------------- src/test_cpu_dispatch.c | 244 +++++++++++++++++++++++++ src/test_kernels.c | 60 ++++++- 13 files changed, 851 insertions(+), 258 deletions(-) create mode 100755 scripts/check-avx2-isolation.py create mode 100644 src/kernels_avx2.c create mode 100644 src/kernels_avx2.h create mode 100644 src/test_cpu_dispatch.c diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1188504..94b8d60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,13 @@ jobs: suffix='' if [ "$RUNNER_OS" = Windows ]; then suffix='.exe'; fi python scripts/check-binary-platform.py "$PLATFORM" "build/embeddinggemma${suffix}" + - name: Verify AVX2 code stays in runtime-dispatched kernels + if: matrix.platform == 'linux-x86_64' + run: python3 scripts/check-avx2-isolation.py build/embeddinggemma + - name: Verify Windows AVX2 code stays in runtime-dispatched kernels + if: runner.os == 'Windows' + shell: msys2 {0} + run: python scripts/check-avx2-isolation.py build/embeddinggemma.exe - name: Inspect Windows runtime dependencies if: runner.os == 'Windows' shell: msys2 {0} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0c78ef3..afdf377 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,8 +87,10 @@ make ``` The output is `build/embeddinggemma`. The CPU route includes scalar C, ARM64 -NEON, AVX2, and SSSE3 kernels and selects the best available implementation at -runtime. +NEON, SSE2/SSSE3, and AVX2 kernels. NEON and SSE2/SSSE3 follow the compiler +target; on x86-64 the AVX2 kernels (`src/kernels_avx2.c`, with per-function +`target("avx2,fma,f16c")` attributes) are selected at runtime when the CPU and +OS support AVX2, FMA, and F16C, so one baseline binary runs on any x86-64 CPU. Build the self-contained Metal server: @@ -231,6 +233,9 @@ Diagnostic controls: - `EI_THREADS` (1..1024; the server's `--threads` sets it): override the CPU thread count. The default is every core up to 32. +- `EI_CPU_ISA=auto|baseline|avx2`: x86-64 kernel family. `baseline` forces the + SSE2 route on an AVX2 host; `avx2` fails at startup when unsupported. + `test_cpu_dispatch` compares the two routes in one process. - `EI_CPU_SHORT_THREADS`: override the short-projection worker width. - `EI_CPU_DUAL_PROJECTION`: control paired projection accumulation. - `EI_CPU_FUSED_RMS_QUANT=0`: materialize normalized activations before Q8. diff --git a/Makefile b/Makefile index 5a8f5b6..ce848f3 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,9 @@ CFLAGS ?= -std=c11 -O2 -Wall -Wextra -Werror -g LDLIBS ?= -lm -pthread ifneq ($(findstring MINGW,$(shell uname -s)),) CFLAGS += -D_WIN32_WINNT=0x0601 -D_CRT_RAND_S -D__USE_MINGW_ANSI_STDIO=1 +# Win64 GCC does not realign the stack for 32-byte AVX spills (GCC bug 54412); +# assembling aligned vector moves as unaligned keeps the AVX2 kernels safe. +CFLAGS += -Wa,-muse-unaligned-vector-move LDLIBS += -lws2_32 -static endif NVCC ?= nvcc @@ -103,7 +106,8 @@ HDRS_NATIVE := $(wildcard src/*.h src/vendor/*.h) SRCS_CORE := src/gguf.c SRCS_MODEL := $(SRCS_CORE) src/model.c SRCS_TOKENIZER := $(SRCS_MODEL) src/tokenizer.c -SRCS_ENGINE := $(SRCS_TOKENIZER) src/quants.c src/kernels.c src/parallel.c src/engine.c src/gemma2.c src/gemma2_fast.c src/gemma2_fast_x86.c src/media.c src/response_cache.c +SRCS_KERNELS := src/quants.c src/kernels.c src/kernels_avx2.c +SRCS_ENGINE := $(SRCS_TOKENIZER) $(SRCS_KERNELS) src/parallel.c src/engine.c src/gemma2.c src/gemma2_fast.c src/gemma2_fast_x86.c src/media.c src/response_cache.c SRCS_SERVICE := src/inference_service.c SRCS_SERVER := src/server.c src/float_format.c @@ -188,8 +192,11 @@ $(BUILD)/test_embed: src/test_embed.c $(SRCS_ENGINE) $(HDRS_NATIVE) | $(BUILD) $(BUILD)/test_batch: src/test_batch.c $(SRCS_ENGINE) $(HDRS_NATIVE) | $(BUILD) $(CC) $(CFLAGS) -o $@ src/test_batch.c $(SRCS_ENGINE) $(LDLIBS) -$(BUILD)/test_kernels: src/test_kernels.c src/quants.c src/kernels.c src/gguf.c $(HDRS_NATIVE) | $(BUILD) - $(CC) $(CFLAGS) -o $@ src/test_kernels.c src/quants.c src/kernels.c src/gguf.c $(LDLIBS) +$(BUILD)/test_cpu_dispatch: src/test_cpu_dispatch.c $(SRCS_ENGINE) $(HDRS_NATIVE) | $(BUILD) + $(CC) $(CFLAGS) -o $@ src/test_cpu_dispatch.c $(SRCS_ENGINE) $(LDLIBS) + +$(BUILD)/test_kernels: src/test_kernels.c $(SRCS_KERNELS) src/gguf.c $(HDRS_NATIVE) | $(BUILD) + $(CC) $(CFLAGS) -o $@ src/test_kernels.c $(SRCS_KERNELS) src/gguf.c $(LDLIBS) $(BUILD)/embeddinggemma: $(SRCS_SERVER) $(SRCS_ENGINE) $(SRCS_SERVICE) $(HDRS_NATIVE) | $(BUILD) $(CC) $(CFLAGS) -o $@ $(SRCS_SERVER) $(SRCS_ENGINE) $(SRCS_SERVICE) $(LDLIBS) @@ -378,8 +385,8 @@ $(BUILD)/test_batch_metal: src/test_batch.c $(SRCS_ENGINE) $(METAL_ENGINE_OBJS) $(CC) $(CFLAGS) -DEI_ENABLE_METAL -o $@ src/test_batch.c $(SRCS_ENGINE) \ $(METAL_ENGINE_OBJS) $(LDLIBS) $(METALLIB_LDFLAGS) $(METAL_FRAMEWORKS) -$(BUILD)/perf_kernels: perf/harness/bench_kernels.c src/quants.c src/kernels.c src/gguf.c $(HDRS_NATIVE) | $(BUILD) - $(CC) $(CFLAGS) -Isrc -o $@ perf/harness/bench_kernels.c src/quants.c src/kernels.c src/gguf.c $(LDLIBS) +$(BUILD)/perf_kernels: perf/harness/bench_kernels.c $(SRCS_KERNELS) src/gguf.c $(HDRS_NATIVE) | $(BUILD) + $(CC) $(CFLAGS) -Isrc -o $@ perf/harness/bench_kernels.c $(SRCS_KERNELS) src/gguf.c $(LDLIBS) $(BUILD)/perf_engine: perf/harness/bench_engine.c $(SRCS_ENGINE) $(HDRS_NATIVE) | $(BUILD) $(CC) $(CFLAGS) -Isrc -o $@ perf/harness/bench_engine.c $(SRCS_ENGINE) $(LDLIBS) @@ -437,13 +444,15 @@ $(BUILD)/xpu: mkdir -p $(BUILD)/xpu .PHONY: test test-unit test-defaults check test-http test-http-metal test-http-cuda test-http-rocm test-http-xpu test-metal test-cuda test-rocm test-xpu perf perf-engine perf-engine-metal perf-engine-cuda perf-engine-rocm perf-engine-xpu perf-concurrency perf-concurrency-cuda perf-concurrency-rocm perf-concurrency-xpu perf-dimensions perf-batch perf-tokenization perf-servers perf-compare-llamacpp metal metal-kernels cuda rocm xpu check-scripts clean clean-cpu clean-metal clean-cuda clean-rocm clean-xpu release-darwin release-linux-cpu release-linux-cuda release-linux-rocm release-linux-xpu release-checksums release-verify release-ready release-info help -test: $(BUILD)/embeddinggemma $(BUILD)/test_gguf $(BUILD)/test_tokenizer $(BUILD)/test_kernels $(BUILD)/test_embed $(BUILD)/test_batch $(BUILD)/test_inference_service $(BUILD)/test_response_cache $(BUILD)/test_float_format +test: $(BUILD)/embeddinggemma $(BUILD)/test_gguf $(BUILD)/test_tokenizer $(BUILD)/test_kernels $(BUILD)/test_embed $(BUILD)/test_batch $(BUILD)/test_cpu_dispatch $(BUILD)/test_inference_service $(BUILD)/test_response_cache $(BUILD)/test_float_format python3 testdata/test_model_manifest.py --binary ./$(BUILD)/test_gguf \ --model $(MODEL) --manifest testdata/model-manifest.json ./$(BUILD)/test_tokenizer $(MODEL) testdata/goldens-tokens.json ./$(BUILD)/test_kernels ./$(BUILD)/test_embed $(MODEL) testdata/goldens-llamacpp.json + EI_CPU_ISA=baseline ./$(BUILD)/test_embed $(MODEL) testdata/goldens-llamacpp.json ./$(BUILD)/test_batch $(MODEL) cpu + ./$(BUILD)/test_cpu_dispatch $(MODEL) testdata/test-strings.json ./$(BUILD)/test_inference_service ./$(BUILD)/test_response_cache ./$(BUILD)/test_float_format diff --git a/PORT_SPEC.md b/PORT_SPEC.md index 9e6c331..2d342b2 100644 --- a/PORT_SPEC.md +++ b/PORT_SPEC.md @@ -144,7 +144,7 @@ llama.cpp deep-read) before implementing. | target | status | implementation/build | |---|---|---| | macOS ARM64 CPU | implemented, parity tested | C11 + NEON dot-product path | -| x86 CPU | implemented, parity tested | scalar, SSSE3, and AVX2 paths | +| x86 CPU | implemented, parity tested | scalar, SSSE3, and runtime-dispatched AVX2 paths | | macOS Metal | implemented, parity tested | Objective-C host + standalone metallib; short-shape residual/next-norm fusion; FP16 K/V at long sequence lengths with FP32 accumulation | | NVIDIA CUDA | implemented, parity tested | Q4 x Q8 DP4A latency path; direct-FP16 combined-QKV epilogue; Flash, online, banded-SWA, and tensor-core attention; native packed-Q4 MMA diagnostics; CUDA graph replay | | ROCm/HIP | implemented, parity tested | portable CDNA fat binary; paired packed-Q4 wave and MFMA projection kernels; exact singleton V-only attention; batched hipBLAS GQA; FP16/FP32 score routing; fused residual/RMS and pooling epilogues | diff --git a/README.md b/README.md index 7f8e42e..525c102 100644 --- a/README.md +++ b/README.md @@ -301,7 +301,9 @@ these are CI builds, not upstream release assets. Existing CUDA, ROCm, XPU and Metal release targets remain available through the upstream build process. CPU builds use the compiler's baseline architecture, without `-march=native`: -x86_64 uses SSE2 and ARM64 uses NEON. Linux artifacts are built on Ubuntu 24.04 +x86_64 uses SSE2 and ARM64 uses NEON. On x86_64 the same executable switches to +AVX2/FMA/F16C kernels at startup when the CPU supports them (Haswell, Zen, and +newer); set `EI_CPU_ISA=baseline` to force the SSE2 route. Linux artifacts are built on Ubuntu 24.04 and require a compatible glibc (2.39 or later); macOS ARM64 is tested on the macOS 15 runner and targets macOS 14 or later. Windows uses a native MinGW UCRT64 executable with statically linked winpthreads and compiler runtime, tested on Windows Server 2022. It requires diff --git a/perf/optimization_status.md b/perf/optimization_status.md index a0a0d76..aa8e629 100644 --- a/perf/optimization_status.md +++ b/perf/optimization_status.md @@ -3927,3 +3927,51 @@ dispatched 4 simdgroups for an 8-simdgroup kernel (NaN output); it now uses - Cross-request image packing (under 10% at 432x432, 3.5% at 768x768). - Hardware image decode (PNG decode 1.7-4.8 ms of a 21-56 ms miss). - Flash softmax fast exp and rescale skip (above). +## 2026-10-08: x86-64 Runtime AVX2 Dispatch + +Status: candidate (fork PR; upstream proposal pending). + +Current implementation: portable x86-64 CPU binaries are built at the compiler +baseline (SSE2), so the existing compile-time `__AVX2__` kernels never ran in +release or CI artifacts. Worse, the Q4_0 x Q8_0 dot products had no SSE2 path +at all, so shipped binaries ran the projections in scalar C +(`ei_quants_kernel_variant() == "cpu-scalar"`). + +Current public route: `src/kernels_avx2.c` holds the AVX2 kernels behind +per-function `target("avx2,fma,f16c")` attributes. `kernels.c` and `quants.c` +call them through `EI_DISPATCH_AVX2` after a one-time CPUID + XGETBV check +(AVX2, FMA, F16C, OS YMM state). `EI_CPU_ISA=auto|baseline|avx2` overrides it. +`scripts/check-avx2-isolation.py` fails CI if VEX code appears outside the +`ei_avx2_*` functions. ARM64 NEON object code is byte-identical to before. + +Correctness: `test_cpu_dispatch` embeds the test strings plus 150- and +600-token inputs and a packed batch under both routes in one process, gated at +cosine >= 0.999 and max abs diff <= 1e-2 at D=768/512/256/128 (the golden +gate; a last-bit summation difference can flip a Q8_0 activation rounding and +cascade, so the routes are not bit-identical on the real model). Against the +llama.cpp goldens on the real model, AVX2 min cosine is 0.999861 and SSE2 is +0.999871. +`test_kernels` runs every kernel test under both routes and checks that Q8_0 +quantization is bit-identical and Q4_0 rows/dual/batch agree within 3e-7. + +Experiments (sandbox Intel Xeon @ 2.80 GHz, 4 vCPU, GCC 13.3, synthetic +random-weight GGUF with the production shapes, 150-token chunk, best of 3): + +- Moving the existing AVX2 kernels behind dispatch: SSE2 1,326 ms vs AVX2 + 535 ms (2.5x). +- Callgrind on the AVX2 route showed 56% of instructions in the software + `ei_fp16_to_fp32` block-scale conversion. Hardware F16C (`_cvtsh_ss`, exact) + in the AVX2 kernels: 387 ms (3.3x vs SSE2), output bit-identical. +- FMA accumulation in the Q4_0 dots: q4gemv 25-27 vs 20-24 GFLOPS with + separate mul+add. Retained. +- `EI_CPU_MULTIROW_MIN_TOKENS=1` or `64` (four-token batch kernel) at 150 + tokens: 461-465 ms vs 387 ms. Rejected; the 512 boundary stays. + +Decision: retain runtime dispatch, F16C, and FMA. + +Open questions: the remaining gap to llama.cpp at 150 tokens is structural, +not ISA width. The CPU engine still runs per-token GEMV below 512 tokens and +re-streams all ~57 MB of layer weights for every token. llama.cpp's x86 +prefill uses tiled/repacked Q4_0 GEMM. A register-blocked Q4_0 x Q8_0 GEMM +(several rows x several tokens per tile) is the next CPU lever; AVX-VNNI would +come after that. diff --git a/scripts/check-avx2-isolation.py b/scripts/check-avx2-isolation.py new file mode 100755 index 0000000..7b3f96b --- /dev/null +++ b/scripts/check-avx2-isolation.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Verify that a portable x86-64 executable keeps AVX code inside the +runtime-dispatched ei_avx2_* kernels. + +Every other function must run on an SSE2-only CPU, so it must not contain a +VEX-encoded instruction (mnemonics starting with "v") or touch a ymm/zmm +register. Fails if the dispatched kernels are missing or AVX leaks elsewhere. +""" +import argparse +import re +import subprocess +import sys + +FUNCTION = re.compile(r"^[0-9a-f]+ <(?P[^>]+)>:$") +INSTRUCTION = re.compile(r"^\s*[0-9a-f]+:\s+(?P[a-z][a-z0-9.]*)\s*(?P.*)$") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary") + parser.add_argument("--objdump", default="objdump") + args = parser.parse_args() + + listing = subprocess.run( + [args.objdump, "-d", "--no-show-raw-insn", args.binary], + check=True, capture_output=True, text=True, + ).stdout + + function = None + dispatched = set() + leaks = {} + for line in listing.splitlines(): + match = FUNCTION.match(line) + if match: + function = match.group("name") + continue + match = INSTRUCTION.match(line) + if not match or function is None: + continue + mnemonic = match.group("mnemonic") + operands = match.group("operands") + if not (mnemonic.startswith("v") or "%ymm" in operands or "%zmm" in operands): + continue + if function.startswith("ei_avx2_"): + dispatched.add(function) + else: + leaks.setdefault(function, f"{mnemonic} {operands}".strip()) + + if leaks: + for name, example in sorted(leaks.items()): + print(f"AVX instruction outside dispatched kernels: {name}: {example}", + file=sys.stderr) + return 1 + if not dispatched: + print("no ei_avx2_* kernels found; runtime dispatch is missing", file=sys.stderr) + return 1 + print(f"AVX code isolated to {len(dispatched)} dispatched ei_avx2_* kernels") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/kernels.c b/src/kernels.c index 0732269..7594120 100644 --- a/src/kernels.c +++ b/src/kernels.c @@ -1,18 +1,23 @@ #include "kernels.h" +#include "kernels_avx2.h" #include #if defined(__ARM_NEON) && defined(__aarch64__) #include -#elif defined(__AVX2__) || defined(__SSE2__) -#include +#elif defined(__SSE2__) +#include #endif +/* On x86-64, EI_DISPATCH_AVX2 hands off to kernels_avx2.c at runtime; the + * inline paths below are the NEON, SSE2 baseline, and scalar routes. */ + const char *ei_cpu_kernel_variant(void) { +#if defined(EI_X86_DISPATCH) + if (ei_cpu_avx2_active()) return "cpu-avx2"; +#endif #if defined(__ARM_NEON) && defined(__aarch64__) return "cpu-neon"; -#elif defined(__AVX2__) - return "cpu-avx2"; #elif defined(__SSSE3__) return "cpu-ssse3"; #elif defined(__SSE2__) @@ -22,16 +27,7 @@ const char *ei_cpu_kernel_variant(void) { #endif } -#if defined(__AVX2__) -static inline float ei_hsum_f32x8(__m256 v) { - __m128 sum = _mm_add_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps(v, 1)); - sum = _mm_add_ps(sum, _mm_movehl_ps(sum, sum)); - sum = _mm_add_ss(sum, _mm_shuffle_ps(sum, sum, 1)); - return _mm_cvtss_f32(sum); -} -#endif - -#if defined(__SSE2__) && !defined(__AVX2__) +#if defined(__SSE2__) static inline float ei_hsum_f32x4(__m128 v) { v = _mm_add_ps(v, _mm_movehl_ps(v, v)); v = _mm_add_ss(v, _mm_shuffle_ps(v, v, 1)); @@ -40,6 +36,7 @@ static inline float ei_hsum_f32x4(__m128 v) { #endif static float ei_sum_squares(const float *x, int32_t n) { + EI_DISPATCH_AVX2(return ei_avx2_sum_squares(x, n)); #if defined(__ARM_NEON) && defined(__aarch64__) float32x4_t sum0 = vdupq_n_f32(0.0f); float32x4_t sum1 = vdupq_n_f32(0.0f); @@ -59,26 +56,6 @@ static float ei_sum_squares(const float *x, int32_t n) { float sum = vaddvq_f32(vaddq_f32(vaddq_f32(sum0, sum1), vaddq_f32(sum2, sum3))); for (; i < n; i++) sum += x[i] * x[i]; return sum; -#elif defined(__AVX2__) - __m256 sum0 = _mm256_setzero_ps(); - __m256 sum1 = _mm256_setzero_ps(); - __m256 sum2 = _mm256_setzero_ps(); - __m256 sum3 = _mm256_setzero_ps(); - int32_t i = 0; - for (; i + 31 < n; i += 32) { - __m256 x0 = _mm256_loadu_ps(x + i); - __m256 x1 = _mm256_loadu_ps(x + i + 8); - __m256 x2 = _mm256_loadu_ps(x + i + 16); - __m256 x3 = _mm256_loadu_ps(x + i + 24); - sum0 = _mm256_add_ps(sum0, _mm256_mul_ps(x0, x0)); - sum1 = _mm256_add_ps(sum1, _mm256_mul_ps(x1, x1)); - sum2 = _mm256_add_ps(sum2, _mm256_mul_ps(x2, x2)); - sum3 = _mm256_add_ps(sum3, _mm256_mul_ps(x3, x3)); - } - float sum = ei_hsum_f32x8(_mm256_add_ps(_mm256_add_ps(sum0, sum1), - _mm256_add_ps(sum2, sum3))); - for (; i < n; i++) sum += x[i] * x[i]; - return sum; #elif defined(__SSE2__) __m128 sum0 = _mm_setzero_ps(); __m128 sum1 = _mm_setzero_ps(); @@ -101,6 +78,7 @@ static float ei_sum_squares(const float *x, int32_t n) { static void ei_norm_scale(const float *x, const float *w, int32_t n, float scale, float *out) { + EI_DISPATCH_AVX2(ei_avx2_norm_scale(x, w, n, scale, out); return); #if defined(__ARM_NEON) && defined(__aarch64__) const float32x4_t scale4 = vdupq_n_f32(scale); int32_t i = 0; @@ -111,16 +89,6 @@ static void ei_norm_scale(const float *x, const float *w, int32_t n, vst1q_f32(out + i + 12, vmulq_f32(vmulq_f32(vld1q_f32(x + i + 12), vld1q_f32(w + i + 12)), scale4)); } for (; i < n; i++) out[i] = x[i] * scale * w[i]; -#elif defined(__AVX2__) - const __m256 scale8 = _mm256_set1_ps(scale); - int32_t i = 0; - for (; i + 15 < n; i += 16) { - _mm256_storeu_ps(out + i, _mm256_mul_ps(_mm256_mul_ps(_mm256_loadu_ps(x + i), - _mm256_loadu_ps(w + i)), scale8)); - _mm256_storeu_ps(out + i + 8, _mm256_mul_ps(_mm256_mul_ps(_mm256_loadu_ps(x + i + 8), - _mm256_loadu_ps(w + i + 8)), scale8)); - } - for (; i < n; i++) out[i] = x[i] * scale * w[i]; #elif defined(__SSE2__) const __m128 scale4 = _mm_set1_ps(scale); int32_t i = 0; @@ -163,6 +131,7 @@ void ei_rms_norm_inplace(float *x, const float *w, int32_t n, float eps) { void ei_rms_norm_residual_inplace(float *residual, const float *x, const float *w, int32_t n, float eps) { const float scale = 1.0f / sqrtf(ei_sum_squares(x, n) / (float)n + eps); + EI_DISPATCH_AVX2(ei_avx2_norm_residual(residual, x, w, n, scale); return); #if defined(__ARM_NEON) && defined(__aarch64__) const float32x4_t scale4 = vdupq_n_f32(scale); int32_t i = 0; @@ -173,15 +142,6 @@ void ei_rms_norm_residual_inplace(float *residual, const float *x, const float * vst1q_f32(residual + i + 4, vaddq_f32(vld1q_f32(residual + i + 4), y1)); } for (; i < n; i++) residual[i] += x[i] * scale * w[i]; -#elif defined(__AVX2__) - const __m256 scale8 = _mm256_set1_ps(scale); - int32_t i = 0; - for (; i + 7 < n; i += 8) { - __m256 y = _mm256_mul_ps(_mm256_mul_ps(_mm256_loadu_ps(x + i), - _mm256_loadu_ps(w + i)), scale8); - _mm256_storeu_ps(residual + i, _mm256_add_ps(_mm256_loadu_ps(residual + i), y)); - } - for (; i < n; i++) residual[i] += x[i] * scale * w[i]; #else for (int32_t i = 0; i < n; i++) residual[i] += x[i] * scale * w[i]; #endif @@ -203,6 +163,7 @@ void ei_rope_neox_inplace(float *x, int32_t n_heads, int32_t pos, float base) { } static void ei_vec_scale_inplace(float *x, float scale, int32_t n) { + EI_DISPATCH_AVX2(ei_avx2_vec_scale_inplace(x, scale, n); return); #if defined(__ARM_NEON) && defined(__aarch64__) const float32x4_t scale4 = vdupq_n_f32(scale); int32_t i = 0; @@ -211,11 +172,6 @@ static void ei_vec_scale_inplace(float *x, float scale, int32_t n) { vst1q_f32(x + i + 4, vmulq_f32(vld1q_f32(x + i + 4), scale4)); } for (; i < n; i++) x[i] *= scale; -#elif defined(__AVX2__) - const __m256 scale8 = _mm256_set1_ps(scale); - int32_t i = 0; - for (; i + 7 < n; i += 8) _mm256_storeu_ps(x + i, _mm256_mul_ps(_mm256_loadu_ps(x + i), scale8)); - for (; i < n; i++) x[i] *= scale; #else for (int32_t i = 0; i < n; i++) x[i] *= scale; #endif @@ -264,6 +220,7 @@ void ei_qk_norm_rope_qk_inplace(float *q, float *k, const float *q_weight, } static float ei_dot_f32(const float *a, const float *b, int32_t n) { + EI_DISPATCH_AVX2(return ei_avx2_dot_f32(a, b, n)); #if defined(__ARM_NEON) && defined(__aarch64__) float32x4_t sum0 = vdupq_n_f32(0.0f); float32x4_t sum1 = vdupq_n_f32(0.0f); @@ -275,17 +232,6 @@ static float ei_dot_f32(const float *a, const float *b, int32_t n) { float sum = vaddvq_f32(vaddq_f32(sum0, sum1)); for (; i < n; i++) sum += a[i] * b[i]; return sum; -#elif defined(__AVX2__) - __m256 sum0 = _mm256_setzero_ps(); - __m256 sum1 = _mm256_setzero_ps(); - int32_t i = 0; - for (; i + 15 < n; i += 16) { - sum0 = _mm256_add_ps(sum0, _mm256_mul_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i))); - sum1 = _mm256_add_ps(sum1, _mm256_mul_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8))); - } - float sum = ei_hsum_f32x8(_mm256_add_ps(sum0, sum1)); - for (; i < n; i++) sum += a[i] * b[i]; - return sum; #elif defined(__SSE2__) __m128 sum = _mm_setzero_ps(); int32_t i = 0; @@ -301,6 +247,7 @@ static float ei_dot_f32(const float *a, const float *b, int32_t n) { } static void ei_axpy(float *dst, const float *src, float scale, int32_t n) { + EI_DISPATCH_AVX2(ei_avx2_axpy(dst, src, scale, n); return); #if defined(__ARM_NEON) && defined(__aarch64__) int32_t i = 0; for (; i + 7 < n; i += 8) { @@ -308,14 +255,6 @@ static void ei_axpy(float *dst, const float *src, float scale, int32_t n) { vst1q_f32(dst + i + 4, vfmaq_n_f32(vld1q_f32(dst + i + 4), vld1q_f32(src + i + 4), scale)); } for (; i < n; i++) dst[i] += scale * src[i]; -#elif defined(__AVX2__) - const __m256 scale8 = _mm256_set1_ps(scale); - int32_t i = 0; - for (; i + 7 < n; i += 8) { - __m256 add = _mm256_mul_ps(_mm256_loadu_ps(src + i), scale8); - _mm256_storeu_ps(dst + i, _mm256_add_ps(_mm256_loadu_ps(dst + i), add)); - } - for (; i < n; i++) dst[i] += scale * src[i]; #else for (int32_t i = 0; i < n; i++) dst[i] += scale * src[i]; #endif diff --git a/src/kernels_avx2.c b/src/kernels_avx2.c new file mode 100644 index 0000000..8184fbb --- /dev/null +++ b/src/kernels_avx2.c @@ -0,0 +1,306 @@ +/* AVX2/FMA/F16C CPU kernels and x86-64 runtime ISA selection. + * + * This is the only translation unit with AVX code. Every kernel below carries + * EI_AVX2_TARGET rather than relying on -mavx2, so the Makefile keeps one + * baseline CFLAGS for every file and nothing outside these functions (headers, + * static inline helpers, initializers) is compiled for AVX2. Callers in + * kernels.c and quants.c reach these only after ei_cpu_avx2_active(). */ +#include "kernels_avx2.h" + +#include + +#if defined(EI_X86_DISPATCH) +#include +#include + +_Atomic int ei_cpu_isa_state = -1; + +/* AVX2, FMA and F16C in CPUID, and YMM state enabled by the OS (XCR0). */ +static bool ei_cpu_detect_avx2(void) { + unsigned eax, ebx, ecx, edx; + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx)) return false; + const unsigned fma = 1u << 12, osxsave = 1u << 27, avx = 1u << 28, f16c = 1u << 29; + if ((ecx & (fma | osxsave | avx | f16c)) != (fma | osxsave | avx | f16c)) return false; + unsigned xcr0_lo, xcr0_hi; + __asm__ volatile("xgetbv" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); + if ((xcr0_lo & 6u) != 6u) return false; /* XMM and YMM state */ + if (!__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx)) return false; + return (ebx & (1u << 5)) != 0; /* AVX2 */ +} + +bool ei_cpu_avx2_supported(void) { + return ei_cpu_detect_avx2(); +} + +static int ei_cpu_isa_from_name(const char *isa) { + if (!isa || !*isa || strcmp(isa, "auto") == 0) return ei_cpu_detect_avx2() ? 1 : 0; + if (strcmp(isa, "baseline") == 0 || strcmp(isa, "sse2") == 0) return 0; + if (strcmp(isa, "avx2") == 0) return ei_cpu_detect_avx2() ? 1 : -1; + return -1; +} + +int ei_cpu_resolve_isa(void) { + const char *env = getenv("EI_CPU_ISA"); + int state = ei_cpu_isa_from_name(env); + if (state < 0) { + ei_die("EI_CPU_ISA=%s is not supported here (use auto, baseline, or avx2 " + "on an AVX2+FMA+F16C CPU)", env); + } + atomic_store_explicit(&ei_cpu_isa_state, state, memory_order_relaxed); + return state; +} + +bool ei_cpu_set_isa(const char *isa) { + int state = ei_cpu_isa_from_name(isa); + if (state < 0) return false; + atomic_store_explicit(&ei_cpu_isa_state, state, memory_order_relaxed); + return true; +} + +#define EI_AVX2_TARGET __attribute__((target("avx2,fma,f16c"))) + +/* Hardware half -> float; exact, so it matches ei_fp16_to_fp32 bit for bit. */ +EI_AVX2_TARGET static inline float ei_avx2_fp16(ei_fp16 h) { + return _cvtsh_ss(h); +} + +EI_AVX2_TARGET static inline float ei_avx2_hsum_f32x8(__m256 v) { + __m128 sum = _mm_add_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps(v, 1)); + sum = _mm_add_ps(sum, _mm_movehl_ps(sum, sum)); + sum = _mm_add_ss(sum, _mm_shuffle_ps(sum, sum, 1)); + return _mm_cvtss_f32(sum); +} + +EI_AVX2_TARGET float ei_avx2_sum_squares(const float *x, int32_t n) { + __m256 sum0 = _mm256_setzero_ps(); + __m256 sum1 = _mm256_setzero_ps(); + __m256 sum2 = _mm256_setzero_ps(); + __m256 sum3 = _mm256_setzero_ps(); + int32_t i = 0; + for (; i + 31 < n; i += 32) { + __m256 x0 = _mm256_loadu_ps(x + i); + __m256 x1 = _mm256_loadu_ps(x + i + 8); + __m256 x2 = _mm256_loadu_ps(x + i + 16); + __m256 x3 = _mm256_loadu_ps(x + i + 24); + sum0 = _mm256_add_ps(sum0, _mm256_mul_ps(x0, x0)); + sum1 = _mm256_add_ps(sum1, _mm256_mul_ps(x1, x1)); + sum2 = _mm256_add_ps(sum2, _mm256_mul_ps(x2, x2)); + sum3 = _mm256_add_ps(sum3, _mm256_mul_ps(x3, x3)); + } + float sum = ei_avx2_hsum_f32x8(_mm256_add_ps(_mm256_add_ps(sum0, sum1), + _mm256_add_ps(sum2, sum3))); + for (; i < n; i++) sum += x[i] * x[i]; + return sum; +} + +EI_AVX2_TARGET void ei_avx2_norm_scale(const float *x, const float *w, int32_t n, + float scale, float *out) { + const __m256 scale8 = _mm256_set1_ps(scale); + int32_t i = 0; + for (; i + 15 < n; i += 16) { + _mm256_storeu_ps(out + i, _mm256_mul_ps(_mm256_mul_ps(_mm256_loadu_ps(x + i), + _mm256_loadu_ps(w + i)), scale8)); + _mm256_storeu_ps(out + i + 8, _mm256_mul_ps(_mm256_mul_ps(_mm256_loadu_ps(x + i + 8), + _mm256_loadu_ps(w + i + 8)), scale8)); + } + for (; i < n; i++) out[i] = x[i] * scale * w[i]; +} + +EI_AVX2_TARGET void ei_avx2_norm_residual(float *residual, const float *x, const float *w, + int32_t n, float scale) { + const __m256 scale8 = _mm256_set1_ps(scale); + int32_t i = 0; + for (; i + 7 < n; i += 8) { + __m256 y = _mm256_mul_ps(_mm256_mul_ps(_mm256_loadu_ps(x + i), + _mm256_loadu_ps(w + i)), scale8); + _mm256_storeu_ps(residual + i, _mm256_add_ps(_mm256_loadu_ps(residual + i), y)); + } + for (; i < n; i++) residual[i] += x[i] * scale * w[i]; +} + +EI_AVX2_TARGET void ei_avx2_vec_scale_inplace(float *x, float scale, int32_t n) { + const __m256 scale8 = _mm256_set1_ps(scale); + int32_t i = 0; + for (; i + 7 < n; i += 8) _mm256_storeu_ps(x + i, _mm256_mul_ps(_mm256_loadu_ps(x + i), scale8)); + for (; i < n; i++) x[i] *= scale; +} + +EI_AVX2_TARGET float ei_avx2_dot_f32(const float *a, const float *b, int32_t n) { + __m256 sum0 = _mm256_setzero_ps(); + __m256 sum1 = _mm256_setzero_ps(); + int32_t i = 0; + for (; i + 15 < n; i += 16) { + sum0 = _mm256_add_ps(sum0, _mm256_mul_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i))); + sum1 = _mm256_add_ps(sum1, _mm256_mul_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8))); + } + float sum = ei_avx2_hsum_f32x8(_mm256_add_ps(sum0, sum1)); + for (; i < n; i++) sum += a[i] * b[i]; + return sum; +} + +EI_AVX2_TARGET void ei_avx2_axpy(float *dst, const float *src, float scale, int32_t n) { + const __m256 scale8 = _mm256_set1_ps(scale); + int32_t i = 0; + for (; i + 7 < n; i += 8) { + __m256 add = _mm256_mul_ps(_mm256_loadu_ps(src + i), scale8); + _mm256_storeu_ps(dst + i, _mm256_add_ps(_mm256_loadu_ps(dst + i), add)); + } + for (; i < n; i++) dst[i] += scale * src[i]; +} + +EI_AVX2_TARGET void ei_avx2_dequantize_row_q8_0(const ei_block_q8_0 *blocks, + uint64_t row_blocks, float scale, + float *out) { + for (uint64_t b = 0; b < row_blocks; b++) { + const __m256 d8 = _mm256_set1_ps(ei_avx2_fp16(blocks[b].d) * scale); + for (int j = 0; j < EI_QK; j += 8) { + const __m128i q8 = _mm_loadl_epi64((const __m128i *)(blocks[b].qs + j)); + const __m256i q32 = _mm256_cvtepi8_epi32(q8); + _mm256_storeu_ps(out + b * EI_QK + (uint64_t)j, + _mm256_mul_ps(_mm256_cvtepi32_ps(q32), d8)); + } + } +} + +EI_AVX2_TARGET void ei_avx2_quantize_row_q8_0(const float *x, ei_block_q8_0 *out, + int32_t nb) { + const __m256 abs_mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffff)); + const __m256 half = _mm256_set1_ps(0.5f); + const __m256 sign = _mm256_set1_ps(-0.0f); + const __m256i order = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); + for (int32_t b = 0; b < nb; b++) { + const float *xb = x + b * EI_QK; + __m256 maxv = _mm256_setzero_ps(); + for (int j = 0; j < EI_QK; j += 8) { + maxv = _mm256_max_ps(maxv, _mm256_and_ps(_mm256_loadu_ps(xb + j), abs_mask)); + } + __m128 max4 = _mm_max_ps(_mm256_castps256_ps128(maxv), _mm256_extractf128_ps(maxv, 1)); + max4 = _mm_max_ps(max4, _mm_movehl_ps(max4, max4)); + max4 = _mm_max_ss(max4, _mm_shuffle_ps(max4, max4, 1)); + const float amax = _mm_cvtss_f32(max4); + const float d = amax / 127.0f; + out[b].d = ei_fp32_to_fp16(d); + if (amax == 0.0f) { + memset(out[b].qs, 0, sizeof out[b].qs); + continue; + } + const __m256 scale = _mm256_set1_ps(1.0f / d); + __m256i quants[4]; + for (int j = 0; j < 4; j++) { + __m256 values = _mm256_mul_ps(_mm256_loadu_ps(xb + j * 8), scale); + __m256 signed_half = _mm256_or_ps(_mm256_and_ps(values, sign), half); + quants[j] = _mm256_cvttps_epi32(_mm256_add_ps(values, signed_half)); + } + __m256i q01 = _mm256_packs_epi32(quants[0], quants[1]); + __m256i q23 = _mm256_packs_epi32(quants[2], quants[3]); + __m256i q8 = _mm256_packs_epi16(q01, q23); + _mm256_storeu_si256((__m256i *)out[b].qs, _mm256_permutevar8x32_epi32(q8, order)); + } +} + +/* Signed 4-bit weights (low nibbles in lanes 0-15, high nibbles in 16-31) for + * one Q4_0 block, matching the Q8_0 activation layout. */ +EI_AVX2_TARGET static inline __m256i ei_avx2_unpack_q4_0(const ei_block_q4_0 *block) { + const __m128i packed = _mm_loadu_si128((const __m128i *)block->qs); + const __m256i q4 = _mm256_inserti128_si256( + _mm256_castsi128_si256(packed), _mm_srli_epi16(packed, 4), 1); + return _mm256_sub_epi8(_mm256_and_si256(q4, _mm256_set1_epi8(0x0F)), + _mm256_set1_epi8(8)); +} + +/* Eight int32 partial sums of the 32 int8 products, as floats. */ +EI_AVX2_TARGET static inline __m256 ei_avx2_dot_i8x32(__m256i x, __m256i y) { + const __m256i dot16 = _mm256_maddubs_epi16(_mm256_sign_epi8(x, x), + _mm256_sign_epi8(y, x)); + return _mm256_cvtepi32_ps(_mm256_madd_epi16(dot16, _mm256_set1_epi16(1))); +} + +EI_AVX2_TARGET float ei_avx2_vec_dot_q4_0_q8_0(const ei_block_q4_0 *x, + const ei_block_q8_0 *y, int32_t nb) { + __m256 sum = _mm256_setzero_ps(); + for (int32_t b = 0; b < nb; b++) { + const __m256i yq = _mm256_loadu_si256((const __m256i *)y[b].qs); + const __m256 dot = ei_avx2_dot_i8x32(ei_avx2_unpack_q4_0(x + b), yq); + const __m256 scale = _mm256_set1_ps(ei_avx2_fp16(x[b].d) * + ei_avx2_fp16(y[b].d)); + sum = _mm256_fmadd_ps(dot, scale, sum); + } + return ei_avx2_hsum_f32x8(sum); +} + +EI_AVX2_TARGET void ei_avx2_vec_dot_q4_0_q8_0_batch4(const ei_block_q4_0 *weights, + const ei_block_q8_0 *inputs, + int32_t input_stride, int32_t count, + int32_t nb, float out[4]) { + __m256 sums[4] = { + _mm256_setzero_ps(), _mm256_setzero_ps(), + _mm256_setzero_ps(), _mm256_setzero_ps(), + }; + for (int32_t block = 0; block < nb; block++) { + const __m256i weight_values = ei_avx2_unpack_q4_0(weights + block); + const float weight_scale = ei_avx2_fp16(weights[block].d); + for (int32_t input = 0; input < count; input++) { + const ei_block_q8_0 *activation = inputs + input * input_stride + block; + const __m256i values = _mm256_loadu_si256((const __m256i *)activation->qs); + const __m256 scale = _mm256_set1_ps( + weight_scale * ei_avx2_fp16(activation->d)); + sums[input] = _mm256_fmadd_ps(ei_avx2_dot_i8x32(weight_values, values), + scale, sums[input]); + } + } + for (int32_t input = 0; input < count; input++) out[input] = ei_avx2_hsum_f32x8(sums[input]); +} + +EI_AVX2_TARGET void ei_avx2_vec_dot_q4_0_q8_0_dual(const ei_block_q4_0 *x0, + const ei_block_q4_0 *x1, + const ei_block_q8_0 *y, int32_t nb, + float *out0, float *out1) { + __m256 sum0 = _mm256_setzero_ps(); + __m256 sum1 = _mm256_setzero_ps(); + for (int32_t b = 0; b < nb; b++) { + const __m256i yq = _mm256_loadu_si256((const __m256i *)y[b].qs); + const float y_scale = ei_avx2_fp16(y[b].d); + sum0 = _mm256_fmadd_ps(ei_avx2_dot_i8x32(ei_avx2_unpack_q4_0(x0 + b), yq), + _mm256_set1_ps(ei_avx2_fp16(x0[b].d) * y_scale), sum0); + sum1 = _mm256_fmadd_ps(ei_avx2_dot_i8x32(ei_avx2_unpack_q4_0(x1 + b), yq), + _mm256_set1_ps(ei_avx2_fp16(x1[b].d) * y_scale), sum1); + } + *out0 = ei_avx2_hsum_f32x8(sum0); + *out1 = ei_avx2_hsum_f32x8(sum1); +} + +EI_AVX2_TARGET void ei_avx2_vec_dot_q4_0_q8_0_triple(const ei_block_q4_0 *x0, + const ei_block_q4_0 *x1, + const ei_block_q4_0 *x2, + const ei_block_q8_0 *y, int32_t nb, + float *out0, float *out1, + float *out2) { + __m256 sum0 = _mm256_setzero_ps(); + __m256 sum1 = _mm256_setzero_ps(); + __m256 sum2 = _mm256_setzero_ps(); + for (int32_t b = 0; b < nb; b++) { + const __m256i yq = _mm256_loadu_si256((const __m256i *)y[b].qs); + const float y_scale = ei_avx2_fp16(y[b].d); + sum0 = _mm256_fmadd_ps(ei_avx2_dot_i8x32(ei_avx2_unpack_q4_0(x0 + b), yq), + _mm256_set1_ps(ei_avx2_fp16(x0[b].d) * y_scale), sum0); + sum1 = _mm256_fmadd_ps(ei_avx2_dot_i8x32(ei_avx2_unpack_q4_0(x1 + b), yq), + _mm256_set1_ps(ei_avx2_fp16(x1[b].d) * y_scale), sum1); + sum2 = _mm256_fmadd_ps(ei_avx2_dot_i8x32(ei_avx2_unpack_q4_0(x2 + b), yq), + _mm256_set1_ps(ei_avx2_fp16(x2[b].d) * y_scale), sum2); + } + *out0 = ei_avx2_hsum_f32x8(sum0); + *out1 = ei_avx2_hsum_f32x8(sum1); + *out2 = ei_avx2_hsum_f32x8(sum2); +} + +#else /* !EI_X86_DISPATCH: NEON and other targets keep compile-time kernels. */ + +bool ei_cpu_avx2_supported(void) { + return false; +} + +bool ei_cpu_set_isa(const char *isa) { + return !isa || !*isa || strcmp(isa, "auto") == 0 || strcmp(isa, "baseline") == 0; +} + +#endif diff --git a/src/kernels_avx2.h b/src/kernels_avx2.h new file mode 100644 index 0000000..ee59f9b --- /dev/null +++ b/src/kernels_avx2.h @@ -0,0 +1,70 @@ +/* x86-64 runtime CPU dispatch for the AVX2/FMA/F16C CPU kernels. + * + * Portable x86-64 builds use the compiler baseline (SSE2). The AVX2 kernels + * live in kernels_avx2.c, where each function carries + * __attribute__((target("avx2,fma,f16c"))), so no other code is compiled with + * AVX instructions. Callers reach them only through ei_cpu_avx2_active(). */ +#ifndef EI_KERNELS_AVX2_H +#define EI_KERNELS_AVX2_H + +#include "quants.h" + +#include + +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) +#define EI_X86_DISPATCH 1 + +/* -1 until first use, then 0 (baseline) or 1 (AVX2 + FMA + F16C). */ +extern _Atomic int ei_cpu_isa_state; +int ei_cpu_resolve_isa(void); + +/* True when the CPU and OS support AVX2, FMA and F16C, unless EI_CPU_ISA=baseline + * (or a test) forced the SSE2 route. A relaxed load: cheap per kernel call. */ +static inline bool ei_cpu_avx2_active(void) { + int state = atomic_load_explicit(&ei_cpu_isa_state, memory_order_relaxed); + if (state < 0) state = ei_cpu_resolve_isa(); + return state == 1; +} + +float ei_avx2_sum_squares(const float *x, int32_t n); +void ei_avx2_norm_scale(const float *x, const float *w, int32_t n, + float scale, float *out); +void ei_avx2_norm_residual(float *residual, const float *x, const float *w, + int32_t n, float scale); +void ei_avx2_vec_scale_inplace(float *x, float scale, int32_t n); +float ei_avx2_dot_f32(const float *a, const float *b, int32_t n); +void ei_avx2_axpy(float *dst, const float *src, float scale, int32_t n); + +void ei_avx2_dequantize_row_q8_0(const ei_block_q8_0 *blocks, uint64_t row_blocks, + float scale, float *out); +void ei_avx2_quantize_row_q8_0(const float *x, ei_block_q8_0 *out, int32_t nb); +float ei_avx2_vec_dot_q4_0_q8_0(const ei_block_q4_0 *x, const ei_block_q8_0 *y, + int32_t nb); +void ei_avx2_vec_dot_q4_0_q8_0_batch4(const ei_block_q4_0 *weights, + const ei_block_q8_0 *inputs, + int32_t input_stride, int32_t count, + int32_t nb, float out[4]); +void ei_avx2_vec_dot_q4_0_q8_0_dual(const ei_block_q4_0 *x0, const ei_block_q4_0 *x1, + const ei_block_q8_0 *y, int32_t nb, + float *out0, float *out1); +void ei_avx2_vec_dot_q4_0_q8_0_triple(const ei_block_q4_0 *x0, const ei_block_q4_0 *x1, + const ei_block_q4_0 *x2, const ei_block_q8_0 *y, + int32_t nb, float *out0, float *out1, + float *out2); + +/* Runs `call` (which must return) when the AVX2 route is active. */ +#define EI_DISPATCH_AVX2(call) do { if (ei_cpu_avx2_active()) { call; } } while (0) +#else +#define EI_DISPATCH_AVX2(call) do { } while (0) +#endif + +/* Select the x86 kernel family: "auto" (default), "baseline" ("sse2"), or + * "avx2". Returns false for an unknown name or AVX2 on a CPU without it. + * Non-x86 builds accept only "auto" and "baseline". On x86-64 the EI_CPU_ISA + * environment variable sets the initial choice. Not for use while kernels run + * on other threads. */ +bool ei_cpu_set_isa(const char *isa); +/* True when this process can run the AVX2 route (x86-64 with AVX2, FMA and F16C). */ +bool ei_cpu_avx2_supported(void); + +#endif /* EI_KERNELS_AVX2_H */ diff --git a/src/quants.c b/src/quants.c index 7716c5a..2fcb97f 100644 --- a/src/quants.c +++ b/src/quants.c @@ -1,4 +1,5 @@ #include "quants.h" +#include "kernels_avx2.h" #include @@ -6,10 +7,13 @@ #include #endif -#if defined(__AVX2__) || defined(__SSSE3__) -#include +#if defined(__SSSE3__) +#include #endif +/* On x86-64, EI_DISPATCH_AVX2 hands off to kernels_avx2.c at runtime; the + * inline paths below are the NEON, SSSE3, and scalar routes. */ + ei_fp16 ei_fp32_to_fp16(float f) { uint32_t x; memcpy(&x, &f, sizeof x); @@ -37,14 +41,15 @@ ei_fp16 ei_fp32_to_fp16(float f) { } const char *ei_quants_kernel_variant(void) { +#if defined(EI_X86_DISPATCH) + if (ei_cpu_avx2_active()) return "cpu-avx2"; +#endif #if defined(__ARM_NEON) && defined(__aarch64__) #if defined(__ARM_FEATURE_DOTPROD) return "cpu-neon-dotprod"; #else return "cpu-neon"; #endif -#elif defined(__AVX2__) - return "cpu-avx2"; #elif defined(__SSSE3__) return "cpu-ssse3"; #else @@ -63,6 +68,7 @@ void ei_dequantize_row_q8_0_scaled(const ei_tensor *t, int32_t row, float scale, uint64_t row_blocks = t->ne[0] / EI_QK; const ei_block_q8_0 *blocks = (const ei_block_q8_0 *)((const uint8_t *)t->data + (uint64_t)row * ei_tensor_row_bytes(t)); + EI_DISPATCH_AVX2(ei_avx2_dequantize_row_q8_0(blocks, row_blocks, scale, out); return); for (uint64_t b = 0; b < row_blocks; b++) { float d = ei_fp16_to_fp32(blocks[b].d) * scale; #if defined(__ARM_NEON) && defined(__aarch64__) @@ -74,14 +80,6 @@ void ei_dequantize_row_q8_0_scaled(const ei_tensor *t, int32_t row, float scale, vst1q_f32(out + b * EI_QK + (uint64_t)j, vmulq_f32(q0, d4)); vst1q_f32(out + b * EI_QK + (uint64_t)j + 4, vmulq_f32(q1, d4)); } -#elif defined(__AVX2__) - const __m256 d8 = _mm256_set1_ps(d); - for (int j = 0; j < EI_QK; j += 8) { - const __m128i q8 = _mm_loadl_epi64((const __m128i *)(blocks[b].qs + j)); - const __m256i q32 = _mm256_cvtepi8_epi32(q8); - _mm256_storeu_ps(out + b * EI_QK + (uint64_t)j, - _mm256_mul_ps(_mm256_cvtepi32_ps(q32), d8)); - } #else for (int j = 0; j < EI_QK; j++) { out[b * EI_QK + (uint64_t)j] = d * (float)blocks[b].qs[j]; @@ -98,6 +96,7 @@ void ei_quantize_row_q8_0(const float *x, ei_block_q8_0 *out, int32_t n) { if (n % EI_QK != 0) ei_die("q8_0 quantize length %d is not a multiple of 32", n); int32_t nb = n / EI_QK; + EI_DISPATCH_AVX2(ei_avx2_quantize_row_q8_0(x, out, nb); return); for (int32_t b = 0; b < nb; b++) { const float *xb = x + b * EI_QK; #if defined(__ARM_NEON) && defined(__aarch64__) @@ -108,16 +107,6 @@ void ei_quantize_row_q8_0(const float *x, ei_block_q8_0 *out, int32_t n) { max1 = vmaxq_f32(max1, vabsq_f32(vld1q_f32(xb + j + 4))); } float amax = vmaxvq_f32(vmaxq_f32(max0, max1)); -#elif defined(__AVX2__) - const __m256 abs_mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffff)); - __m256 maxv = _mm256_setzero_ps(); - for (int j = 0; j < EI_QK; j += 8) { - maxv = _mm256_max_ps(maxv, _mm256_and_ps(_mm256_loadu_ps(xb + j), abs_mask)); - } - __m128 max4 = _mm_max_ps(_mm256_castps256_ps128(maxv), _mm256_extractf128_ps(maxv, 1)); - max4 = _mm_max_ps(max4, _mm_movehl_ps(max4, max4)); - max4 = _mm_max_ss(max4, _mm_shuffle_ps(max4, max4, 1)); - float amax = _mm_cvtss_f32(max4); #else float amax = 0.0f; for (int j = 0; j < EI_QK; j++) { @@ -148,23 +137,6 @@ void ei_quantize_row_q8_0(const float *x, ei_block_q8_0 *out, int32_t n) { const int16x8_t q67 = vcombine_s16(vqmovn_s32(q6), vqmovn_s32(q7)); vst1q_s8(out[b].qs, vcombine_s8(vqmovn_s16(q01), vqmovn_s16(q23))); vst1q_s8(out[b].qs + 16, vcombine_s8(vqmovn_s16(q45), vqmovn_s16(q67))); -#elif defined(__AVX2__) - const __m256 scale = _mm256_set1_ps(id); - const __m256 half = _mm256_set1_ps(0.5f); - const __m256 sign = _mm256_set1_ps(-0.0f); - __m256 values[4]; - __m256i quants[4]; - for (int j = 0; j < 4; j++) { - values[j] = _mm256_mul_ps(_mm256_loadu_ps(xb + j * 8), scale); - __m256 signed_half = _mm256_or_ps(_mm256_and_ps(values[j], sign), half); - quants[j] = _mm256_cvttps_epi32(_mm256_add_ps(values[j], signed_half)); - } - __m256i q01 = _mm256_packs_epi32(quants[0], quants[1]); - __m256i q23 = _mm256_packs_epi32(quants[2], quants[3]); - __m256i q8 = _mm256_packs_epi16(q01, q23); - const __m256i order = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); - _mm256_storeu_si256((__m256i *)out[b].qs, - _mm256_permutevar8x32_epi32(q8, order)); #else for (int j = 0; j < EI_QK; j++) { int q = (int)roundf(xb[j] * id); @@ -210,56 +182,20 @@ static float ei_vec_dot_q4_0_q8_0_neon(const ei_block_q4_0 *x, } #endif -#if defined(__AVX2__) || defined(__SSSE3__) +#if defined(__SSSE3__) static inline float ei_hsum_f32x4(__m128 v) { v = _mm_add_ps(v, _mm_movehl_ps(v, v)); v = _mm_add_ss(v, _mm_shuffle_ps(v, v, 1)); return _mm_cvtss_f32(v); } -#if !defined(__AVX2__) static inline __m128i ei_mul_sum_i8_pairs_sse(__m128i x, __m128i y) { const __m128i ax = _mm_sign_epi8(x, x); const __m128i sy = _mm_sign_epi8(y, x); const __m128i dot16 = _mm_maddubs_epi16(ax, sy); return _mm_madd_epi16(dot16, _mm_set1_epi16(1)); } -#endif -#if defined(__AVX2__) -static inline float ei_hsum_f32x8(__m256 v) { - return ei_hsum_f32x4(_mm_add_ps(_mm256_castps256_ps128(v), - _mm256_extractf128_ps(v, 1))); -} - -static float ei_vec_dot_q4_0_q8_0_avx2(const ei_block_q4_0 *x, - const ei_block_q8_0 *y, - int32_t nb) { - const __m256i low_mask = _mm256_set1_epi8(0x0F); - const __m256i off = _mm256_set1_epi8(8); - const __m256i ones = _mm256_set1_epi16(1); - __m256 sum = _mm256_setzero_ps(); - - for (int32_t b = 0; b < nb; b++) { - const __m128i q4_128 = _mm_loadu_si128((const __m128i *)x[b].qs); - const __m256i q4 = _mm256_inserti128_si256( - _mm256_castsi128_si256(q4_128), _mm_srli_epi16(q4_128, 4), 1); - const __m256i xq = _mm256_sub_epi8(_mm256_and_si256(q4, low_mask), off); - const __m256i yq = _mm256_loadu_si256((const __m256i *)y[b].qs); - const __m256i ax = _mm256_sign_epi8(xq, xq); - const __m256i sy = _mm256_sign_epi8(yq, xq); - const __m256i dot16 = _mm256_maddubs_epi16(ax, sy); - const __m256 dot = _mm256_cvtepi32_ps(_mm256_madd_epi16(dot16, ones)); - const __m256 scale = _mm256_set1_ps(ei_fp16_to_fp32(x[b].d) * - ei_fp16_to_fp32(y[b].d)); - sum = _mm256_add_ps(sum, _mm256_mul_ps(dot, scale)); - } - - return ei_hsum_f32x8(sum); -} -#endif - -#if !defined(__AVX2__) static float ei_vec_dot_q4_0_q8_0_ssse3(const ei_block_q4_0 *x, const ei_block_q8_0 *y, int32_t nb) { @@ -283,15 +219,13 @@ static float ei_vec_dot_q4_0_q8_0_ssse3(const ei_block_q4_0 *x, return ei_hsum_f32x4(sum); } #endif -#endif float ei_vec_dot_q4_0_q8_0(const ei_block_q4_0 *x, const ei_block_q8_0 *y, int32_t n) { if (n % EI_QK != 0) ei_die("q4_0 dot length %d is not a multiple of 32", n); int32_t nb = n / EI_QK; + EI_DISPATCH_AVX2(return ei_avx2_vec_dot_q4_0_q8_0(x, y, nb)); #if defined(__ARM_NEON) && defined(__aarch64__) return ei_vec_dot_q4_0_q8_0_neon(x, y, nb); -#elif defined(__AVX2__) - return ei_vec_dot_q4_0_q8_0_avx2(x, y, nb); #elif defined(__SSSE3__) return ei_vec_dot_q4_0_q8_0_ssse3(x, y, nb); #else @@ -314,6 +248,8 @@ static void ei_vec_dot_q4_0_q8_0_batch4(const ei_block_q4_0 *weights, const ei_block_q8_0 *inputs, int32_t input_stride, int32_t count, int32_t n, float out[4]) { + EI_DISPATCH_AVX2(ei_avx2_vec_dot_q4_0_q8_0_batch4(weights, inputs, input_stride, + count, n / EI_QK, out); return); #if defined(__ARM_NEON) && defined(__aarch64__) int32_t nb = n / EI_QK; const uint8x16_t low_mask = vdupq_n_u8(0x0Fu); @@ -340,37 +276,6 @@ static void ei_vec_dot_q4_0_q8_0_batch4(const ei_block_q4_0 *weights, } } for (int32_t input = 0; input < count; input++) out[input] = vaddvq_f32(sums[input]); -#elif defined(__AVX2__) - int32_t nb = n / EI_QK; - const __m256i low_mask = _mm256_set1_epi8(0x0F); - const __m256i off = _mm256_set1_epi8(8); - const __m256i ones = _mm256_set1_epi16(1); - __m256 sums[4] = { - _mm256_setzero_ps(), _mm256_setzero_ps(), - _mm256_setzero_ps(), _mm256_setzero_ps(), - }; - for (int32_t block = 0; block < nb; block++) { - const __m128i packed = _mm_loadu_si128((const __m128i *)weights[block].qs); - const __m256i q4 = _mm256_inserti128_si256( - _mm256_castsi128_si256(packed), _mm_srli_epi16(packed, 4), 1); - const __m256i weight_values = _mm256_sub_epi8( - _mm256_and_si256(q4, low_mask), off); - const float weight_scale = ei_fp16_to_fp32(weights[block].d); - for (int32_t input = 0; input < count; input++) { - const ei_block_q8_0 *activation = inputs + input * input_stride + block; - const __m256i values = _mm256_loadu_si256( - (const __m256i *)activation->qs); - const __m256i dot16 = _mm256_maddubs_epi16( - _mm256_sign_epi8(weight_values, weight_values), - _mm256_sign_epi8(values, weight_values)); - const __m256 dot = _mm256_cvtepi32_ps( - _mm256_madd_epi16(dot16, ones)); - const __m256 scale = _mm256_set1_ps( - weight_scale * ei_fp16_to_fp32(activation->d)); - sums[input] = _mm256_add_ps(sums[input], _mm256_mul_ps(dot, scale)); - } - } - for (int32_t input = 0; input < count; input++) out[input] = ei_hsum_f32x8(sums[input]); #else for (int32_t input = 0; input < count; input++) { out[input] = ei_vec_dot_q4_0_q8_0( @@ -383,6 +288,7 @@ void ei_vec_dot_q4_0_q8_0_dual(const ei_block_q4_0 *x0, const ei_block_q4_0 *x1, const ei_block_q8_0 *y, int32_t n, float *out0, float *out1) { if (n % EI_QK != 0) ei_die("dual q4_0 dot length %d is not a multiple of 32", n); + EI_DISPATCH_AVX2(ei_avx2_vec_dot_q4_0_q8_0_dual(x0, x1, y, n / EI_QK, out0, out1); return); #if defined(__ARM_NEON) && defined(__aarch64__) int32_t nb = n / EI_QK; const uint8x16_t low_mask = vdupq_n_u8(0x0Fu); @@ -414,33 +320,6 @@ void ei_vec_dot_q4_0_q8_0_dual(const ei_block_q4_0 *x0, const ei_block_q4_0 *x1, } *out0 = vaddvq_f32(sum0); *out1 = vaddvq_f32(sum1); -#elif defined(__AVX2__) - int32_t nb = n / EI_QK; - const __m256i low_mask = _mm256_set1_epi8(0x0F); - const __m256i off = _mm256_set1_epi8(8); - const __m256i ones = _mm256_set1_epi16(1); - __m256 sum0 = _mm256_setzero_ps(); - __m256 sum1 = _mm256_setzero_ps(); - for (int32_t b = 0; b < nb; b++) { - const __m256i yq = _mm256_loadu_si256((const __m256i *)y[b].qs); - const float y_scale = ei_fp16_to_fp32(y[b].d); - const ei_block_q4_0 *blocks[2] = { x0 + b, x1 + b }; - __m256 *sums[2] = { &sum0, &sum1 }; - for (int i = 0; i < 2; i++) { - const __m128i packed = _mm_loadu_si128((const __m128i *)blocks[i]->qs); - const __m256i q4 = _mm256_inserti128_si256( - _mm256_castsi128_si256(packed), _mm_srli_epi16(packed, 4), 1); - const __m256i xq = _mm256_sub_epi8(_mm256_and_si256(q4, low_mask), off); - const __m256i dot16 = _mm256_maddubs_epi16( - _mm256_sign_epi8(xq, xq), _mm256_sign_epi8(yq, xq)); - const __m256 dot = _mm256_cvtepi32_ps(_mm256_madd_epi16(dot16, ones)); - const __m256 scale = _mm256_set1_ps( - ei_fp16_to_fp32(blocks[i]->d) * y_scale); - *sums[i] = _mm256_add_ps(*sums[i], _mm256_mul_ps(dot, scale)); - } - } - *out0 = ei_hsum_f32x8(sum0); - *out1 = ei_hsum_f32x8(sum1); #else *out0 = ei_vec_dot_q4_0_q8_0(x0, y, n); *out1 = ei_vec_dot_q4_0_q8_0(x1, y, n); @@ -451,6 +330,8 @@ void ei_vec_dot_q4_0_q8_0_triple(const ei_block_q4_0 *x0, const ei_block_q4_0 *x const ei_block_q4_0 *x2, const ei_block_q8_0 *y, int32_t n, float *out0, float *out1, float *out2) { if (n % EI_QK != 0) ei_die("triple q4_0 dot length %d is not a multiple of 32", n); + EI_DISPATCH_AVX2(ei_avx2_vec_dot_q4_0_q8_0_triple(x0, x1, x2, y, n / EI_QK, + out0, out1, out2); return); #if defined(__ARM_NEON) && defined(__aarch64__) int32_t nb = n / EI_QK; const uint8x16_t low_mask = vdupq_n_u8(0x0Fu); @@ -478,34 +359,6 @@ void ei_vec_dot_q4_0_q8_0_triple(const ei_block_q4_0 *x0, const ei_block_q4_0 *x *out0 = vaddvq_f32(sums[0]); *out1 = vaddvq_f32(sums[1]); *out2 = vaddvq_f32(sums[2]); -#elif defined(__AVX2__) - int32_t nb = n / EI_QK; - const __m256i low_mask = _mm256_set1_epi8(0x0F); - const __m256i off = _mm256_set1_epi8(8); - const __m256i ones = _mm256_set1_epi16(1); - __m256 sums[3] = { - _mm256_setzero_ps(), _mm256_setzero_ps(), _mm256_setzero_ps(), - }; - for (int32_t b = 0; b < nb; b++) { - const __m256i yq = _mm256_loadu_si256((const __m256i *)y[b].qs); - const float y_scale = ei_fp16_to_fp32(y[b].d); - const ei_block_q4_0 *blocks[3] = { x0 + b, x1 + b, x2 + b }; - for (int i = 0; i < 3; i++) { - const __m128i packed = _mm_loadu_si128((const __m128i *)blocks[i]->qs); - const __m256i q4 = _mm256_inserti128_si256( - _mm256_castsi128_si256(packed), _mm_srli_epi16(packed, 4), 1); - const __m256i xq = _mm256_sub_epi8(_mm256_and_si256(q4, low_mask), off); - const __m256i dot16 = _mm256_maddubs_epi16( - _mm256_sign_epi8(xq, xq), _mm256_sign_epi8(yq, xq)); - const __m256 dot = _mm256_cvtepi32_ps(_mm256_madd_epi16(dot16, ones)); - const __m256 scale = _mm256_set1_ps( - ei_fp16_to_fp32(blocks[i]->d) * y_scale); - sums[i] = _mm256_add_ps(sums[i], _mm256_mul_ps(dot, scale)); - } - } - *out0 = ei_hsum_f32x8(sums[0]); - *out1 = ei_hsum_f32x8(sums[1]); - *out2 = ei_hsum_f32x8(sums[2]); #else *out0 = ei_vec_dot_q4_0_q8_0(x0, y, n); *out1 = ei_vec_dot_q4_0_q8_0(x1, y, n); diff --git a/src/test_cpu_dispatch.c b/src/test_cpu_dispatch.c new file mode 100644 index 0000000..33e49b0 --- /dev/null +++ b/src/test_cpu_dispatch.c @@ -0,0 +1,244 @@ +/* x86-64 runtime dispatch parity: embeds the same inputs through the forced + * SSE2 baseline and the AVX2/FMA/F16C kernels in one process and compares them at + * every Matryoshka dimension. Also times one ~150-token chunk per route. */ +#define _POSIX_C_SOURCE 200809L + +#include "engine.h" +#include "kernels_avx2.h" + +#include +#include +#include + +/* The routes differ only in float summation order and FMA, but a last-bit + * difference can flip a Q8_0 activation rounding and cascade through 24 + * layers, so the gate matches the llama.cpp golden gate rather than + * bit-equality. Real-model results sit near cosine 0.9999. */ +#define PARITY_MIN_COSINE 0.999 +#define PARITY_MAX_ABS_DIFF 1e-2 + +static const int32_t dimensions[] = { EI_N_EMBD, 512, 256, 128 }; +#define N_DIMS ((int)(sizeof dimensions / sizeof dimensions[0])) + +typedef struct { + char **items; + size_t n; +} string_list; + +static char *read_file(const char *path) { + FILE *f = fopen(path, "rb"); + if (!f) ei_die("%s: cannot open", path); + if (fseek(f, 0, SEEK_END) != 0) ei_die("%s: seek failed", path); + long n = ftell(f); + if (n < 0 || fseek(f, 0, SEEK_SET) != 0) ei_die("%s: tell failed", path); + char *data = ei_xmalloc((size_t)n + 1u); + if (fread(data, 1, (size_t)n, f) != (size_t)n) ei_die("%s: read failed", path); + fclose(f); + data[n] = '\0'; + return data; +} + +/* A flat JSON array of strings with simple escapes (testdata/test-strings.json). */ +static string_list parse_strings(const char *path) { + char *data = read_file(path); + string_list list = { 0 }; + const char *p = strchr(data, '['); + if (!p) ei_die("%s: expected a JSON array", path); + for (p++; *p && *p != ']'; p++) { + if (*p != '"') continue; + char *s = ei_xmalloc(strlen(p) + 1u); + size_t len = 0; + for (p++; *p != '"'; p++) { + if (!*p) ei_die("%s: unterminated string", path); + char c = *p; + if (c == '\\') { + c = *++p; + if (c == 'n') c = '\n'; + else if (c == 't') c = '\t'; + else if (c != '"' && c != '\\' && c != '/') ei_die("%s: unsupported escape", path); + } + s[len++] = c; + } + s[len] = '\0'; + list.items = ei_xrealloc(list.items, sizeof(char *) * (list.n + 1u)); + list.items[list.n++] = s; + } + free(data); + return list; +} + +/* An input of exactly n_tokens ids, cycling through the tokenized samples. */ +static ei_tokens repeated_tokens(const ei_tokens *samples, size_t n_samples, size_t n_tokens) { + ei_tokens out = { ei_xmalloc(sizeof(int32_t) * n_tokens), n_tokens }; + size_t len = 0; + for (size_t i = 0; len < n_tokens; i = (i + 1) % n_samples) { + for (size_t j = 0; j < samples[i].n && len < n_tokens; j++) { + out.ids[len++] = samples[i].ids[j]; + } + } + return out; +} + +static void embed_as(ei_engine *e, const char *isa, const ei_tokens *tokens, + float out[EI_N_EMBD]) { + if (!ei_cpu_set_isa(isa)) ei_die("cannot select CPU ISA %s", isa); + char err[256]; + if (!ei_engine_embed_tokens(e, tokens->ids, tokens->n, out, err, sizeof err)) { + ei_die("embedding failed under %s: %s", isa, err); + } +} + +static void embed_batch_as(ei_engine *e, const char *isa, const ei_tokens *tokens, + size_t n, float *out) { + if (!ei_cpu_set_isa(isa)) ei_die("cannot select CPU ISA %s", isa); + size_t total = 0; + for (size_t i = 0; i < n; i++) total += tokens[i].n; + int32_t *ids = ei_xmalloc(sizeof(int32_t) * total); + size_t *offsets = ei_xmalloc(sizeof(size_t) * (n + 1u)); + offsets[0] = 0; + for (size_t i = 0; i < n; i++) { + memcpy(ids + offsets[i], tokens[i].ids, sizeof(int32_t) * tokens[i].n); + offsets[i + 1] = offsets[i] + tokens[i].n; + } + char err[256]; + if (!ei_engine_embed_tokens_batch(e, ids, offsets, n, out, err, sizeof err)) { + ei_die("batch embedding failed under %s: %s", isa, err); + } + free(ids); + free(offsets); +} + +static double max_abs[N_DIMS]; +static double min_cos[N_DIMS]; +static int failures; + +static void compare(const char *label, const float *baseline, const float *avx2) { + for (int d = 0; d < N_DIMS; d++) { + float a[EI_N_EMBD], b[EI_N_EMBD]; + memcpy(a, baseline, sizeof a); + memcpy(b, avx2, sizeof b); + if (!ei_embedding_normalize_prefix(a, dimensions[d]) || + !ei_embedding_normalize_prefix(b, dimensions[d])) { + ei_die("bad dimension %d", dimensions[d]); + } + double dot = 0.0, na = 0.0, nb = 0.0, diff = 0.0; + for (int32_t i = 0; i < dimensions[d]; i++) { + dot += (double)a[i] * b[i]; + na += (double)a[i] * a[i]; + nb += (double)b[i] * b[i]; + double delta = fabs((double)a[i] - b[i]); + if (!(delta <= diff)) diff = delta; /* also catches NaN */ + } + double cosine = dot / sqrt(na * nb); + if (!(cosine >= min_cos[d])) min_cos[d] = cosine; + if (!(diff <= max_abs[d])) max_abs[d] = diff; + if (!(cosine >= PARITY_MIN_COSINE) || !(diff <= PARITY_MAX_ABS_DIFF)) { + printf("FAIL %s D=%d: SSE2 vs AVX2 cosine %.9f, max abs diff %.3g\n", + label, dimensions[d], cosine, diff); + failures++; + } + } +} + +static double now_seconds(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + ei_die("clock_gettime failed: %s", strerror(errno)); + } + return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9; +} + +static double time_chunk(ei_engine *e, const char *isa, const ei_tokens *tokens, int reps) { + float out[EI_N_EMBD]; + embed_as(e, isa, tokens, out); /* warm-up */ + double best = INFINITY; + for (int r = 0; r < reps; r++) { + double start = now_seconds(); + embed_as(e, isa, tokens, out); + double elapsed = now_seconds() - start; + if (elapsed < best) best = elapsed; + } + return best; +} + +int main(int argc, char **argv) { + if (argc != 3) ei_die("usage: %s ", argv[0]); + if (!ei_cpu_avx2_supported()) { + printf("cpu dispatch parity: skipped (no x86-64 AVX2+FMA+F16C route on this host)\n"); + return 0; + } + + ei_engine engine; + ei_engine_load_backend(&engine, argv[1], "cpu"); + string_list texts = parse_strings(argv[2]); + /* The samples, then 150 tokens (one typical chunk) and 600 tokens (past + * the 512-token sliding window). */ + const size_t n_samples = texts.n, n_inputs = texts.n + 2u; + ei_tokens *tokens = ei_xmalloc(sizeof(ei_tokens) * n_inputs); + for (size_t i = 0; i < n_samples; i++) { + ei_tokenize_spm(&engine.tokenizer, texts.items[i], strlen(texts.items[i]), + true, false, &tokens[i]); + } + tokens[n_samples] = repeated_tokens(tokens, n_samples, 150u); + tokens[n_samples + 1u] = repeated_tokens(tokens, n_samples, 600u); + + for (int d = 0; d < N_DIMS; d++) { + max_abs[d] = 0.0; + min_cos[d] = 1.0; + } + + for (size_t i = 0; i < n_inputs; i++) { + float baseline[EI_N_EMBD], avx2[EI_N_EMBD]; + embed_as(&engine, "baseline", &tokens[i], baseline); + embed_as(&engine, "avx2", &tokens[i], avx2); + char label[64]; + snprintf(label, sizeof label, "input %zu (%zu tokens)", i, tokens[i].n); + compare(label, baseline, avx2); + } + + /* The batched route uses the multi-input Q4_0 kernel. */ + float *batch_baseline = ei_xmalloc(sizeof(float) * EI_N_EMBD * n_samples); + float *batch_avx2 = ei_xmalloc(sizeof(float) * EI_N_EMBD * n_samples); + embed_batch_as(&engine, "baseline", tokens, n_samples, batch_baseline); + embed_batch_as(&engine, "avx2", tokens, n_samples, batch_avx2); + for (size_t i = 0; i < n_samples; i++) { + char label[64]; + snprintf(label, sizeof label, "batch item %zu", i); + compare(label, batch_baseline + i * EI_N_EMBD, batch_avx2 + i * EI_N_EMBD); + } + + for (int d = 0; d < N_DIMS; d++) { + printf("cpu dispatch parity D=%d: min cosine %.9f, max abs diff %.3g\n", + dimensions[d], min_cos[d], max_abs[d]); + } + if (failures) { + ei_die("cpu dispatch parity: %d comparisons outside cosine >= %.3f, " + "max abs <= %.0e", failures, PARITY_MIN_COSINE, PARITY_MAX_ABS_DIFF); + } + printf("cpu dispatch parity: %zu inputs + %zu batched, SSE2 vs AVX2 within " + "cosine >= %.3f and max abs <= %.0e\n", + n_inputs, n_samples, PARITY_MIN_COSINE, PARITY_MAX_ABS_DIFF); + + const ei_tokens *chunk = &tokens[n_samples]; + size_t chunk_tokens = chunk->n; + double sse2 = time_chunk(&engine, "baseline", chunk, 3); + double avx2 = time_chunk(&engine, "avx2", chunk, 3); + printf("cpu dispatch timing: %zu-token chunk, %d threads: SSE2 %.1f ms " + "(%.0f tok/s), AVX2 %.1f ms (%.0f tok/s), speedup %.2fx\n", + chunk_tokens, ei_engine_threads(&engine), sse2 * 1e3, + (double)chunk_tokens / sse2, avx2 * 1e3, (double)chunk_tokens / avx2, + sse2 / avx2); + + for (size_t i = 0; i < n_samples; i++) { + ei_tokens_free(&tokens[i]); + free(texts.items[i]); + } + free(tokens[n_samples].ids); + free(tokens[n_samples + 1u].ids); + free(tokens); + free(texts.items); + free(batch_baseline); + free(batch_avx2); + ei_engine_free(&engine); + return 0; +} diff --git a/src/test_kernels.c b/src/test_kernels.c index fca1d70..b1dd258 100644 --- a/src/test_kernels.c +++ b/src/test_kernels.c @@ -1,4 +1,5 @@ #include "kernels.h" +#include "kernels_avx2.h" #include "quants.h" #include @@ -254,12 +255,59 @@ static void test_matryoshka_dimensions(void) { } } +/* Runs the quantized kernels under the forced SSE2 baseline and under AVX2 on + * the same inputs. Integer dot products are exact, so only the float + * accumulation order (and FMA) separates the two routes. */ +static void test_isa_parity(void) { + enum { K = EI_N_FF, ROWS = 64, INPUTS = 6, BLOCKS = K / EI_QK }; + static ei_block_q4_0 weights[ROWS * BLOCKS]; + static float x[INPUTS * K]; + static ei_block_q8_0 xq[2][INPUTS * BLOCKS]; + static float rows[2][ROWS], dual[2][2 * ROWS], batch[2][INPUTS * ROWS]; + static float dequant[2][K]; + for (int i = 0; i < ROWS * BLOCKS; i++) { + weights[i].d = ei_fp32_to_fp16(0.002f + 0.0001f * (float)(i % 97)); + for (int j = 0; j < 16; j++) weights[i].qs[j] = (uint8_t)(next_f32() * 127.0f + 128.0f); + } + fill(x, INPUTS * K); + ei_tensor w = { .type = EI_T_Q4_0, .ne = { K, ROWS, 1, 1 }, .data = weights }; + ei_tensor xt = { .type = EI_T_Q8_0, .ne = { K, INPUTS, 1, 1 } }; + const char *isas[2] = { "baseline", "avx2" }; + for (int r = 0; r < 2; r++) { + if (!ei_cpu_set_isa(isas[r])) ei_die("cannot select %s", isas[r]); + for (int t = 0; t < INPUTS; t++) { + ei_quantize_row_q8_0(x + t * K, xq[r] + t * BLOCKS, K); + } + ei_matmul_q4_0_q8_0_rows3(&w, xq[0], rows[r], 0, ROWS); + ei_matmul_q4_0_q8_0_dual_rows(&w, &w, xq[0], dual[r], dual[r] + ROWS, 0, ROWS); + ei_matmul_q4_0_q8_0_batch_rows(&w, xq[0], INPUTS, batch[r], 0, ROWS); + xt.data = xq[0]; + ei_dequantize_row_q8_0_scaled(&xt, 1, 0.5f, dequant[r]); + } + if (!ei_cpu_set_isa("auto")) ei_die("cannot restore auto ISA"); + if (memcmp(xq[0], xq[1], sizeof xq[0]) != 0) ei_die("q8_0 quantization differs by ISA"); + printf("q8 quantize ISA parity: identical\n"); + require_close("q4 rows ISA parity", rows[1], rows[0], ROWS, 2e-5f); + require_close("q4 dual ISA parity", dual[1], dual[0], 2 * ROWS, 2e-5f); + require_close("q4 batch ISA parity", batch[1], batch[0], INPUTS * ROWS, 2e-5f); + require_close("q8 dequant ISA parity", dequant[1], dequant[0], K, 0.0f); +} + int main(void) { - test_q4_triple(); - test_attention(); - test_norm_residual(); - test_qk_norm_rope(); - test_mean_pool(); - test_matryoshka_dimensions(); + const char *isas[] = { "baseline", "avx2" }; + for (int i = 0; i < 2; i++) { + if (!ei_cpu_set_isa(isas[i])) { + printf("CPU kernels: %s route unavailable on this host\n", isas[i]); + continue; + } + printf("CPU kernels: %s / %s\n", ei_cpu_kernel_variant(), ei_quants_kernel_variant()); + test_q4_triple(); + test_attention(); + test_norm_residual(); + test_qk_norm_rope(); + test_mean_pool(); + test_matryoshka_dimensions(); + } + if (ei_cpu_avx2_supported()) test_isa_parity(); return 0; } From 7a5f816ae9101e0d3c5a589dd2ce7bd98e04d6fd Mon Sep 17 00:00:00 2001 From: Eric Hartford Date: Sun, 18 Oct 2026 06:02:42 -0400 Subject: [PATCH 8/9] Port the Windows build to the EmbeddingGemma 2 profile Main gained a setenv call (--threads, --cuda-devices) and defaults to the embeddinggemma-2 profile. Add a setenv shim to windows_compat.h, pin the 300m profile in the downloader test, and read test_gemma2's UTF-8 text arguments from the wide command line (the CRT argv is in the ANSI code page). Drop a duplicate include. Co-Authored-By: Claude Opus 5.5 --- src/inference_service.c | 1 - src/test_gemma2.c | 21 +++++++++++++++++++++ src/windows_compat.h | 6 ++++++ testdata/test_windows_download.py | 2 +- 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/inference_service.c b/src/inference_service.c index a173170..5cba6a4 100644 --- a/src/inference_service.c +++ b/src/inference_service.c @@ -10,7 +10,6 @@ #include #include #include "windows_compat.h" -#include "windows_compat.h" typedef enum { EI_ENTRY_PENDING, diff --git a/src/test_gemma2.c b/src/test_gemma2.c index 7ec4ff5..9530822 100644 --- a/src/test_gemma2.c +++ b/src/test_gemma2.c @@ -7,6 +7,24 @@ #if defined(EI_ENABLE_CUDA) #include "gemma2_cuda.h" #endif +#ifdef _WIN32 +#include "windows_compat.h" +#include +// The CRT argv is in the ANSI code page; the goldens are UTF-8 text. +static char **utf8_argv(void) { + int n = 0; + wchar_t **wide = CommandLineToArgvW(GetCommandLineW(), &n); + if (!wide) ei_die("CommandLineToArgvW failed"); + char **out = ei_xcalloc((size_t)n + 1, sizeof *out); + for (int i = 0; i < n; i++) { + int bytes = WideCharToMultiByte(CP_UTF8, 0, wide[i], -1, NULL, 0, NULL, NULL); + out[i] = ei_xmalloc((size_t)bytes); + WideCharToMultiByte(CP_UTF8, 0, wide[i], -1, out[i], bytes, NULL, NULL); + } + LocalFree(wide); + return out; +} +#endif static void vector(const float *x, size_t n) { putchar('['); @@ -15,6 +33,9 @@ static void vector(const float *x, size_t n) { putchar(']'); } int main(int argc, char **argv) { +#ifdef _WIN32 + argv = utf8_argv(); +#endif if (argc < 4) ei_die("usage: %s text.gguf projector.gguf text|vision|audio [text]", argv[0]); ei_engine e; diff --git a/src/windows_compat.h b/src/windows_compat.h index 2ac13e8..8f3b977 100644 --- a/src/windows_compat.h +++ b/src/windows_compat.h @@ -51,6 +51,12 @@ static inline int ei_mkstemp(char *pattern) { return -1; } #define mkstemp ei_mkstemp +/* The server maps --threads and --cuda-devices onto environment variables. */ +static inline int ei_setenv(const char *name, const char *value, int overwrite) { + if (!overwrite && getenv(name)) return 0; + return _putenv_s(name, value) == 0 ? 0 : -1; +} +#define setenv ei_setenv /* Windows rename does not replace a destination; cache publication must. */ static inline int ei_replace_file(const char *from, const char *to) { if (MoveFileExA(from, to, MOVEFILE_REPLACE_EXISTING)) return 0; diff --git a/testdata/test_windows_download.py b/testdata/test_windows_download.py index 2d46b62..a15a581 100644 --- a/testdata/test_windows_download.py +++ b/testdata/test_windows_download.py @@ -30,7 +30,7 @@ def main(): for key in ("XDG_CACHE_HOME", "EI_MODEL_PATH", "EI_TEST_CURL_FAIL", "HOME"): env.pop(key, None) env.update(PATH=str(tools), LOCALAPPDATA=str(directory), USERPROFILE=str(directory)) - command = [binary] + command = [binary, "--profile", "embeddinggemma-300m"] expected = directory / "embeddinggemma.c" / "embeddinggemma-300M-qat-Q4_0.gguf" if mode == "fallback": env["EI_TEST_CURL_FAIL"] = "1" From 6adc619e17a1cdf7be6b7920dd1528552698b870 Mon Sep 17 00:00:00 2001 From: Eric Hartford Date: Sun, 18 Oct 2026 06:17:17 -0400 Subject: [PATCH 9/9] Fix the new CI jobs against the EmbeddingGemma 2 sources The AVX isolation check now also accepts the g2x_*/q8_tile_* kernels in gemma2_fast_x86.c, which run only after __builtin_cpu_supports. Guard weight_cache_lock to the targets that use it so linux-arm64 builds with -Werror. Co-Authored-By: Claude Opus 5.5 --- scripts/check-avx2-isolation.py | 6 +++++- src/gemma2_fast.c | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/check-avx2-isolation.py b/scripts/check-avx2-isolation.py index 7b3f96b..7f88281 100755 --- a/scripts/check-avx2-isolation.py +++ b/scripts/check-avx2-isolation.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Verify that a portable x86-64 executable keeps AVX code inside the -runtime-dispatched ei_avx2_* kernels. +runtime-dispatched kernels: ei_avx2_* (kernels_avx2.c) and the EmbeddingGemma 2 +g2x_* / q8_tile_* kernels (gemma2_fast_x86.c, behind __builtin_cpu_supports). Every other function must run on an SSE2-only CPU, so it must not contain a VEX-encoded instruction (mnemonics starting with "v") or touch a ymm/zmm @@ -12,6 +13,7 @@ import sys FUNCTION = re.compile(r"^[0-9a-f]+ <(?P[^>]+)>:$") +DISPATCHED_PREFIXES = ("ei_avx2_", "g2x_", "q8_tile_") INSTRUCTION = re.compile(r"^\s*[0-9a-f]+:\s+(?P[a-z][a-z0-9.]*)\s*(?P.*)$") @@ -43,6 +45,8 @@ def main() -> int: continue if function.startswith("ei_avx2_"): dispatched.add(function) + elif function.startswith(DISPATCHED_PREFIXES): + pass else: leaks.setdefault(function, f"{mnemonic} {operands}".strip()) diff --git a/src/gemma2_fast.c b/src/gemma2_fast.c index f69362a..db4163f 100644 --- a/src/gemma2_fast.c +++ b/src/gemma2_fast.c @@ -83,7 +83,9 @@ struct ei_gemma2_fast { workspace ws; }; static ei_gemma2_fast default_ctx = {PTHREAD_MUTEX_INITIALIZER, {0}}; +#if (defined(__aarch64__) && defined(__APPLE__)) || defined(__x86_64__) static pthread_mutex_t weight_cache_lock = PTHREAD_MUTEX_INITIALIZER; +#endif static void reserve(workspace *ws, size_t n) { if (ws->tokens >= n)