From e5e474a958bee95d6162eac8be01cc815ff40423 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 14 Sep 2026 15:36:39 +0200 Subject: [PATCH 1/6] refactor(grpc-web): top-level pyodide imports, [grpc-web] extra, in-Pyodide unit tests The web package now imports pyodide at module scope and is importable only under Emscripten/Pyodide. The base client gains a grpc-web extra (weaviate-client[grpc-web], marker-gated to Emscripten) as the documented install path. The unit tests stay in packages/web/tests but run inside Pyodide via ci/pyodide-e2e/units.mjs (async-native, no pytest); a conftest keeps CPython pytest from collecting them, and the CPython-only testing seams (install force flags, make_httpx_sender) are removed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Caze9m6PBSfYkt77mMKj2b --- .github/workflows/main.yaml | 30 +- ci/pyodide-e2e/units.mjs | 154 ++++ packages/web/README.md | 42 +- .../web/src/weaviate_client_web/__init__.py | 30 +- .../web/src/weaviate_client_web/_channel.py | 4 +- .../src/weaviate_client_web/_httpx_fetch.py | 23 +- .../web/src/weaviate_client_web/_sender.py | 42 +- packages/web/src/weaviate_client_web/_shim.py | 10 +- packages/web/tests/conftest.py | 13 +- packages/web/tests/harness.py | 53 ++ packages/web/tests/runner.py | 42 ++ packages/web/tests/test_framing.py | 26 +- packages/web/tests/test_httpx_fetch.py | 694 +++++++----------- packages/web/tests/test_shim_install.py | 164 ++--- packages/web/tests/test_single_import.py | 139 ---- packages/web/tests/test_transport.py | 313 ++++---- setup.cfg | 6 + weaviate/__init__.py | 5 +- weaviate/connect/base.py | 9 +- 19 files changed, 866 insertions(+), 933 deletions(-) create mode 100644 ci/pyodide-e2e/units.mjs create mode 100644 packages/web/tests/harness.py create mode 100644 packages/web/tests/runner.py delete mode 100644 packages/web/tests/test_single_import.py diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 163c70afa..44e52d6d0 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -105,28 +105,8 @@ jobs: name: coverage-report-${{ matrix.folder }} path: coverage-${{ matrix.folder }}.xml - grpc-web-tests: - name: Run gRPC-Web Package Tests - runs-on: ubuntu-latest - timeout-minutes: 5 - strategy: - fail-fast: false - matrix: - version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: ${{ matrix.version }} - cache: 'pip' # caching pip dependencies - - run: | - pip install -r requirements-test.txt -r requirements-devel.txt - pip install -e . -e packages/web - - name: Run grpc-web package tests - run: pytest packages/web/tests - pyodide-e2e: - name: Run Pyodide (WASM) e2e Tests + name: Run Pyodide (WASM) Unit + e2e Tests runs-on: ubuntu-latest timeout-minutes: 15 # No Python matrix: the pinned Pyodide bundle fixes the interpreter (see @@ -154,6 +134,12 @@ jobs: pip install build python -m build --wheel --outdir dist . python -m build --wheel --outdir dist packages/web + - name: Run the unit suite inside Pyodide under Node + # The grpc-web package imports pyodide at module scope, so its unit tests only + # run here. Needs no running Weaviate. + run: | + npm install --prefix ci/pyodide-e2e + node ci/pyodide-e2e/units.mjs dist - name: start weaviate run: | source ./ci/compose.sh @@ -418,7 +404,7 @@ jobs: build-and-publish: name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI - needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package, proto-test, grpc-web-tests, pyodide-e2e] + needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package, proto-test, pyodide-e2e] runs-on: ubuntu-latest timeout-minutes: 20 steps: diff --git a/ci/pyodide-e2e/units.mjs b/ci/pyodide-e2e/units.mjs new file mode 100644 index 000000000..61ba72c2c --- /dev/null +++ b/ci/pyodide-e2e/units.mjs @@ -0,0 +1,154 @@ +// Runs the weaviate_client_web unit suite (packages/web/tests) inside Pyodide (WASM) +// under Node, plus bootstrap scenarios that each need a fresh interpreter. No running +// Weaviate is needed — everything is driven through fake senders / a fake pyfetch. +// +// Usage: node units.mjs +// must contain exactly the two locally-built pure wheels: +// weaviate_client-*.whl and weaviate_client_web-*.whl (same layout as run.mjs). +// +// The package imports pyodide at module scope, so this harness is the only place its +// unit tests can run; scenarios needing a clean import state (bootstrap and +// install-hint semantics) get one loadPyodide() each. Deliberately untested: the +// fall-through in weaviate/__init__.py when a real grpc module exists — no grpcio +// wheel exists for Emscripten, so that branch is CPython-only defence and cannot be +// exercised here. +import { readdirSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { loadPyodide } from "pyodide"; + +if (!process.argv[2]) { + console.error("usage: node units.mjs "); + process.exit(2); +} +const wheelsDir = resolve(process.argv[2]); +const here = dirname(fileURLToPath(import.meta.url)); + +const wheels = readdirSync(wheelsDir) + .filter((f) => f.endsWith(".whl")) + .sort(); // installs weaviate_client before weaviate_client_web, which depends on it +const prefixes = ["weaviate_client-", "weaviate_client_web-"]; +if ( + wheels.length !== 2 || + !prefixes.every((p) => wheels.some((w) => w.startsWith(p))) +) { + console.error( + `expected exactly one weaviate_client-*.whl and one weaviate_client_web-*.whl in ${wheelsDir}, found: ${JSON.stringify(wheels)}`, + ); + process.exit(2); +} + +// Fresh interpreter with micropip ready and the wheels dir mounted; `only` restricts +// which of the two wheels get installed (the install-hint scenarios need the base +// client without its companion). +async function freshPyodide(only = prefixes) { + const pyodide = await loadPyodide({}); + await pyodide.loadPackage("micropip"); + const micropip = pyodide.pyimport("micropip"); + pyodide.FS.mkdirTree("/wheels"); + pyodide.mountNodeFS("/wheels", wheelsDir); + for (const wheel of wheels) { + if (only.some((p) => wheel.startsWith(p))) { + await micropip.install(`emfs:/wheels/${wheel}`); + } + } + return pyodide; +} + +async function scenario(name, pyodide, code) { + try { + pyodide.runPython(code); + console.log(`OK scenario: ${name}`); + } catch (err) { + console.error(`FAIL scenario: ${name}`); + console.error(err); + process.exit(1); + } +} + +// --- fresh-interpreter bootstrap scenarios ----------------------------------------- + +await scenario( + "bare 'import weaviate' bootstraps the companion", + await freshPyodide(), + ` +import sys +assert "weaviate_client_web" not in sys.modules +import weaviate # the ONLY weaviate-side import: must bootstrap the companion +assert "weaviate_client_web" in sys.modules, "hook did not import the companion" +import weaviate_client_web +import grpc +import httpx +assert weaviate_client_web.is_installed() +assert weaviate_client_web.is_fetch_transport_installed() +assert getattr(grpc, "__weaviate_client_web_shim__", False) is True +assert getattr( + httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False +) is True +`, +); + +await scenario( + "missing companion raises the install hint", + await freshPyodide(["weaviate_client-"]), + ` +try: + import weaviate +except ImportError as e: + assert "weaviate-client-web" in str(e), str(e) + assert "weaviate-client[grpc-web]" in str(e), str(e) + assert "WebAssembly/Pyodide" in str(e), str(e) +else: + raise AssertionError("expected ImportError without the companion") +`, +); + +await scenario( + "a broken companion surfaces its own error, not the install hint", + await freshPyodide(["weaviate_client-"]), + ` +# An INSTALLED companion whose import fails (here: a missing dependency of its own) +# must raise that error — the "install weaviate-client[grpc-web]" hint would send the +# user to reinstall a package that is already there. +import pathlib, sys +pkg = pathlib.Path("/broken/weaviate_client_web") +pkg.mkdir(parents=True) +(pkg / "__init__.py").write_text( + "raise ModuleNotFoundError(\\"No module named 'anyio'\\", name='anyio')\\n" +) +sys.path.insert(0, "/broken") +try: + import weaviate +except ImportError as e: + assert e.name == "anyio", (e.name, str(e)) + assert "anyio" in str(e), str(e) + assert "grpc-web" not in str(e), str(e) +else: + raise AssertionError("expected the companion's own ImportError to surface") +`, +); + +// --- the main unit suite ----------------------------------------------------------- + +const testsDir = resolve(here, "../../packages/web/tests"); +const pyodide = await freshPyodide(); +console.log( + `pyodide ${pyodide.version} / python ${pyodide.runPython("import sys; sys.version.split()[0]")}`, +); +pyodide.FS.mkdirTree("/units"); +pyodide.mountNodeFS("/units", testsDir); +try { + await pyodide.runPythonAsync(` +import sys +sys.path.insert(0, "/units") +import runner +await runner.main() +`); +} catch (err) { + console.error(err); + process.exit(1); +} +// The interpreters loaded above keep live handles on the Node event loop, so the +// process does not exit on its own. +process.exit(0); diff --git a/packages/web/README.md b/packages/web/README.md index 582ab2796..c17e7f8df 100644 --- a/packages/web/README.md +++ b/packages/web/README.md @@ -12,6 +12,23 @@ Requires Weaviate ≥ 1.38.3 (the first release to serve grpc-web natively) or a transcoder in front of an older server. Pyodide ≥ 0.27 recommended; verified on Pyodide 314.0.4 (CPython 3.14). +## Installation + +Install through the base client's `grpc-web` extra: + +```python +import micropip +await micropip.install("weaviate-client[grpc-web]") +``` + +The extra carries a `sys_platform == "emscripten"` marker, so the same requirement is a +no-op on CPython — one requirements list works everywhere. Installing the companion +directly (`micropip.install("weaviate-client-web")`) works too. + +This package is defined for its environment: it imports `pyodide` at module scope and is +therefore only importable under Emscripten/Pyodide. On CPython the base client never +imports it (and the extra never installs it). + ## How it works Under Pyodide there is no `grpcio` Emscripten wheel, and `import weaviate` hard-imports @@ -81,7 +98,7 @@ await collection.query.near_text("hello", limit=3) Nothing selects grpc-web: `use_async_with_local()`, `use_async_with_weaviate_cloud()` and `use_async_with_custom()` all route gRPC onto the REST endpoint under `/v1/grpc-web` when -they run under Emscripten, and behave exactly as before everywhere else. +they run under Emscripten, and use native gRPC everywhere else. ```python client = weaviate.use_async_with_weaviate_cloud( @@ -161,11 +178,20 @@ deployments that go through a grpc-web transcoder or a proxy must configure CORS - note that a CORS-blocked request is indistinguishable from a network failure in the browser (`TypeError: Failed to fetch`), and is retried as UNAVAILABLE. -## Testing on CPython +## Testing + +Because the package imports `pyodide` at module scope, its unit tests run inside +Pyodide. From the repository root: + +```sh +python -m build --wheel --outdir dist . +python -m build --wheel --outdir dist packages/web +npm install --prefix ci/pyodide-e2e +node ci/pyodide-e2e/units.mjs dist # unit suite + bootstrap scenarios, no Weaviate needed +node ci/pyodide-e2e/run.mjs dist # e2e suite, needs a running Weaviate (see ci/) +``` -`weaviate_client_web.install(force=True)` installs the shim on a normal CPython -interpreter (run it in a fresh process, before importing `weaviate`). Inject a sender -with `weaviate_client_web.set_sender(...)` (e.g. `make_httpx_sender()`) to exercise the -transport against an Envoy/vanguard transcoder without a browser. -`install_fetch_transport(force=True)` likewise patches httpx on CPython, given an -importable `pyodide.http` stand-in. +`units.mjs` runs the unit suite in `packages/web/tests/` (driven entirely through fake +senders and a fake `pyfetch`) plus fresh-interpreter bootstrap scenarios; `run.mjs` +runs the e2e suite against a live Weaviate. A `conftest.py` keeps pytest from +collecting the test modules on CPython, where they cannot import. diff --git a/packages/web/src/weaviate_client_web/__init__.py b/packages/web/src/weaviate_client_web/__init__.py index 2e9217991..d573b7af3 100644 --- a/packages/web/src/weaviate_client_web/__init__.py +++ b/packages/web/src/weaviate_client_web/__init__.py @@ -23,14 +23,22 @@ warns that it overrode them. An explicit ``import weaviate_client_web`` before ``import weaviate`` also works and -remains the explicit form. The shim is installed automatically only under Emscripten, so -importing this package on a normal CPython install never clobbers a real, working -``grpcio``. Async clients only — the synchronous client is not supported in the browser. +remains the explicit form. This package is defined for its environment: it imports +``pyodide`` at module scope, so it is only importable under Emscripten/Pyodide — on +CPython the base client never imports it, and the ``weaviate-client[grpc-web]`` extra +does not install it there (platform marker). Async clients only — the synchronous client +is not supported in the browser. """ import os import sys +from ._channel import GrpcWebChannel, set_sender +from ._httpx_fetch import ( + install_fetch_transport, + is_fetch_transport_installed, + uninstall_fetch_transport, +) from ._shim import StatusCode, install, is_installed __all__ = [ @@ -40,7 +48,6 @@ "uninstall_fetch_transport", "is_fetch_transport_installed", "set_sender", - "make_httpx_sender", "GrpcWebChannel", "StatusCode", ] @@ -54,21 +61,8 @@ def _bootstrap() -> None: os.environ.setdefault("PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", "python") install() # The REST path needs fetch too: httpx/httpcore open raw sockets, which do - # not exist under WASM. Imported lazily so CPython imports stay light. - from ._httpx_fetch import install_fetch_transport - + # not exist under WASM. install_fetch_transport() _bootstrap() - -# Imported after the bootstrap. These modules pull their grpc base classes directly from -# ``._shim`` (not via ``sys.modules['grpc']``), so importing them is safe regardless of -# whether the shim was installed. -from ._channel import GrpcWebChannel, set_sender # noqa: E402 -from ._httpx_fetch import ( # noqa: E402 - install_fetch_transport, - is_fetch_transport_installed, - uninstall_fetch_transport, -) -from ._sender import make_httpx_sender # noqa: E402 diff --git a/packages/web/src/weaviate_client_web/_channel.py b/packages/web/src/weaviate_client_web/_channel.py index 428c4f8f8..64e6d06be 100644 --- a/packages/web/src/weaviate_client_web/_channel.py +++ b/packages/web/src/weaviate_client_web/_channel.py @@ -23,12 +23,12 @@ from ._sender import Sender, pyfetch_sender from ._shim import AioChannel, AioRpcError, StatusCode, status_from_int -# Module-level default sender; overridable for tests / non-browser runtimes. +# Module-level default sender; overridable for tests. _default_sender: Sender = pyfetch_sender def set_sender(sender: Sender) -> None: - """Override the default async sender used by new channels (tests/integration).""" + """Override the default async sender used by new channels (tests).""" global _default_sender _default_sender = sender diff --git a/packages/web/src/weaviate_client_web/_httpx_fetch.py b/packages/web/src/weaviate_client_web/_httpx_fetch.py index b8f4be69c..b9d400465 100644 --- a/packages/web/src/weaviate_client_web/_httpx_fetch.py +++ b/packages/web/src/weaviate_client_web/_httpx_fetch.py @@ -21,6 +21,10 @@ - the browser's fetch follows redirects internally, so httpx never sees a 3xx; - multi-value response headers (e.g. Set-Cookie) are folded into one value; - responses are fully buffered (no streaming). + +Like the rest of this package, this module imports ``pyodide`` at module scope and is +therefore only importable under Emscripten/Pyodide (or with a ``pyodide`` stand-in +pre-installed in ``sys.modules``). """ import math @@ -28,13 +32,14 @@ from typing import Callable, Dict, Optional import httpx +from pyodide.http import pyfetch # type: ignore[import-not-found] _installed = False _original_handle_async_request: Optional[Callable] = None # Hop-by-hop / connection-managed headers that the browser's fetch controls itself. -# Browsers silently drop forbidden headers, but Node's undici (used by the CPython/Node -# test path) rejects some of them outright, so strip them before handing off. +# Browsers silently drop forbidden headers, but Node's undici (behind pyfetch in the +# Node-based test harness) rejects some of them outright, so strip them before handing off. _FETCH_MANAGED_HEADERS = { "host", "connection", @@ -122,8 +127,6 @@ def _validate_header(name: str, value: str) -> None: async def _fetch_handle_async_request( self: httpx.AsyncHTTPTransport, request: httpx.Request ) -> httpx.Response: - from pyodide.http import pyfetch # type: ignore[import-not-found] - headers: Dict[str, str] = {} for k, v in request.headers.items(): if k.lower() in _FETCH_MANAGED_HEADERS: @@ -177,21 +180,17 @@ async def _fetch_handle_async_request( _fetch_handle_async_request.__weaviate_fetch_shim__ = True # type: ignore[attr-defined] -def install_fetch_transport(force: bool = False) -> None: +def install_fetch_transport() -> None: """Patch ``httpx.AsyncHTTPTransport`` to send requests through ``fetch``. - Installs only under Emscripten unless ``force=True`` (CPython testing, where a - ``pyodide`` stub must be importable). Idempotent. + Installs only under Emscripten (elsewhere httpx's own socket transports work and + must be left in place). Idempotent. """ global _installed, _original_handle_async_request if _installed: return - if not force and sys.platform != "emscripten": + if sys.platform != "emscripten": return - # Fail fast: the handler imports pyfetch per request, so a missing pyodide module - # would otherwise surface as a confusing ModuleNotFoundError on the first request. - from pyodide.http import pyfetch # type: ignore[import-not-found] # noqa: F401 - _original_handle_async_request = httpx.AsyncHTTPTransport.handle_async_request httpx.AsyncHTTPTransport.handle_async_request = _fetch_handle_async_request # type: ignore[method-assign] _installed = True diff --git a/packages/web/src/weaviate_client_web/_sender.py b/packages/web/src/weaviate_client_web/_sender.py index d41f9f6c0..fd2a3488a 100644 --- a/packages/web/src/weaviate_client_web/_sender.py +++ b/packages/web/src/weaviate_client_web/_sender.py @@ -2,11 +2,17 @@ A *sender* is ``async def sender(url, headers, body, timeout) -> (status, headers, body)``. The default uses ``pyodide.http.pyfetch`` (browser fetch); a sender can be injected for -testing or for non-browser runtimes via :func:`weaviate_client_web.set_sender`. +testing via :func:`weaviate_client_web.set_sender`. + +Like the rest of this package, this module imports ``pyodide`` at module scope and is +therefore only importable under Emscripten/Pyodide (or with a ``pyodide`` stand-in +pre-installed in ``sys.modules``). """ from typing import Awaitable, Callable, Dict, Optional, Tuple +from pyodide.http import pyfetch # type: ignore[import-not-found] + Sender = Callable[ [str, Dict[str, str], bytes, Optional[float]], Awaitable[Tuple[int, Dict[str, str], bytes]], @@ -18,12 +24,9 @@ async def pyfetch_sender( ) -> Tuple[int, Dict[str, str], bytes]: """Default browser sender. - Imports ``pyodide.http`` lazily so this module stays importable on CPython (where - ``pyodide`` does not exist). ``pyfetch`` has no timeout parameter of its own; the - call deadline is enforced by ``GrpcWebChannel._unary`` via ``asyncio.wait_for``. + ``pyfetch`` has no timeout parameter of its own; the call deadline is enforced by + ``GrpcWebChannel._unary`` via ``asyncio.wait_for``. """ - from pyodide.http import pyfetch # type: ignore[import-not-found] - response = await pyfetch(url, method="POST", headers=headers, body=body) data = await response.bytes() try: @@ -31,30 +34,3 @@ async def pyfetch_sender( except Exception: # pragma: no cover - header shape varies across Pyodide versions resp_headers = {} return int(response.status), resp_headers, data - - -def make_httpx_sender(client: Optional[object] = None) -> Sender: - """Build a sender backed by ``httpx.AsyncClient`` for CPython tests/integration. - - Targets a grpc-web transcoder (Envoy / connectrpc vanguard). - """ - import httpx - - async def _send( - url: str, headers: Dict[str, str], body: bytes, timeout: Optional[float] - ) -> Tuple[int, Dict[str, str], bytes]: - owns_client = client is None - active = client or httpx.AsyncClient() - assert isinstance(active, httpx.AsyncClient) - try: - response = await active.post(url, headers=headers, content=body, timeout=timeout) - return ( - response.status_code, - {k.lower(): v for k, v in response.headers.items()}, - response.content, - ) - finally: - if owns_client: - await active.aclose() - - return _send diff --git a/packages/web/src/weaviate_client_web/_shim.py b/packages/web/src/weaviate_client_web/_shim.py index b04b8837a..0624f5259 100644 --- a/packages/web/src/weaviate_client_web/_shim.py +++ b/packages/web/src/weaviate_client_web/_shim.py @@ -217,14 +217,14 @@ def is_installed() -> bool: return getattr(sys.modules.get("grpc"), _SHIM_MARKER, False) is True -def install(force: bool = False) -> bool: +def install() -> bool: """Install the shim into ``sys.modules`` as ``grpc`` and submodules. - On normal platforms this is a no-op unless ``force=True`` — we must never clobber a - real, working ``grpcio``. Under Emscripten the bootstrap calls this automatically. - Returns ``True`` if the shim is in place afterwards. + On normal platforms this is a no-op — a real, working ``grpcio`` must be left in + place. Under Emscripten the bootstrap calls this automatically. Returns ``True`` + if the shim is in place afterwards. """ - if not force and sys.platform != "emscripten": + if sys.platform != "emscripten": return False if is_installed(): return True diff --git a/packages/web/tests/conftest.py b/packages/web/tests/conftest.py index fe4afb09d..9bc903df3 100644 --- a/packages/web/tests/conftest.py +++ b/packages/web/tests/conftest.py @@ -1,7 +1,8 @@ -import pathlib -import sys +"""Keep pytest away from this directory. -# Make the package importable without an editable install. -_SRC = pathlib.Path(__file__).resolve().parents[1] / "src" -if str(_SRC) not in sys.path: - sys.path.insert(0, str(_SRC)) +The package under test imports ``pyodide`` at module scope, so the test modules here +are only importable under Emscripten/Pyodide. They are run by +``ci/pyodide-e2e/units.mjs`` (via ``runner.py``), not by pytest. +""" + +collect_ignore_glob = ["*.py"] diff --git a/packages/web/tests/harness.py b/packages/web/tests/harness.py new file mode 100644 index 000000000..c5c8c0ad2 --- /dev/null +++ b/packages/web/tests/harness.py @@ -0,0 +1,53 @@ +"""Shared helpers for the in-Pyodide test modules (pytest is not available here).""" + +import contextlib +import sys +from typing import Any, Optional + + +class _Caught: + value: Any + + +@contextlib.contextmanager +def raises(exc_type, contains: Optional[str] = None): + """``pytest.raises`` stand-in; ``contains`` is a plain substring, not a regex.""" + caught = _Caught() + try: + yield caught + except exc_type as e: + caught.value = e + if contains is not None and contains not in str(e): + raise AssertionError(f"{e!r} does not contain {contains!r}") from e + else: + raise AssertionError(f"expected {exc_type.__name__} to be raised") + + +@contextlib.contextmanager +def patched(obj, name: str, value): + """``monkeypatch.setattr`` stand-in with restore-on-exit.""" + sentinel = object() + old = getattr(obj, name, sentinel) + setattr(obj, name, value) + try: + yield + finally: + if old is sentinel: + delattr(obj, name) + else: + setattr(obj, name, old) + + +@contextlib.contextmanager +def sys_module(name: str, module): + """Set ``sys.modules[name]`` (``None`` makes ``import name`` raise) and restore.""" + present = name in sys.modules + old = sys.modules.get(name) + sys.modules[name] = module + try: + yield + finally: + if present: + sys.modules[name] = old # type: ignore[assignment] + else: + del sys.modules[name] diff --git a/packages/web/tests/runner.py b/packages/web/tests/runner.py new file mode 100644 index 000000000..e9a6ae1fc --- /dev/null +++ b/packages/web/tests/runner.py @@ -0,0 +1,42 @@ +"""Collects and runs the test modules in this directory on Pyodide's event loop. + +Invoked by ``ci/pyodide-e2e/units.mjs``, which mounts this directory into the +interpreter and awaits :func:`main` (``asyncio.run()`` cannot be used inside Pyodide). +Test functions are ``test_*`` callables — plain or ``async`` — collected per module, +so names are qualified and cannot shadow across files. +""" + +import inspect +import traceback +from typing import List + +import test_framing +import test_httpx_fetch +import test_shim_install +import test_transport + +_MODULES = (test_framing, test_transport, test_httpx_fetch, test_shim_install) + + +async def main() -> None: + tests = [] + for module in _MODULES: + for name in sorted(vars(module)): + fn = getattr(module, name) + if name.startswith("test_") and callable(fn): + tests.append((f"{module.__name__}.{name}", fn)) + failures: List[str] = [] + for name, fn in tests: + try: + result = fn() + if inspect.iscoroutine(result): + await result + except Exception: + failures.append(name) + traceback.print_exc() + print(f"FAIL {name}", flush=True) + else: + print(f"OK {name}", flush=True) + print(f"{len(tests) - len(failures)}/{len(tests)} unit tests passed", flush=True) + if failures: + raise SystemExit(f"{len(failures)} unit test(s) failed: {', '.join(failures)}") diff --git a/packages/web/tests/test_framing.py b/packages/web/tests/test_framing.py index fd18a35a3..16aeebe69 100644 --- a/packages/web/tests/test_framing.py +++ b/packages/web/tests/test_framing.py @@ -1,6 +1,8 @@ +"""grpc-web framing tests. Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``.""" + import struct -import pytest +from harness import raises from weaviate_client_web._framing import ( FrameError, @@ -42,7 +44,7 @@ def test_split_response_multiple_messages(): def test_split_response_message_after_trailer_raises(): body = _frame(b"a") + _frame(b"grpc-status:0\r\n", 0x80) + _frame(b"late") - with pytest.raises(FrameError, match="after the trailer"): + with raises(FrameError, contains="after the trailer"): split_response(body) @@ -88,25 +90,25 @@ def test_parse_trailers_accepts_lf_only_lines(): def test_parse_trailers_keeps_status_when_a_key_is_not_ascii(): # one odd key from a proxy must not throw away the whole block - parsed = parse_trailers("x-caf\u00e9:1\r\ngrpc-status:0\r\n".encode("utf-8")) + parsed = parse_trailers("x-café:1\r\ngrpc-status:0\r\n".encode("utf-8")) assert parsed["grpc-status"] == "0" - assert parsed["x-caf\u00e9"] == "1" + assert parsed["x-café"] == "1" def test_truncated_frame_raises(): framed = encode_message(b"hello")[:-2] - with pytest.raises(TruncatedFrameError): + with raises(TruncatedFrameError): list(iter_frames(framed)) - with pytest.raises(TruncatedFrameError): + with raises(TruncatedFrameError): list(iter_frames(b"\x00\x00\x00")) # shorter than one frame header -@pytest.mark.parametrize("first_byte", [b"{", b"<", b"\x02", b"\x40", b"\xff"]) -def test_unknown_frame_flag_raises(first_byte): +def test_unknown_frame_flag_raises(): # a JSON / HTML body, or a flag bit this transport does not know - body = first_byte + b"\x00\x00\x00\x01x" - with pytest.raises(UnknownFrameFlagError, match="unknown grpc-web frame flag"): - list(iter_frames(body)) + for first_byte in (b"{", b"<", b"\x02", b"\x40", b"\xff"): + body = first_byte + b"\x00\x00\x00\x01x" + with raises(UnknownFrameFlagError, contains="unknown grpc-web frame flag"): + list(iter_frames(body)) def test_frame_errors_are_value_errors(): @@ -116,5 +118,5 @@ def test_frame_errors_are_value_errors(): def test_compressed_message_frame_rejected(): body = _frame(b"x", 0x01) - with pytest.raises(FrameError, match="compressed"): + with raises(FrameError, contains="compressed"): split_response(body) diff --git a/packages/web/tests/test_httpx_fetch.py b/packages/web/tests/test_httpx_fetch.py index c34f8e35d..c12a29d5f 100644 --- a/packages/web/tests/test_httpx_fetch.py +++ b/packages/web/tests/test_httpx_fetch.py @@ -1,30 +1,24 @@ -"""Tests for the fetch-based httpx transport (_httpx_fetch.py). +"""Tests for the fetch-based httpx transport (``_httpx_fetch.py``). -In-process tests call ``_fetch_handle_async_request`` directly with a fake -``pyodide.http`` module injected into ``sys.modules`` — no global monkeypatch of -``httpx.AsyncHTTPTransport`` is needed, so the real httpx in the dev environment is left -untouched. Install semantics (which DO patch the class globally) run in fresh -subprocesses, mirroring test_shim_install.py. +Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``. ``pyfetch`` is bound at import +time in ``_httpx_fetch``, so fakes patch that module attribute; ``from js import +AbortSignal`` is resolved per request, so a fake js module in ``sys.modules`` +intercepts it even under real Pyodide. + +The install-semantics tests at the bottom run against this interpreter's real +installation: importing ``weaviate_client_web`` bootstrapped the transport globally. """ -import asyncio -import pathlib -import subprocess -import sys -import textwrap +import contextlib import types from typing import Any, Dict, List, Optional import httpx -import pytest - -from weaviate_client_web._httpx_fetch import ( - _MAX_ABORT_SIGNAL_MS, - _abort_signal_ms, - _fetch_handle_async_request, -) +from harness import patched, raises, sys_module -_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") +import weaviate_client_web +import weaviate_client_web._httpx_fetch as _httpx_fetch +from weaviate_client_web._httpx_fetch import _MAX_ABORT_SIGNAL_MS, _abort_signal_ms class FakeFetchResponse: @@ -50,105 +44,125 @@ async def __call__(self, url: str, **kwargs: Any) -> FakeFetchResponse: return self.response -@pytest.fixture -def fake_pyfetch(monkeypatch) -> FakePyfetch: +@contextlib.contextmanager +def fake_pyfetch(): fetch = FakePyfetch() - pyodide_mod = types.ModuleType("pyodide") - http_mod = types.ModuleType("pyodide.http") - http_mod.pyfetch = fetch # type: ignore[attr-defined] - pyodide_mod.http = http_mod # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "pyodide", pyodide_mod) - monkeypatch.setitem(sys.modules, "pyodide.http", http_mod) - return fetch + with patched(_httpx_fetch, "pyfetch", fetch): + yield fetch + + +class _AbortSignalRecorder: + def __init__(self): + self.timeouts: List[int] = [] + + def timeout(self, ms: int): + self.timeouts.append(ms) + return f"signal-{ms}" + + +@contextlib.contextmanager +def fake_abort_signal(): + recorder = _AbortSignalRecorder() + js_mod = types.ModuleType("js") + js_mod.AbortSignal = recorder # type: ignore[attr-defined] + with sys_module("js", js_mod): + yield recorder + + +@contextlib.contextmanager +def missing_js_module(): + # sys.modules[name] = None makes ``import js`` raise ImportError — the closest + # in-process stand-in for an environment without the js bridge. + with sys_module("js", None): + yield -async def _handle_async(request: httpx.Request) -> httpx.Response: +async def _handle(request: httpx.Request) -> httpx.Response: # self is unused by the handler implementation; a bare transport instance suffices transport = httpx.AsyncHTTPTransport.__new__(httpx.AsyncHTTPTransport) - response = await _fetch_handle_async_request(transport, request) + response = await _httpx_fetch._fetch_handle_async_request(transport, request) await response.aread() # httpx.AsyncClient reads non-streamed responses the same way return response -def _handle(request: httpx.Request) -> httpx.Response: - return asyncio.run(_handle_async(request)) - - class _FetchTransport(httpx.AsyncBaseTransport): - """Route an ``httpx.AsyncClient`` through the handler without patching httpx globally.""" + """Route an ``httpx.AsyncClient`` through the handler without touching global state.""" async def handle_async_request(self, request: httpx.Request) -> httpx.Response: - return await _fetch_handle_async_request(self, request) # type: ignore[arg-type] - + return await _httpx_fetch._fetch_handle_async_request(self, request) # type: ignore[arg-type] -def _via_client(method: str, url: str, **kwargs: Any) -> httpx.Response: - async def main() -> httpx.Response: - async with httpx.AsyncClient(transport=_FetchTransport()) as client: - return await client.request(method, url, **kwargs) - return asyncio.run(main()) +async def _via_client(method: str, url: str, **kwargs: Any) -> httpx.Response: + async with httpx.AsyncClient(transport=_FetchTransport()) as client: + return await client.request(method, url, **kwargs) -def test_basic_get_round_trip(fake_pyfetch): - fake_pyfetch.response = FakeFetchResponse( - status=200, headers={"content-type": "application/json"}, body=b'{"version": "1.30.0"}' - ) - response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) +async def test_basic_get_round_trip(): + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse( + status=200, headers={"content-type": "application/json"}, body=b'{"version": "1.30.0"}' + ) + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert response.status_code == 200 - assert response.json() == {"version": "1.30.0"} - assert response.headers["content-type"] == "application/json" - call = fake_pyfetch.calls[0] - assert call["url"] == "http://h:8080/v1/meta" - assert call["method"] == "GET" + assert response.status_code == 200 + assert response.json() == {"version": "1.30.0"} + assert response.headers["content-type"] == "application/json" + call = fetch.calls[0] + assert call["url"] == "http://h:8080/v1/meta" + assert call["method"] == "GET" -def test_response_has_request_attached_for_raise_for_status(fake_pyfetch): - fake_pyfetch.response = FakeFetchResponse(status=404, body=b"") - response = _handle(httpx.Request("GET", "http://h:8080/v1/schema/Nope")) - with pytest.raises(httpx.HTTPStatusError): - response.raise_for_status() +async def test_response_has_request_attached_for_raise_for_status(): + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse(status=404, body=b"") + response = await _handle(httpx.Request("GET", "http://h:8080/v1/schema/Nope")) + with raises(httpx.HTTPStatusError): + response.raise_for_status() -@pytest.mark.parametrize("status", [204, 404]) -def test_head_response_without_body_yields_empty_content(fake_pyfetch, status): +async def test_head_response_without_body_yields_empty_content(): # data.exists() / tenants.exists() are HEAD requests answered 204/404 with a null # body; the transport must hand httpx an empty response, not fail on the missing body - fake_pyfetch.response = FakeFetchResponse(status=status, body=None) - response = _handle(httpx.Request("HEAD", "http://h:8080/v1/objects/A/uuid")) - assert response.status_code == status - assert response.content == b"" - assert "body" not in fake_pyfetch.calls[0] + for status in (204, 404): + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse(status=status, body=None) + response = await _handle(httpx.Request("HEAD", "http://h:8080/v1/objects/A/uuid")) + assert response.status_code == status + assert response.content == b"" + assert "body" not in fetch.calls[0] -def test_delete_204_without_body_yields_empty_content(fake_pyfetch): +async def test_delete_204_without_body_yields_empty_content(): # data.delete_by_id() / reference_delete() are answered 204 with a null body - fake_pyfetch.response = FakeFetchResponse(status=204, body=None) - response = _handle(httpx.Request("DELETE", "http://h:8080/v1/objects/A/uuid")) - assert response.status_code == 204 - assert response.content == b"" + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse(status=204, body=None) + response = await _handle(httpx.Request("DELETE", "http://h:8080/v1/objects/A/uuid")) + assert response.status_code == 204 + assert response.content == b"" -def test_body_less_response_through_async_client(fake_pyfetch): +async def test_body_less_response_through_async_client(): # the full httpx.AsyncClient path (stream wrapping + read) on a body-less response - fake_pyfetch.response = FakeFetchResponse(status=204, body=None) - response = _via_client("HEAD", "http://h:8080/v1/objects/A/uuid") - assert response.status_code == 204 - assert response.content == b"" + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse(status=204, body=None) + response = await _via_client("HEAD", "http://h:8080/v1/objects/A/uuid") + assert response.status_code == 204 + assert response.content == b"" -def test_response_through_async_client_exposes_elapsed_and_content(fake_pyfetch): +async def test_response_through_async_client_exposes_elapsed_and_content(): # the batch-references path reads ``res.elapsed``, which httpx only sets after it # has read/closed a stream-backed response; a pre-loaded body never gets one payload = b'[{"result": {"status": "SUCCESS"}}]' - fake_pyfetch.response = FakeFetchResponse(status=200, body=payload) - response = _via_client("POST", "http://h:8080/v1/batch/references", content=b"[]") - assert response.content == payload - assert response.json() == [{"result": {"status": "SUCCESS"}}] - assert response.elapsed.total_seconds() >= 0 + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse(status=200, body=payload) + response = await _via_client("POST", "http://h:8080/v1/batch/references", content=b"[]") + assert response.content == payload + assert response.json() == [{"result": {"status": "SUCCESS"}}] + assert response.elapsed.total_seconds() >= 0 -def test_fetch_managed_request_headers_stripped(fake_pyfetch): +async def test_fetch_managed_request_headers_stripped(): request = httpx.Request( "POST", "http://h:8080/v1/objects", @@ -162,79 +176,76 @@ def test_fetch_managed_request_headers_stripped(fake_pyfetch): }, content=b"{}", ) - _handle(request) - sent = fake_pyfetch.calls[0]["headers"] - assert sent["authorization"] == "Bearer k" - assert sent["content-type"] == "application/json" - for managed in ("host", "connection", "accept-encoding", "content-length", "transfer-encoding"): - assert managed not in sent - - -def test_get_without_body_omits_body_kwarg(fake_pyfetch): + with fake_pyfetch() as fetch: + await _handle(request) + sent = fetch.calls[0]["headers"] + assert sent["authorization"] == "Bearer k" + assert sent["content-type"] == "application/json" + for managed in ( + "host", + "connection", + "accept-encoding", + "content-length", + "transfer-encoding", + ): + assert managed not in sent, managed + + +async def test_get_without_body_omits_body_kwarg(): # fetch rejects GET/HEAD requests that carry a body, so the kwarg must be absent - _handle(httpx.Request("GET", "http://h:8080/v1/.well-known/ready")) - assert "body" not in fake_pyfetch.calls[0] + with fake_pyfetch() as fetch: + await _handle(httpx.Request("GET", "http://h:8080/v1/.well-known/ready")) + assert "body" not in fetch.calls[0] -def test_post_body_passed(fake_pyfetch): - _handle(httpx.Request("POST", "http://h:8080/v1/graphql", content=b'{"query": "x"}')) - assert fake_pyfetch.calls[0]["body"] == b'{"query": "x"}' +async def test_post_body_passed(): + with fake_pyfetch() as fetch: + await _handle(httpx.Request("POST", "http://h:8080/v1/graphql", content=b'{"query": "x"}')) + assert fetch.calls[0]["body"] == b'{"query": "x"}' -def test_delete_with_body_passed(fake_pyfetch): +async def test_delete_with_body_passed(): # the REST batch-delete path sends DELETE with a JSON body - _handle(httpx.Request("DELETE", "http://h:8080/v1/batch/objects", content=b'{"match": {}}')) - assert fake_pyfetch.calls[0]["body"] == b'{"match": {}}' + with fake_pyfetch() as fetch: + await _handle( + httpx.Request("DELETE", "http://h:8080/v1/batch/objects", content=b'{"match": {}}') + ) + assert fetch.calls[0]["body"] == b'{"match": {}}' -def test_query_string_preserved_in_url(fake_pyfetch): - _handle(httpx.Request("GET", "http://h:8080/v1/objects?class=A&limit=10&after=a%20b")) - assert fake_pyfetch.calls[0]["url"] == "http://h:8080/v1/objects?class=A&limit=10&after=a%20b" +async def test_query_string_preserved_in_url(): + with fake_pyfetch() as fetch: + await _handle(httpx.Request("GET", "http://h:8080/v1/objects?class=A&limit=10&after=a%20b")) + assert fetch.calls[0]["url"] == "http://h:8080/v1/objects?class=A&limit=10&after=a%20b" -def test_content_encoding_stripped_from_response(fake_pyfetch): +async def test_content_encoding_stripped_from_response(): # fetch hands back ALREADY-decompressed bytes; if the original content-encoding # header were passed through, httpx.Response would gunzip a second time and raise # DecodingError. content-length is stale for the same reason. - fake_pyfetch.response = FakeFetchResponse( - status=200, - headers={"content-encoding": "gzip", "content-length": "23", "x-other": "kept"}, - body=b'{"version": "1.30.0"}', - ) - response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert response.json() == {"version": "1.30.0"} - assert "content-encoding" not in response.headers - assert response.headers["x-other"] == "kept" + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse( + status=200, + headers={"content-encoding": "gzip", "content-length": "23", "x-other": "kept"}, + body=b'{"version": "1.30.0"}', + ) + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert response.json() == {"version": "1.30.0"} + assert "content-encoding" not in response.headers + assert response.headers["x-other"] == "kept" -def test_unreadable_response_headers_tolerated(fake_pyfetch): +async def test_unreadable_response_headers_tolerated(): class BadHeaders: def keys(self): raise TypeError("header shape varies across Pyodide versions") - fake_pyfetch.response = FakeFetchResponse(status=200, body=b"ok") - fake_pyfetch.response.headers = BadHeaders() - response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert response.status_code == 200 - assert response.content == b"ok" - - -class _AbortSignalRecorder: - def __init__(self): - self.timeouts: List[int] = [] - - def timeout(self, ms: int): - self.timeouts.append(ms) - return f"signal-{ms}" - - -@pytest.fixture -def fake_abort_signal(monkeypatch) -> _AbortSignalRecorder: - recorder = _AbortSignalRecorder() - js_mod = types.ModuleType("js") - js_mod.AbortSignal = recorder # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "js", js_mod) - return recorder + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse(status=200, body=b"ok") + fetch.response.headers = BadHeaders() + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert response.status_code == 200 + assert response.content == b"ok" def _request_with_timeout(timeouts: Dict[str, Optional[float]]) -> httpx.Request: @@ -243,58 +254,68 @@ def _request_with_timeout(timeouts: Dict[str, Optional[float]]) -> httpx.Request return request -def test_read_timeout_maps_to_abort_signal_ms(fake_pyfetch, fake_abort_signal): +async def test_read_timeout_maps_to_abort_signal_ms(): # mirrors what weaviate's AsyncClient puts in extensions: connect/read/write/pool - _handle(_request_with_timeout({"connect": 2.0, "read": 30.0, "write": 5.0, "pool": 9.0})) - assert fake_abort_signal.timeouts == [30000] - assert fake_pyfetch.calls[0]["signal"] == "signal-30000" + with fake_pyfetch() as fetch, fake_abort_signal() as signals: + await _handle( + _request_with_timeout({"connect": 2.0, "read": 30.0, "write": 5.0, "pool": 9.0}) + ) + assert signals.timeouts == [30000] + assert fetch.calls[0]["signal"] == "signal-30000" -def test_read_none_means_no_deadline_even_with_pool_and_connect_set( - fake_pyfetch, fake_abort_signal -): +async def test_read_none_means_no_deadline_even_with_pool_and_connect_set(): # what the base client hands over for a non-finite request timeout: read=None with the # session pool timeout still set; falling back to pool/connect would abort a long # insert after 5 s - _handle(_request_with_timeout({"connect": None, "read": None, "write": None, "pool": 5})) - _handle(_request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0})) - assert fake_abort_signal.timeouts == [] - assert all("signal" not in c for c in fake_pyfetch.calls) + with fake_pyfetch() as fetch, fake_abort_signal() as signals: + await _handle( + _request_with_timeout({"connect": None, "read": None, "write": None, "pool": 5}) + ) + await _handle( + _request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0}) + ) + assert signals.timeouts == [] + assert all("signal" not in c for c in fetch.calls) -def test_read_timeout_alone_sets_the_deadline(fake_pyfetch, fake_abort_signal): - _handle(_request_with_timeout({"connect": None, "read": 7, "write": None, "pool": 5})) - assert fake_abort_signal.timeouts == [7000] - assert fake_pyfetch.calls[0]["signal"] == "signal-7000" +async def test_read_timeout_alone_sets_the_deadline(): + with fake_pyfetch() as fetch, fake_abort_signal() as signals: + await _handle(_request_with_timeout({"connect": None, "read": 7, "write": None, "pool": 5})) + assert signals.timeouts == [7000] + assert fetch.calls[0]["signal"] == "signal-7000" -def test_no_timeout_extension_sends_no_signal(fake_pyfetch, fake_abort_signal): - _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert fake_abort_signal.timeouts == [] - assert "signal" not in fake_pyfetch.calls[0] +async def test_no_timeout_extension_sends_no_signal(): + with fake_pyfetch() as fetch, fake_abort_signal() as signals: + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert signals.timeouts == [] + assert "signal" not in fetch.calls[0] -def test_missing_js_module_degrades_to_no_signal(fake_pyfetch): - # off-browser (no js module) the AbortSignal import fails; the request must still go out - assert "js" not in sys.modules - response = _handle( - _request_with_timeout({"connect": 2.0, "read": 30.0, "write": None, "pool": None}) - ) - assert response.status_code == 200 - assert "signal" not in fake_pyfetch.calls[0] +async def test_missing_js_module_degrades_to_no_signal(): + # without the js bridge the AbortSignal import fails; the request must still go out + with fake_pyfetch() as fetch, missing_js_module(): + response = await _handle( + _request_with_timeout({"connect": 2.0, "read": 30.0, "write": None, "pool": None}) + ) + assert response.status_code == 200 + assert "signal" not in fetch.calls[0] -def test_zero_timeout_means_no_deadline(fake_pyfetch, fake_abort_signal): +async def test_zero_timeout_means_no_deadline(): # an explicit read=0 must not fall through to the 5s connect timeout, nor become an # immediate AbortSignal.timeout(0) - _handle(_request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None})) - assert fake_abort_signal.timeouts == [] - assert "signal" not in fake_pyfetch.calls[0] + with fake_pyfetch() as fetch, fake_abort_signal() as signals: + await _handle( + _request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None}) + ) + assert signals.timeouts == [] + assert "signal" not in fetch.calls[0] -@pytest.mark.parametrize( - "timeout,expected_ms", - [ +def test_abort_signal_ms_bounds(): + cases = [ (None, None), (0, None), (-1, None), @@ -305,26 +326,29 @@ def test_zero_timeout_means_no_deadline(fake_pyfetch, fake_abort_signal): (1e8, _MAX_ABORT_SIGNAL_MS), (1e10, _MAX_ABORT_SIGNAL_MS), (1e308, _MAX_ABORT_SIGNAL_MS), # finite, but *1000 overflows: capped, not an error - ], -) -def test_abort_signal_ms_bounds(timeout, expected_ms): - assert _abort_signal_ms(timeout) == expected_ms + ] + for timeout, expected_ms in cases: + assert _abort_signal_ms(timeout) == expected_ms, timeout -def test_infinite_timeout_sends_no_signal(fake_pyfetch, fake_abort_signal): +async def test_infinite_timeout_sends_no_signal(): # an inf read deadline reaching the transport: no signal, not an OverflowError - _handle( - _request_with_timeout({"connect": None, "read": float("inf"), "write": None, "pool": 5}) - ) - assert fake_abort_signal.timeouts == [] - assert "signal" not in fake_pyfetch.calls[0] + with fake_pyfetch() as fetch, fake_abort_signal() as signals: + await _handle( + _request_with_timeout({"connect": None, "read": float("inf"), "write": None, "pool": 5}) + ) + assert signals.timeouts == [] + assert "signal" not in fetch.calls[0] -def test_huge_timeout_is_capped_to_int32_ms(fake_pyfetch, fake_abort_signal): +async def test_huge_timeout_is_capped_to_int32_ms(): # setTimeout delays above 2^31-1 ms overflow and fire at once, aborting the request - _handle(_request_with_timeout({"connect": None, "read": 1e10, "write": None, "pool": None})) - assert fake_abort_signal.timeouts == [_MAX_ABORT_SIGNAL_MS] - assert fake_pyfetch.calls[0]["signal"] == f"signal-{_MAX_ABORT_SIGNAL_MS}" + with fake_pyfetch() as fetch, fake_abort_signal() as signals: + await _handle( + _request_with_timeout({"connect": None, "read": 1e10, "write": None, "pool": None}) + ) + assert signals.timeouts == [_MAX_ABORT_SIGNAL_MS] + assert fetch.calls[0]["signal"] == f"signal-{_MAX_ABORT_SIGNAL_MS}" class RaisingPyfetch: @@ -335,271 +359,123 @@ async def __call__(self, url: str, **kwargs: Any): raise self.exc -def _install_raising_pyfetch(monkeypatch, exc: BaseException) -> None: - pyodide_mod = types.ModuleType("pyodide") - http_mod = types.ModuleType("pyodide.http") - http_mod.pyfetch = RaisingPyfetch(exc) # type: ignore[attr-defined] - pyodide_mod.http = http_mod # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "pyodide", pyodide_mod) - monkeypatch.setitem(sys.modules, "pyodide.http", http_mod) +@contextlib.contextmanager +def raising_pyfetch(exc: BaseException): + with patched(_httpx_fetch, "pyfetch", RaisingPyfetch(exc)): + yield -def test_fetch_failure_maps_to_httpx_connect_error(monkeypatch): +async def test_fetch_failure_maps_to_httpx_connect_error(): # pyodide surfaces JS fetch rejections as OSError; the base client can only classify # httpx exceptions (WeaviateConnectionError etc.), so the shim must translate - _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) - with pytest.raises(httpx.ConnectError, match="Failed to fetch") as excinfo: - _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert isinstance(excinfo.value.__cause__, OSError) + with raising_pyfetch(OSError("TypeError: Failed to fetch")): + with raises(httpx.ConnectError, contains="Failed to fetch") as excinfo: + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert isinstance(excinfo.value.__cause__, OSError) -def test_fetch_abort_with_deadline_maps_to_read_timeout(monkeypatch, fake_abort_signal): +async def test_fetch_abort_with_deadline_maps_to_read_timeout(): # AbortSignal.timeout firing surfaces as an OSError subclass mentioning the abort; # with a deadline set this must classify as a timeout, not a connection error - _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) - with pytest.raises(httpx.ReadTimeout, match="signal timed out"): - _handle(_request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None})) + with raising_pyfetch(OSError("AbortError: signal timed out")), fake_abort_signal(): + with raises(httpx.ReadTimeout, contains="signal timed out"): + await _handle( + _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) + ) -def test_fetch_failure_with_deadline_but_no_timeout_message_stays_connect_error( - monkeypatch, fake_abort_signal -): +async def test_fetch_failure_with_deadline_but_no_timeout_message_stays_connect_error(): # nearly every weaviate request sets a read deadline; a plain network failure on # such a request must remain a connection error, not become a timeout - _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) - with pytest.raises(httpx.ConnectError, match="Failed to fetch"): - _handle(_request_with_timeout({"connect": None, "read": 30.0, "write": None, "pool": None})) + with raising_pyfetch(OSError("TypeError: Failed to fetch")), fake_abort_signal(): + with raises(httpx.ConnectError, contains="Failed to fetch"): + await _handle( + _request_with_timeout({"connect": None, "read": 30.0, "write": None, "pool": None}) + ) -def test_fetch_abort_without_deadline_stays_connect_error(monkeypatch): - # the same message without a deadline set (no js module -> no signal) is not OUR +async def test_fetch_abort_without_deadline_stays_connect_error(): + # the same message without a deadline set (no js bridge -> no signal) is not OUR # timeout, so it must stay a connection error - _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) - assert "js" not in sys.modules - with pytest.raises(httpx.ConnectError): - _handle(_request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None})) + with raising_pyfetch(OSError("AbortError: signal timed out")), missing_js_module(): + with raises(httpx.ConnectError): + await _handle( + _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) + ) -def test_empty_oserror_str_keeps_repr_detail(monkeypatch): - _install_raising_pyfetch(monkeypatch, OSError()) - with pytest.raises(httpx.ConnectError) as excinfo: - _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert "OSError" in str(excinfo.value) +async def test_empty_oserror_str_keeps_repr_detail(): + with raising_pyfetch(OSError()): + with raises(httpx.ConnectError) as excinfo: + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert "OSError" in str(excinfo.value) -def test_crlf_in_header_value_rejected(fake_pyfetch): +async def test_crlf_in_header_value_rejected(): # httpx.Request accepts CR/LF in header values and relies on h11 to reject them at # send time; this transport bypasses h11 and must keep that defence request = httpx.Request( "GET", "http://h:8080/v1/meta", headers={"x-key": "val\r\nx-injected: evil"} ) - with pytest.raises(httpx.LocalProtocolError): - _handle(request) - assert fake_pyfetch.calls == [] + with fake_pyfetch() as fetch: + with raises(httpx.LocalProtocolError): + await _handle(request) + assert fetch.calls == [] # --------------------------------------------------------------------------- -# Install semantics: these patch httpx.AsyncHTTPTransport globally, so each -# scenario runs in a fresh subprocess (same pattern as test_shim_install.py). +# install semantics, against this interpreter's real installation # --------------------------------------------------------------------------- -_FAKE_PYODIDE_PRELUDE = """ -import sys, types - -class _FakeResponse: - status = 200 - headers = {"content-type": "application/json"} - async def bytes(self): - return b'{"ok": true}' - -CALLS = [] -async def pyfetch(url, **kwargs): - CALLS.append((url, kwargs)) - return _FakeResponse() - -_pyodide = types.ModuleType("pyodide") -_http = types.ModuleType("pyodide.http") -_http.pyfetch = pyfetch -_pyodide.http = _http -sys.modules["pyodide"] = _pyodide -sys.modules["pyodide.http"] = _http -""" - - -def _run(body: str, prelude: str = "") -> subprocess.CompletedProcess: - script = f"import sys\nsys.path.insert(0, {_SRC!r})\n" + prelude + textwrap.dedent(body) - return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) +def test_bootstrap_installed_fetch_transport(): + # This interpreter imported weaviate_client_web at the top of this file, so the real + # bootstrap ran — even though Pyodide's bundled httpx carries its own jsfetch + # transport, the package's transport must be the active one. + assert weaviate_client_web.is_fetch_transport_installed() + assert ( + getattr(httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False) + is True + ) -def test_force_install_routes_async_client_through_pyfetch(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import asyncio, httpx - from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed - - install_fetch_transport(force=True) - assert is_fetch_transport_installed() - async def main(): - async with httpx.AsyncClient() as client: - return await client.get("http://h:8080/v1/meta") - - resp = asyncio.run(main()) - assert resp.status_code == 200, resp.status_code - assert resp.json() == {"ok": True} - assert CALLS and CALLS[0][0] == "http://h:8080/v1/meta" - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_install_without_force_is_noop_off_emscripten(): - result = _run( - """ - import sys - assert sys.platform != "emscripten" - import httpx - before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed - install_fetch_transport() - assert not is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is before - print("OK") - """ - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_force_install_is_idempotent(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import httpx - from weaviate_client_web import install_fetch_transport - install_fetch_transport(force=True) - patched = httpx.AsyncHTTPTransport.handle_async_request - install_fetch_transport(force=True) - assert httpx.AsyncHTTPTransport.handle_async_request is patched - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout +def test_install_fetch_transport_is_idempotent(): + patched_method = httpx.AsyncHTTPTransport.handle_async_request + weaviate_client_web.install_fetch_transport() + assert httpx.AsyncHTTPTransport.handle_async_request is patched_method def test_sync_transport_left_untouched(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import httpx - sync_before = httpx.HTTPTransport.handle_request - from weaviate_client_web import install_fetch_transport - install_fetch_transport(force=True) - assert httpx.HTTPTransport.handle_request is sync_before - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout + assert not getattr(httpx.HTTPTransport.handle_request, "__weaviate_fetch_shim__", False) def test_uninstall_restores_original_transport(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import httpx - before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_client_web import ( - install_fetch_transport, - is_fetch_transport_installed, - uninstall_fetch_transport, - ) - uninstall_fetch_transport() # no-op when not installed - install_fetch_transport(force=True) - assert is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is not before - uninstall_fetch_transport() - assert not is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is before - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_patched_method_carries_sentinel(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import httpx - from weaviate_client_web import install_fetch_transport + patched_method = httpx.AsyncHTTPTransport.handle_async_request + try: + weaviate_client_web.uninstall_fetch_transport() + assert not weaviate_client_web.is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is not patched_method assert not getattr( httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False ) - install_fetch_transport(force=True) - assert getattr( - httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False - ) is True - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_force_install_without_pyodide_fails_fast(): - # without a pyodide module the install must raise immediately, not let every later - # request die with a lazy ModuleNotFoundError - result = _run( - """ - import httpx - before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed - try: - install_fetch_transport(force=True) - except ModuleNotFoundError: - assert not is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is before - print("OK") - else: - raise AssertionError("expected install to fail fast without pyodide") - """ + weaviate_client_web.uninstall_fetch_transport() # no-op when not installed + finally: + weaviate_client_web.install_fetch_transport() + assert weaviate_client_web.is_fetch_transport_installed() + assert ( + getattr(httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False) + is True ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_emscripten_installs_even_when_platform_httpx_has_jsfetch(): - # Pyodide's bundled httpx ships a jsfetch transport that crashes on body-less - # responses (HEAD / 204); the shim must take over regardless of the httpx build - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import importlib.machinery, sys, types - - sys.platform = "emscripten" - fake = types.ModuleType("httpx._transports.jsfetch") - fake.__spec__ = importlib.machinery.ModuleSpec( - "httpx._transports.jsfetch", loader=None + + +async def test_installed_transport_routes_async_client_through_pyfetch(): + # the globally installed transport (no custom transport argument) must reach pyfetch + with fake_pyfetch() as fetch: + fetch.response = FakeFetchResponse( + status=200, headers={"content-type": "application/json"}, body=b'{"ok": true}' ) - sys.modules["httpx._transports.jsfetch"] = fake - - import httpx - before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed - install_fetch_transport() # no force: the platform alone must trigger it - assert is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is not before - assert getattr( - httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False - ) is True - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout + async with httpx.AsyncClient() as client: + response = await client.get("http://h:8080/v1/meta") + assert response.status_code == 200 + assert response.json() == {"ok": True} + assert fetch.calls and fetch.calls[0]["url"] == "http://h:8080/v1/meta" diff --git a/packages/web/tests/test_shim_install.py b/packages/web/tests/test_shim_install.py index cdcbed56d..d5dede428 100644 --- a/packages/web/tests/test_shim_install.py +++ b/packages/web/tests/test_shim_install.py @@ -1,123 +1,83 @@ -"""Shim/import tests. +"""grpc shim tests, against this interpreter's real installation. -Installing the shim replaces ``sys.modules['grpc']`` process-wide, so each scenario runs -in a fresh subprocess to avoid clobbering the real ``grpc`` used by the rest of the suite. +Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``: importing ``weaviate_client_web`` +bootstrapped the shim, so ``sys.modules['grpc']`` here IS the shim. Bootstrap scenarios +that need a clean import state (a bare ``import weaviate``, a missing or broken +companion) live in ``units.mjs``, one fresh interpreter each. """ -import pathlib -import subprocess -import sys -import textwrap +import struct -_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") +from harness import raises +import weaviate_client_web +from weaviate_client_web import GrpcWebChannel, set_sender +from weaviate_client_web._sender import pyfetch_sender +from weaviate_client_web._shim import FAKE_GRPC_VERSION -def _run(body: str) -> subprocess.CompletedProcess: - script = f"import sys\nsys.path.insert(0, {_SRC!r})\n" + textwrap.dedent(body) - return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + +def _frame(payload: bytes, flag: int = 0x00) -> bytes: + return struct.pack(">BI", flag, len(payload)) + payload def test_import_weaviate_under_shim(): - result = _run( - """ - import weaviate_client_web - assert weaviate_client_web.install(force=True) is True - assert weaviate_client_web.is_installed() - - import grpc - assert getattr(grpc, "__weaviate_client_web_shim__", False) is True - assert grpc.__version__ == "1.72.1" - assert grpc._utilities.first_version_is_lower("1.0.0", "2.0.0") is False - from grpc.aio._typing import ChannelArgumentType # noqa: F401 - - import weaviate # must not raise even though grpcio is shimmed - from weaviate.proto.v1 import weaviate_pb2_grpc - from weaviate_client_web import GrpcWebChannel + import grpc - ch = GrpcWebChannel("localhost:50051", secure=False) - stub = weaviate_pb2_grpc.WeaviateStub(ch) - assert stub.Search is not None - assert stub.BatchObjects is not None - assert stub.BatchDelete is not None - assert isinstance(ch, grpc.aio.Channel) - print("OK") - """ - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout + assert weaviate_client_web.is_installed() + assert getattr(grpc, "__weaviate_client_web_shim__", False) is True + assert grpc.__version__ == FAKE_GRPC_VERSION + assert grpc._utilities.first_version_is_lower("1.0.0", "2.0.0") is False # type: ignore[attr-defined] + from grpc.aio._typing import ChannelArgumentType # noqa: F401 + + import weaviate # noqa: F401 # must not raise even though grpcio is shimmed + from weaviate.proto.v1 import weaviate_pb2_grpc + + ch = GrpcWebChannel("localhost:50051", secure=False) + stub = weaviate_pb2_grpc.WeaviateStub(ch) + assert stub.Search is not None + assert stub.BatchObjects is not None + assert stub.BatchDelete is not None + assert isinstance(ch, grpc.aio.Channel) def test_sync_channel_factory_raises_async_only(): - result = _run( - """ - import weaviate_client_web - weaviate_client_web.install(force=True) - import grpc - try: - grpc.insecure_channel("localhost:50051") - except RuntimeError as exc: - assert "async" in str(exc).lower() - print("OK") - else: - raise AssertionError("expected sync channel factory to raise") - """ - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_real_proto_unary_round_trip_under_shim(): - result = _run( - """ - import asyncio - import struct - import weaviate_client_web - weaviate_client_web.install(force=True) - - import weaviate # noqa: F401 - from weaviate.proto.v1 import tenants_pb2, weaviate_pb2_grpc - - reply = tenants_pb2.TenantsGetReply() - payload = reply.SerializeToString() - - def frame(p, flag=0x00): - return struct.pack(">BI", flag, len(p)) + p - - body = frame(payload) + frame(b"grpc-status:0\\r\\n", 0x80) - - async def sender(url, headers, body_in, timeout): - assert headers["authorization"] == "Bearer k" - assert url.endswith("/weaviate.v1.Weaviate/TenantsGet") - return 200, {}, body - - weaviate_client_web.set_sender(sender) - from weaviate_client_web import GrpcWebChannel - ch = GrpcWebChannel("localhost:50051", secure=False) - stub = weaviate_pb2_grpc.WeaviateStub(ch) + import grpc - async def main(): - res = await stub.TenantsGet( - tenants_pb2.TenantsGetRequest(), - metadata=[("authorization", "Bearer k")], - timeout=5, - ) - assert isinstance(res, tenants_pb2.TenantsGetReply) - print("OK") + with raises(RuntimeError) as excinfo: + grpc.insecure_channel("localhost:50051") + assert "async" in str(excinfo.value).lower() - asyncio.run(main()) - """ - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout + +async def test_real_proto_unary_round_trip_under_shim(): + from weaviate.proto.v1 import tenants_pb2, weaviate_pb2_grpc + + reply = tenants_pb2.TenantsGetReply() + payload = reply.SerializeToString() + body = _frame(payload) + _frame(b"grpc-status:0\r\n", 0x80) + + async def sender(url, headers, body_in, timeout): + assert headers["authorization"] == "Bearer k" + assert url.endswith("/weaviate.v1.Weaviate/TenantsGet") + return 200, {}, body + + set_sender(sender) + try: + ch = GrpcWebChannel("localhost:50051", secure=False) + stub = weaviate_pb2_grpc.WeaviateStub(ch) + res = await stub.TenantsGet( + tenants_pb2.TenantsGetRequest(), + metadata=[("authorization", "Bearer k")], + timeout=5, + ) + assert isinstance(res, tenants_pb2.TenantsGetReply) + finally: + set_sender(pyfetch_sender) def test_fake_grpc_version_matches_base_fallback(): - # In-process on purpose: nothing here installs the shim, we only compare the two - # copies of the pinned version. The shim advertises FAKE_GRPC_VERSION as - # grpc.__version__ and the base package falls back to _GRPCIO_FALLBACK_VERSION - # under Emscripten — the vendored stubs' version gates see both, so they must - # never drift apart. + # The shim advertises FAKE_GRPC_VERSION as grpc.__version__ and the base package + # falls back to _GRPCIO_FALLBACK_VERSION under Emscripten — the vendored stubs' + # version gates see both, so they must never drift apart. from weaviate.proto.v1 import _GRPCIO_FALLBACK_VERSION - from weaviate_client_web._shim import FAKE_GRPC_VERSION assert FAKE_GRPC_VERSION == _GRPCIO_FALLBACK_VERSION diff --git a/packages/web/tests/test_single_import.py b/packages/web/tests/test_single_import.py deleted file mode 100644 index 562f31e0a..000000000 --- a/packages/web/tests/test_single_import.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Tests for the single-import hook in the base client (``weaviate/__init__.py``). - -The hook fires on ``sys.platform == "emscripten"`` and (via the companion's bootstrap) -replaces ``sys.modules['grpc']`` process-wide, so each scenario runs in a fresh -subprocess with the platform faked before ``import weaviate`` — the same pattern as -test_shim_install.py / test_httpx_fetch.py's install tests. -""" - -import pathlib -import subprocess -import sys -import textwrap - -_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") -_REPO_ROOT = str(pathlib.Path(__file__).resolve().parents[3]) - -# CPython derives the _sysconfigdata module name from sys.platform on first use, so a -# faked platform breaks any later sysconfig lookup (pydantic imports zoneinfo, which -# calls sysconfig.get_config_var). Prime the cache before faking. -_PRIME_SYSCONFIG = """ -import sysconfig - -sysconfig.get_config_vars() -""" - -# The companion's bootstrap installs the fetch transport under Emscripten and fails fast -# if pyodide.http cannot be imported, so a faked platform needs a stand-in module. -_FAKE_PYODIDE = """ -import types - -_pyodide = types.ModuleType("pyodide") -_http = types.ModuleType("pyodide.http") -_http.pyfetch = None -_pyodide.http = _http -sys.modules["pyodide"] = _pyodide -sys.modules["pyodide.http"] = _http -""" - - -def _run( - body: str, *, prelude: str = "", path_entry: str = _SRC, no_site: bool = False -) -> subprocess.CompletedProcess: - # -I -S: skip site-packages entirely (plain -I still processes the venv's .pth - # files), so nothing pip-installed is importable — only stdlib plus `path_entry`. - interp = [sys.executable, "-I", "-S"] if no_site else [sys.executable] - script = f"import sys\nsys.path.insert(0, {path_entry!r})\n" + prelude + textwrap.dedent(body) - return subprocess.run([*interp, "-c", script], capture_output=True, text=True) - - -def test_bare_import_weaviate_installs_shim_under_emscripten(): - result = _run( - prelude=_PRIME_SYSCONFIG + _FAKE_PYODIDE, - body=""" - sys.platform = "emscripten" - - import weaviate # the ONLY weaviate-side import: must bootstrap the companion - - assert "weaviate_client_web" in sys.modules, "hook did not import the companion" - import weaviate_client_web - assert weaviate_client_web.is_installed() - assert weaviate_client_web.is_fetch_transport_installed() - import grpc - assert getattr(grpc, "__weaviate_client_web_shim__", False) is True - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_bare_import_without_companion_raises_clear_import_error(): - # No site-packages, so neither weaviate_client_web nor grpcio is importable; the repo - # root goes on sys.path so the weaviate package itself is still found. - result = _run( - """ - sys.platform = "emscripten" - try: - import weaviate - except ImportError as e: - assert "weaviate-client-web" in str(e), str(e) - assert "WebAssembly/Pyodide" in str(e), str(e) - print("OK") - else: - raise AssertionError("expected ImportError without the companion") - """, - path_entry=_REPO_ROOT, - no_site=True, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_bare_import_with_grpc_present_falls_through_silently(): - # Companion blocked but a real grpc IS importable (grpcio in the dev env): the hook - # must fall through and leave the normal import path untouched. - result = _run( - prelude=_PRIME_SYSCONFIG, - body=""" - sys.platform = "emscripten" - sys.modules["weaviate_client_web"] = None # makes its import raise ImportError - - import weaviate - import grpc - - assert not getattr(grpc, "__weaviate_client_web_shim__", False) - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_bare_import_with_broken_companion_surfaces_its_own_error(tmp_path): - # An INSTALLED companion whose import fails (here: a missing dependency of its own) - # must raise that error, not the "install weaviate-client-web" hint — the hint would - # send the user to reinstall a package that is already there. - fake_pkg = tmp_path / "weaviate_client_web" - fake_pkg.mkdir() - (fake_pkg / "__init__.py").write_text( - "raise ModuleNotFoundError(\"No module named 'anyio'\", name='anyio')\n" - ) - result = _run( - prelude=_PRIME_SYSCONFIG, - body=""" - sys.platform = "emscripten" - try: - import weaviate - except ImportError as e: - assert e.name == "anyio", (e.name, str(e)) - assert "anyio" in str(e), str(e) - assert "weaviate-client-web" not in str(e), str(e) - print("OK") - else: - raise AssertionError("expected the companion's own ImportError to surface") - """, - path_entry=str(tmp_path), - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout diff --git a/packages/web/tests/test_transport.py b/packages/web/tests/test_transport.py index e1a43492b..9bb118b97 100644 --- a/packages/web/tests/test_transport.py +++ b/packages/web/tests/test_transport.py @@ -1,8 +1,8 @@ -"""In-process tests for the grpc-web channel/multicallable. +"""grpc-web channel / multicallable tests. Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``. -These exercise the transport classes directly (they import their grpc base classes from -``weaviate_client_web._shim``, not from ``sys.modules['grpc']``), so no shim install is -needed and the real ``grpc`` in the dev environment is left untouched. +These exercise the transport classes directly through fake senders — no network, no +running Weaviate. Tests are ``async def`` awaited on Pyodide's event loop by +``runner.py`` (``asyncio.run()`` cannot be used inside Pyodide). """ import asyncio @@ -10,15 +10,17 @@ import sys from typing import Dict, List, Optional, Tuple -import pytest +from harness import patched, raises -from weaviate_client_web._channel import ( - GrpcWebChannel, - _body_excerpt, - _encode_timeout, - set_sender, +from weaviate_client_web import GrpcWebChannel, set_sender +from weaviate_client_web._channel import _body_excerpt, _encode_timeout +from weaviate_client_web._sender import pyfetch_sender +from weaviate_client_web._shim import ( + AioChannel, + AioRpcError, + StatusCode, + _aio_insecure_channel, ) -from weaviate_client_web._shim import AioChannel, AioRpcError, StatusCode def _frame(payload: bytes, flag: int = 0x00) -> bytes: @@ -52,7 +54,7 @@ def test_grpcwebchannel_is_grpc_aio_channel(): assert isinstance(_channel(FakeSender()), AioChannel) -def test_unary_success_round_trip(): +async def test_unary_success_round_trip(): sender = FakeSender(body=_ok_response(b"reply-bytes")) channel = _channel(sender) mc = channel.unary_unary( @@ -62,7 +64,7 @@ def test_unary_success_round_trip(): _registered_method=True, ) - result = asyncio.run(mc(b"request-bytes", metadata=[("authorization", "Bearer k")], timeout=5)) + result = await mc(b"request-bytes", metadata=[("authorization", "Bearer k")], timeout=5) assert result == b"reply-bytes" url, headers, body, timeout = sender.calls[0] @@ -74,50 +76,50 @@ def test_unary_success_round_trip(): assert timeout == 5 -def test_secure_channel_uses_https(): +async def test_secure_channel_uses_https(): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender, secure=True) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - asyncio.run(mc(b"q")) + await mc(b"q") assert sender.calls[0][0].startswith("https://example.com:443/") -def test_health_call_without_metadata(): +async def test_health_call_without_metadata(): sender = FakeSender(body=_ok_response(b"pong")) channel = _channel(sender) mc = channel.unary_unary("/grpc.health.v1.Health/Check", lambda x: x, lambda b: b) # mirrors the health check in connect/v4.py — request + timeout, no metadata - assert asyncio.run(mc(b"ping", timeout=2)) == b"pong" + assert await mc(b"ping", timeout=2) == b"pong" -def test_error_trailer_raises_aiorpcerror(): +async def test_error_trailer_raises_aiorpcerror(): body = _frame(b"grpc-status:7\r\ngrpc-message:nope\r\n", 0x80) channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.PERMISSION_DENIED assert excinfo.value.code().name == "PERMISSION_DENIED" assert excinfo.value.details() == "nope" -def test_percent_encoded_grpc_message_decoded(): +async def test_percent_encoded_grpc_message_decoded(): body = _frame(b"grpc-status:5\r\ngrpc-message:not%20found\r\n", 0x80) channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.details() == "not found" -def test_trailers_only_status_in_http_headers(): +async def test_trailers_only_status_in_http_headers(): channel = _channel( FakeSender(status=200, headers={"grpc-status": "16", "grpc-message": "auth"}, body=b"") ) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.UNAUTHENTICATED @@ -149,19 +151,19 @@ def test_trailers_only_status_in_http_headers(): ) -def _details_of(status, body, headers=None, path="/grpc.health.v1.Health/Check"): +async def _details_of(status, body, headers=None, path="/grpc.health.v1.Health/Check"): """Run one request against a canned HTTP response and return the AioRpcError.""" channel = _channel(FakeSender(status=status, headers=headers or {}, body=body)) mc = channel.unary_unary(path, lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") return excinfo.value -def test_weaviate_404_json_names_both_candidate_causes(): +async def test_weaviate_404_json_names_both_candidate_causes(): # A 404 means EITHER the server predates the native /v1/grpc-web endpoint OR the # configured path prefix is wrong. The channel cannot tell which, so it must say both. - err = _details_of(404, WEAVIATE_404_JSON, {"content-type": "application/json"}) + err = await _details_of(404, WEAVIATE_404_JSON, {"content-type": "application/json"}) details = err.details() assert err.code() is StatusCode.UNIMPLEMENTED @@ -174,44 +176,46 @@ def test_weaviate_404_json_names_both_candidate_causes(): assert "malformed grpc-web response" not in details -def test_nginx_502_maps_to_unavailable_so_the_client_retries(): +async def test_nginx_502_maps_to_unavailable_so_the_client_retries(): # weaviate/retry.py retries UNAVAILABLE and nothing else; a gateway error arriving # as INTERNAL is silently un-retried, which is the regression this pins. - err = _details_of(502, NGINX_502_HTML) + err = await _details_of(502, NGINX_502_HTML) assert err.code() is StatusCode.UNAVAILABLE assert err.details().startswith("HTTP 502 ") assert "502 Bad Gateway" in err.details() -@pytest.mark.parametrize("status", [503, 504]) -def test_gateway_errors_are_unavailable(status): - err = _details_of(status, b"upstream down") - assert err.code() is StatusCode.UNAVAILABLE +async def test_gateway_errors_are_unavailable(): + for status in (503, 504): + err = await _details_of(status, b"upstream down") + assert err.code() is StatusCode.UNAVAILABLE, status -def test_nginx_404_html_is_reported_as_an_http_404(): - err = _details_of(404, NGINX_404_HTML) +async def test_nginx_404_html_is_reported_as_an_http_404(): + err = await _details_of(404, NGINX_404_HTML) assert err.code() is StatusCode.UNIMPLEMENTED assert err.details().startswith("HTTP 404 ") assert "404 Not Found" in err.details() assert "malformed grpc-web response" not in err.details() -def test_405_names_the_wrong_prefix(): +async def test_405_names_the_wrong_prefix(): # a 405 can only come from an existing HTTP route (measured live: a prefix pointing # at /v1/objects answers "method POST is not allowed"), so the prefix is wrong - err = _details_of(405, b'{"code":405,"message":"method POST is not allowed, but [GET] are"}') + err = await _details_of( + 405, b'{"code":405,"message":"method POST is not allowed, but [GET] are"}' + ) assert err.details().startswith("HTTP 405 ") assert "path prefix" in err.details() assert "/v1/grpc-web" in err.details() assert "method POST is not allowed" in err.details() -def test_truncated_grpc_web_body_is_reported_as_truncated_not_as_wrong_prefix(): +async def test_truncated_grpc_web_body_is_reported_as_truncated_not_as_wrong_prefix(): # a valid frame header whose payload was cut short: the endpoint IS grpc-web, so the # SPA / path-prefix hint would send the user the wrong way body = _ok_response(b"reply-bytes")[:-6] - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.INTERNAL assert "truncated" in err.details() assert "cut short" in err.details() @@ -219,43 +223,43 @@ def test_truncated_grpc_web_body_is_reported_as_truncated_not_as_wrong_prefix(): assert "path prefix" not in err.details() -def test_spa_html_body_is_not_reported_as_truncated(): +async def test_spa_html_body_is_not_reported_as_truncated(): # text bodies decode to an unknown flag byte and a garbage length; they must read as # "not grpc-web framing", never as a truncated grpc-web body - err = _details_of(200, SPA_INDEX_HTML) + err = await _details_of(200, SPA_INDEX_HTML) assert "truncated" not in err.details() assert "not grpc-web framing" in err.details() -def test_message_frame_after_trailer_is_internal(): +async def test_message_frame_after_trailer_is_internal(): body = _frame(b"a") + _frame(b"grpc-status:0\r\n", 0x80) + _frame(b"late") - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.INTERNAL assert "malformed grpc-web response" in err.details() assert "after the trailer" in err.details() assert "single-page-app" not in err.details() -def test_multiple_message_frames_in_unary_response_is_internal(): +async def test_multiple_message_frames_in_unary_response_is_internal(): # a unary RPC has exactly one message; silently taking the first would hide a proxy # or server that streams several body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:0\r\n", 0x80) - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.INTERNAL assert "2 message frames" in err.details() -def test_error_status_wins_over_multiple_message_frames(): +async def test_error_status_wins_over_multiple_message_frames(): body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:5\r\ngrpc-message:gone\r\n", 0x80) - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.NOT_FOUND assert err.details() == "gone" -def test_spa_fallback_html_200_is_distinguishable_from_a_404(): +async def test_spa_fallback_html_200_is_distinguishable_from_a_404(): # An HTTP 200 serving index.html is the other half of a wrong path prefix: the app's # catch-all route answers instead of Weaviate. It must not read as malformed framing. - err = _details_of(200, SPA_INDEX_HTML) + err = await _details_of(200, SPA_INDEX_HTML) details = err.details() assert details.startswith("HTTP 200 ") @@ -263,158 +267,157 @@ def test_spa_fallback_html_200_is_distinguishable_from_a_404(): assert "single-page-app" in details # names the actual cause assert "malformed grpc-web response" not in details # distinguishable from the 404 case, not the same generic message - assert details != _details_of(404, NGINX_404_HTML).details() + assert details != (await _details_of(404, NGINX_404_HTML)).details() -def test_401_json_body_maps_to_unauthenticated(): - err = _details_of(401, b'{"error":[{"message":"anonymous access not enabled"}]}') +async def test_401_json_body_maps_to_unauthenticated(): + err = await _details_of(401, b'{"error":[{"message":"anonymous access not enabled"}]}') assert err.code() is StatusCode.UNAUTHENTICATED assert err.details().startswith("HTTP 401 ") assert "anonymous access not enabled" in err.details() -def test_403_error_body_reaches_details(): +async def test_403_error_body_reaches_details(): # regression: the response body is the most actionable part of the error and must # survive into details() rather than being parsed as frames and discarded - err = _details_of(403, b'{"code":403,"message":"forbidden: rbac denied"}') + err = await _details_of(403, b'{"code":403,"message":"forbidden: rbac denied"}') assert err.code() is StatusCode.PERMISSION_DENIED assert "forbidden: rbac denied" in err.details() -def test_error_body_excerpt_is_capped(): - err = _details_of(500, b"E" * 5000) +async def test_error_body_excerpt_is_capped(): + err = await _details_of(500, b"E" * 5000) details = err.details() assert "EEEE" in details assert details.endswith("...") assert len(details) < 600 # the 5000-byte body is excerpted, not pasted in -def test_binary_error_body_does_not_break_the_error(): +async def test_binary_error_body_does_not_break_the_error(): # a proxy answering with a binary payload must not raise UnicodeDecodeError while # the error message is being built - err = _details_of(502, b"\xff\xfe\x00\x01\x02") + err = await _details_of(502, b"\xff\xfe\x00\x01\x02") assert err.code() is StatusCode.UNAVAILABLE assert err.details().startswith("HTTP 502 ") -def test_non_200_with_valid_grpc_web_trailers_still_uses_grpc_status(): +async def test_non_200_with_valid_grpc_web_trailers_still_uses_grpc_status(): # guard on the fix's shape: the HTTP status must not shadow a real grpc-status that # a proxy shipped alongside a non-200 - err = _details_of(500, _frame(b"grpc-status:7\r\ngrpc-message:denied\r\n", 0x80)) + err = await _details_of(500, _frame(b"grpc-status:7\r\ngrpc-message:denied\r\n", 0x80)) assert err.code() is StatusCode.PERMISSION_DENIED assert err.details() == "denied" -def test_non_ascii_grpc_message_preserves_the_status(): +async def test_non_ascii_grpc_message_preserves_the_status(): # a trailer carrying raw UTF-8 (an un-percent-encoded proxy, or an error quoting a # collection name) must not degrade to INTERNAL and lose grpc-status body = _frame("grpc-status:5\r\ngrpc-message:collection Café not found\r\n".encode(), 0x80) - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.NOT_FOUND assert "Caf" in err.details() -def test_invalid_utf8_grpc_message_preserves_the_status(): +async def test_invalid_utf8_grpc_message_preserves_the_status(): # latin-1 bytes are not valid UTF-8; the status must still survive body = _frame(b"grpc-status:9\r\ngrpc-message:tenant caf\xe9 is COLD\r\n", 0x80) - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.FAILED_PRECONDITION assert "tenant caf" in err.details() -def test_binary_metadata_base64_encoded(): +async def test_binary_metadata_base64_encoded(): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - asyncio.run(mc(b"q", metadata=[("trace-bin", b"\x00\x01\x02")])) + await mc(b"q", metadata=[("trace-bin", b"\x00\x01\x02")]) assert sender.calls[0][1]["trace-bin"] == "AAEC" def test_stream_stream_raises_clear_error(): channel = _channel(FakeSender()) mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) - with pytest.raises(RuntimeError) as excinfo: + with raises(RuntimeError, contains="not supported over grpc-web"): mc(request_iterator=iter([]), timeout=5, metadata=None) - assert "not supported over grpc-web" in str(excinfo.value) -def test_timeout_maps_to_deadline_exceeded(): +async def test_timeout_maps_to_deadline_exceeded(): async def slow_sender(url, headers, body, timeout): await asyncio.sleep(0.5) return 200, {}, _ok_response(b"x") channel = GrpcWebChannel("h:1", secure=False, sender=slow_sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q", timeout=0.01)) + with raises(AioRpcError) as excinfo: + await mc(b"q", timeout=0.01) assert excinfo.value.code() is StatusCode.DEADLINE_EXCEEDED -def test_transport_exception_maps_to_unavailable(): +async def test_transport_exception_maps_to_unavailable(): async def boom(url, headers, body, timeout): raise ConnectionError("connection refused") channel = GrpcWebChannel("h:1", secure=False, sender=boom) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.UNAVAILABLE assert "ConnectionError: connection refused" in str(excinfo.value.details()) -def test_transport_exception_with_empty_str_keeps_type(): +async def test_transport_exception_with_empty_str_keeps_type(): # httpx transport errors commonly stringify to '' — the detail must still name them async def boom(url, headers, body, timeout): raise ConnectionError() channel = GrpcWebChannel("h:1", secure=False, sender=boom) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert "ConnectionError" in str(excinfo.value.details()) -def test_empty_ok_response_hints_at_cors_expose_headers(): +async def test_empty_ok_response_hints_at_cors_expose_headers(): # HTTP 200, empty body, no grpc-status anywhere: the shape of a trailers-only error # whose grpc-status/grpc-message headers were stripped by CORS channel = _channel(FakeSender(status=200, headers={}, body=b"")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "Access-Control-Expose-Headers" in str(excinfo.value.details()) -def test_empty_ok_response_with_grpc_status_has_no_cors_hint(): +async def test_empty_ok_response_with_grpc_status_has_no_cors_hint(): # when grpc-status WAS visible (status 0, no frames), it is a malformed response, # not a CORS problem — the hint must not appear channel = _channel(FakeSender(status=200, headers={"grpc-status": "0"}, body=b"")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "Access-Control-Expose-Headers" not in str(excinfo.value.details()) -def test_message_frame_without_grpc_status_is_internal_not_success(): +async def test_message_frame_without_grpc_status_is_internal_not_success(): # HTTP 200 with a valid message frame but no grpc-status anywhere (e.g. a proxy # dropped the trailer frame) must be an error, never a fabricated success channel = _channel(FakeSender(status=200, headers={}, body=_frame(b"reply-bytes"))) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "missing grpc-status" in str(excinfo.value.details()) -def test_message_frame_with_grpc_status_header_still_succeeds(): +async def test_message_frame_with_grpc_status_header_still_succeeds(): # trailers-only-in-headers responses (grpc-status as an HTTP header, no trailer # frame) remain valid per the grpc-web contract channel = _channel( FakeSender(status=200, headers={"grpc-status": "0"}, body=_frame(b"reply-bytes")) ) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - assert asyncio.run(mc(b"q")) == b"reply-bytes" + assert await mc(b"q") == b"reply-bytes" def test_stream_stream_error_recommends_insert_many_only(): @@ -422,37 +425,37 @@ def test_stream_stream_error_recommends_insert_many_only(): # only one supported under WASM), so the error must not recommend them channel = _channel(FakeSender()) mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) - with pytest.raises(RuntimeError) as excinfo: + with raises(RuntimeError) as excinfo: mc(request_iterator=iter([]), timeout=5, metadata=None) assert "insert_many" in str(excinfo.value) for sync_only in ("dynamic", "fixed_size", "rate_limit"): assert sync_only not in str(excinfo.value) -def test_malformed_frame_maps_to_internal(): +async def test_malformed_frame_maps_to_internal(): # A 3-byte body cannot contain even a 5-byte frame header -> framing ValueError. channel = _channel(FakeSender(body=b"\x00\x00\x00")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL -def test_malformed_grpc_status_maps_to_internal(): +async def test_malformed_grpc_status_maps_to_internal(): body = _frame(b"grpc-status:notanint\r\n", 0x80) channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL -def test_grpc_timeout_header_rounds_up(): +async def test_grpc_timeout_header_rounds_up(): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) # 123.4ms must round UP to 124ms (never advertise a shorter deadline than requested). - asyncio.run(mc(b"q", timeout=0.1234)) + await mc(b"q", timeout=0.1234) assert sender.calls[0][1]["grpc-timeout"] == "124m" @@ -462,9 +465,8 @@ def test_body_excerpt_empty_and_non_printable(): assert _body_excerpt(b"ok\x00\x01") == "ok" -@pytest.mark.parametrize( - "seconds,expected", - [ +def test_encode_timeout_stays_within_eight_digits(): + cases = [ (None, None), (float("inf"), None), (float("nan"), None), @@ -479,123 +481,118 @@ def test_body_excerpt_empty_and_non_printable(): (1e10, None), # would need hours, which transcoders reject above 8H: no deadline (1e15, None), (1e308, None), # finite, but *1000 overflows to infinity: must not raise - ], -) -def test_encode_timeout_stays_within_eight_digits(seconds, expected): - encoded = _encode_timeout(seconds) - assert encoded == expected - if encoded is not None: - assert len(encoded) <= 9 # 8 digits + unit - assert not encoded.endswith("H") + ] + for seconds, expected in cases: + encoded = _encode_timeout(seconds) + assert encoded == expected, (seconds, encoded, expected) + if encoded is not None: + assert len(encoded) <= 9, encoded # 8 digits + unit + assert not encoded.endswith("H"), encoded -def test_infinite_timeout_sends_no_deadline(): +async def test_infinite_timeout_sends_no_deadline(): # Timeout(query=inf): neither a grpc-timeout header nor a client-side wait sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - assert asyncio.run(mc(b"q", timeout=float("inf"))) == b"x" + assert await mc(b"q", timeout=float("inf")) == b"x" _, headers, _, timeout = sender.calls[0] assert "grpc-timeout" not in headers assert timeout is None -def test_huge_timeout_uses_minutes_then_no_deadline(): +async def test_huge_timeout_uses_minutes_then_no_deadline(): # 1e8 s in milliseconds is 12 digits; the server rejects more than 8 ("timeout is # too long", HTTP 400) and transcoders reject hour values above 8H, so past the # minute range the request carries no deadline at all sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - asyncio.run(mc(b"q", timeout=1e8)) - asyncio.run(mc(b"q", timeout=1e9)) - asyncio.run(mc(b"q", timeout=1e10)) + await mc(b"q", timeout=1e8) + await mc(b"q", timeout=1e9) + await mc(b"q", timeout=1e10) assert sender.calls[0][1]["grpc-timeout"] == "1666667M" assert sender.calls[1][1]["grpc-timeout"] == "16666667M" assert "grpc-timeout" not in sender.calls[2][1] assert sender.calls[2][3] is None # no client-side wait either -@pytest.mark.parametrize("bad", ["val\r\nx-injected: evil", "val\nx", "v\0"]) -def test_crlf_in_metadata_rejected(bad): +async def test_crlf_in_metadata_rejected(): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(ValueError, match="Illegal character"): - asyncio.run(mc(b"q", metadata=[("x-key", bad)])) - with pytest.raises(ValueError, match="Illegal character"): - asyncio.run(mc(b"q", metadata=[("x-key\r\n", "v")])) + for bad in ("val\r\nx-injected: evil", "val\nx", "v\0"): + with raises(ValueError, contains="Illegal character"): + await mc(b"q", metadata=[("x-key", bad)]) + with raises(ValueError, contains="Illegal character"): + await mc(b"q", metadata=[("x-key\r\n", "v")]) assert sender.calls == [] -def _unavailable_details(monkeypatch, path_prefix, platform): +async def _unavailable_details(path_prefix, platform): async def boom(url, headers, body, timeout): raise ConnectionError("Failed to fetch") - monkeypatch.setattr(sys, "platform", platform) - channel = GrpcWebChannel("h:50051", secure=False, sender=boom, path_prefix=path_prefix) - mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + with patched(sys, "platform", platform): + channel = GrpcWebChannel("h:50051", secure=False, sender=boom, path_prefix=path_prefix) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.UNAVAILABLE return excinfo.value.details() -def test_unavailable_without_path_prefix_under_emscripten_hints_at_grpc_path_prefix(monkeypatch): +async def test_unavailable_without_path_prefix_under_emscripten_hints_at_grpc_path_prefix(): # the connect helpers always set the prefix under Emscripten, so a prefix-less channel # here means hand-built ConnectionParams; the error must say what to do instead - details = _unavailable_details(monkeypatch, path_prefix="", platform="emscripten") + details = await _unavailable_details(path_prefix="", platform="emscripten") assert "grpc_path_prefix='/v1/grpc-web'" in details assert "1.38.3" in details assert "connect helpers" in details -def test_unavailable_with_path_prefix_has_no_prefix_hint(monkeypatch): - details = _unavailable_details(monkeypatch, path_prefix="/v1/grpc-web", platform="emscripten") +async def test_unavailable_with_path_prefix_has_no_prefix_hint(): + details = await _unavailable_details(path_prefix="/v1/grpc-web", platform="emscripten") assert "no grpc_path_prefix" not in details -def test_unavailable_without_path_prefix_off_emscripten_has_no_prefix_hint(monkeypatch): - # on CPython an empty prefix against a transcoder is the normal configuration - details = _unavailable_details(monkeypatch, path_prefix="", platform="linux") +async def test_unavailable_without_path_prefix_off_emscripten_has_no_prefix_hint(): + # off Emscripten an empty prefix against a transcoder is the normal configuration + details = await _unavailable_details(path_prefix="", platform="linux") assert "no grpc_path_prefix" not in details -def test_close_is_awaitable_noop(): +async def test_close_is_awaitable_noop(): channel = _channel(FakeSender()) - assert asyncio.run(channel.close()) is None + assert await channel.close() is None -def test_path_prefix_prepended_to_url(): +async def test_path_prefix_prepended_to_url(): sender = FakeSender(body=_ok_response(b"r")) channel = GrpcWebChannel( "example.com:8090", secure=False, sender=sender, path_prefix="/grpc-web" ) mc = channel.unary_unary("/weaviate.v1.Weaviate/Search", lambda x: x, lambda b: b) - asyncio.run(mc(b"q")) + await mc(b"q") assert sender.calls[0][0] == "http://example.com:8090/grpc-web/weaviate.v1.Weaviate/Search" -@pytest.mark.parametrize( - "raw,expected_url", - [ +async def test_path_prefix_normalized_in_url(): + cases = [ ("grpc-web", "http://h:1/grpc-web/svc/M"), ("/grpc-web/", "http://h:1/grpc-web/svc/M"), ("/a/b", "http://h:1/a/b/svc/M"), ("", "http://h:1/svc/M"), - ], -) -def test_path_prefix_normalized_in_url(raw, expected_url): - sender = FakeSender(body=_ok_response(b"r")) - channel = GrpcWebChannel("h:1", secure=False, sender=sender, path_prefix=raw) - mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - asyncio.run(mc(b"q")) - assert sender.calls[0][0] == expected_url + ] + for raw, expected_url in cases: + sender = FakeSender(body=_ok_response(b"r")) + channel = GrpcWebChannel("h:1", secure=False, sender=sender, path_prefix=raw) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + await mc(b"q") + assert sender.calls[0][0] == expected_url, raw def test_shim_factory_extracts_path_prefix_option(): - from weaviate_client_web._shim import _aio_insecure_channel - with_prefix = _aio_insecure_channel( target="h:1", options=[("grpc.max_send_message_length", 1), ("grpc-web.path_prefix", "/grpc-web")], @@ -608,15 +605,13 @@ def test_shim_factory_extracts_path_prefix_option(): assert without_prefix._path_prefix == "" -def test_set_sender_overrides_default(): +async def test_set_sender_overrides_default(): sender = FakeSender(body=_ok_response(b"y")) set_sender(sender) try: channel = GrpcWebChannel("h:1", secure=False) # no explicit sender mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - assert asyncio.run(mc(b"q")) == b"y" + assert await mc(b"q") == b"y" finally: - # restore the real default so other tests/processes are unaffected - from weaviate_client_web._sender import pyfetch_sender - + # restore the real default so other tests are unaffected set_sender(pyfetch_sender) diff --git a/setup.cfg b/setup.cfg index 7343116a5..4ebf52c4b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,12 @@ python_requires = >=3.10 [options.extras_require] agents = weaviate-agents >=1.0.0, <2.0.0 +grpc-web = + # The Pyodide/WASM grpc-web companion, built from packages/web in this repo and + # versioned in lockstep (see TODO(lockstep) in packages/web/pyproject.toml). The + # marker makes the extra a no-op on CPython: the companion imports pyodide at + # module scope and only makes sense under Emscripten. + weaviate-client-web; sys_platform == "emscripten" [options.package_data] # If any package or subpackage contains *.txt, *.rst or *.md files, include them: diff --git a/weaviate/__init__.py b/weaviate/__init__.py index fbd0fdabf..ef40e7a96 100644 --- a/weaviate/__init__.py +++ b/weaviate/__init__.py @@ -19,8 +19,9 @@ "weaviate requires the weaviate-client-web package under " "WebAssembly/Pyodide: there is no grpcio wheel for Emscripten, and " "weaviate-client-web provides the grpc-web (fetch) transport in its " - "place. Install it (e.g. micropip.install('weaviate-client-web')) and " - "import weaviate again." + "place. Install it via the extra (e.g. " + "micropip.install('weaviate-client[grpc-web]')) and import weaviate " + "again." ) from exc import os diff --git a/weaviate/connect/base.py b/weaviate/connect/base.py index 0f7d40bd1..82c8b3851 100644 --- a/weaviate/connect/base.py +++ b/weaviate/connect/base.py @@ -165,10 +165,11 @@ def _check_grpc_web_usable(self, is_async: bool) -> None: raise WeaviateInvalidInputError( "grpc_path_prefix enables grpc-web, which requires the " "'weaviate-client-web' package (it installs a grpc shim before " - "'import weaviate'); it is not active in this environment. Under Pyodide a " - "plain `import weaviate` activates it; on CPython call " - "weaviate_client_web.install(force=True) and set_sender(make_httpx_sender()) " - "before importing weaviate (intended for integration testing)." + "'import weaviate'); it is not active in this environment. grpc-web is " + "only available under WebAssembly/Pyodide, where a plain `import " + "weaviate` activates it (install the companion with " + "micropip.install('weaviate-client[grpc-web]')); on CPython use native " + "gRPC instead." ) def _grpc_channel( From f7912ed3f263e694e003fe13a2deffae481c9cc3 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:28:56 +0200 Subject: [PATCH 2/6] build(grpc-web): version weaviate-client-web in lockstep via setuptools_scm The companion's version now derives from the repository's git tags (setuptools_scm with its root at the repo root) instead of a hardcoded 0.0.1.dev0, so every build carries the same version as weaviate-client; the pyodide-e2e job asserts the two built wheels match. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Caze9m6PBSfYkt77mMKj2b --- .github/workflows/main.yaml | 8 ++++++++ packages/web/pyproject.toml | 11 +++++++---- setup.cfg | 8 ++++---- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 44e52d6d0..8a1f1d3f8 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -134,6 +134,14 @@ jobs: pip install build python -m build --wheel --outdir dist . python -m build --wheel --outdir dist packages/web + - name: Assert both wheels carry the same version + # weaviate-client-web is versioned in lockstep with weaviate-client: both derive + # from the same git tag via setuptools_scm. + run: | + base=$(basename dist/weaviate_client-*.whl); base=${base#weaviate_client-}; base=${base%%-*} + web=$(basename dist/weaviate_client_web-*.whl); web=${web#weaviate_client_web-}; web=${web%%-*} + echo "weaviate-client=$base weaviate-client-web=$web" + test "$base" = "$web" - name: Run the unit suite inside Pyodide under Node # The grpc-web package imports pyodide at module scope, so its unit tests only # run here. Needs no running Weaviate. diff --git a/packages/web/pyproject.toml b/packages/web/pyproject.toml index 95792c315..1ad43f705 100644 --- a/packages/web/pyproject.toml +++ b/packages/web/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=65", "wheel"] +requires = ["setuptools>=65", "setuptools_scm[toml] >6.2", "wheel"] build-backend = "setuptools.build_meta" [project] @@ -10,9 +10,9 @@ requires-python = ">=3.10" license = { text = "BSD-3-Clause" } authors = [{ name = "Weaviate", email = "hello@weaviate.io" }] keywords = ["weaviate", "grpc-web", "pyodide", "wasm", "emscripten"] -# Version is kept in lockstep with weaviate-client. TODO(lockstep): derive from the same -# git tag via setuptools_scm and assert the built versions match in CI before publishing. -version = "0.0.1.dev0" +# Version comes from the repository's git tags via setuptools_scm (root below), so every +# build carries the same version as weaviate-client; CI asserts the built wheels match. +dynamic = ["version"] # Deliberately depends on weaviate-client WITHOUT grpcio (grpcio is excluded under # Emscripten by the `sys_platform != "emscripten"` marker in the base package's deps). dependencies = [ @@ -25,6 +25,9 @@ dependencies = [ Source = "https://github.com/weaviate/weaviate-python-client" Tracker = "https://github.com/weaviate/weaviate-python-client/issues" +[tool.setuptools_scm] +root = "../.." + [tool.setuptools.packages.find] where = ["src"] diff --git a/setup.cfg b/setup.cfg index 4ebf52c4b..26b397613 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,10 +49,10 @@ python_requires = >=3.10 agents = weaviate-agents >=1.0.0, <2.0.0 grpc-web = - # The Pyodide/WASM grpc-web companion, built from packages/web in this repo and - # versioned in lockstep (see TODO(lockstep) in packages/web/pyproject.toml). The - # marker makes the extra a no-op on CPython: the companion imports pyodide at - # module scope and only makes sense under Emscripten. + # The Pyodide/WASM grpc-web companion, built from packages/web in this repo. Both + # packages derive their version from the same git tag (setuptools_scm; CI asserts + # the built wheels match). The marker makes the extra a no-op on CPython: the + # companion imports pyodide at module scope and only makes sense under Emscripten. weaviate-client-web; sys_platform == "emscripten" [options.package_data] From e9cd331fb600d8499313824d848cf378c99b8308 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:17:30 +0200 Subject: [PATCH 3/6] build(grpc-web): pin weaviate-client to the lockstep version at build time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit packages/web/setup.py injects weaviate-client== into the companion's requirements when the wheel is built, so a mismatched pair can never resolve at install time — the two packages share private contracts (error-string markers, exception constants). Consequence: every tag must publish both packages. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Caze9m6PBSfYkt77mMKj2b --- packages/web/pyproject.toml | 11 +++-------- packages/web/setup.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 packages/web/setup.py diff --git a/packages/web/pyproject.toml b/packages/web/pyproject.toml index 1ad43f705..0f4872e18 100644 --- a/packages/web/pyproject.toml +++ b/packages/web/pyproject.toml @@ -12,14 +12,9 @@ authors = [{ name = "Weaviate", email = "hello@weaviate.io" }] keywords = ["weaviate", "grpc-web", "pyodide", "wasm", "emscripten"] # Version comes from the repository's git tags via setuptools_scm (root below), so every # build carries the same version as weaviate-client; CI asserts the built wheels match. -dynamic = ["version"] -# Deliberately depends on weaviate-client WITHOUT grpcio (grpcio is excluded under -# Emscripten by the `sys_platform != "emscripten"` marker in the base package's deps). -dependencies = [ - "weaviate-client", - # Pyodide's bundled httpx build omits anyio, but authlib imports it directly. - 'anyio ; sys_platform == "emscripten"', -] +# Dependencies are computed in setup.py: the weaviate-client requirement is pinned to +# that same version at build time, so mismatched pairs cannot resolve at install time. +dynamic = ["version", "dependencies"] [project.urls] Source = "https://github.com/weaviate/weaviate-python-client" diff --git a/packages/web/setup.py b/packages/web/setup.py new file mode 100644 index 000000000..e99c3f653 --- /dev/null +++ b/packages/web/setup.py @@ -0,0 +1,25 @@ +"""Injects the lockstep ``weaviate-client==`` pin at build time. + +Both packages derive their version from the repository's git tags (setuptools_scm), so +the version is only known when the wheel is built and a static ``dependencies`` list +cannot express the pin. The pin makes mismatched pairs unresolvable at install time: +the two packages share private contracts (the ``HTTP `` error-string markers, +the exception constants ``_channel`` imports), so a companion must only ever install +next to the base client it was built with. + +Consequence for releasing: every tag must publish BOTH packages — a base-only release +would leave the extra pointing at a companion whose pin no longer resolves. +""" + +from setuptools import setup +from setuptools_scm import get_version + +version = get_version(root="../..", relative_to=__file__) + +setup( + install_requires=[ + f"weaviate-client=={version}", + # Pyodide's bundled httpx build omits anyio, but authlib imports it directly. + 'anyio ; sys_platform == "emscripten"', + ] +) From c3ff78089cc02ff34c486baa9c8711b895b4afb5 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:17:37 +0200 Subject: [PATCH 4/6] test(grpc-web): run the unit suite with pytest inside Pyodide via JSPI pytest executes the packages/web/tests suite inside Pyodide under Node with --experimental-wasm-jspi: async tests run through run_until_complete, which stack-switches when the runner enters via callPromising(). This replaces the hand-rolled runner/harness with standard pytest collection, fixtures and parametrize, and makes empty or partial collection fail the run (pytest exit codes reach JS as a return value, never as an exception across the bridge). The base client's import-hook branches (missing companion, broken companion, grpc-present fall-through) are covered by subprocess tests in test/test_wasm_compat.py on CPython; the bootstrap scenario in units.mjs keeps the one path that needs real Pyodide, micropip and the wheels. Also strengthens assertions the port had weakened (identity restore on uninstall, the literal grpc version pin, install() returning True) and fixes two stale README claims about direct installs and off-Emscripten imports. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Caze9m6PBSfYkt77mMKj2b --- .github/workflows/main.yaml | 6 +- ci/pyodide-e2e/units.mjs | 140 ++++---- packages/web/README.md | 21 +- packages/web/tests/conftest.py | 14 +- packages/web/tests/harness.py | 53 --- packages/web/tests/runner.py | 42 --- packages/web/tests/test_framing.py | 22 +- packages/web/tests/test_httpx_fetch.py | 421 +++++++++++------------- packages/web/tests/test_shim_install.py | 17 +- packages/web/tests/test_transport.py | 127 +++---- test/test_wasm_compat.py | 105 ++++++ 11 files changed, 473 insertions(+), 495 deletions(-) delete mode 100644 packages/web/tests/harness.py delete mode 100644 packages/web/tests/runner.py diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 8a1f1d3f8..da9ea3302 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -144,10 +144,12 @@ jobs: test "$base" = "$web" - name: Run the unit suite inside Pyodide under Node # The grpc-web package imports pyodide at module scope, so its unit tests only - # run here. Needs no running Weaviate. + # run here. Needs no running Weaviate. The JSPI flag lets pytest's synchronous + # runner await async tests via stack switching; without it the run fails + # loudly at startup (never a false green). run: | npm install --prefix ci/pyodide-e2e - node ci/pyodide-e2e/units.mjs dist + node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist - name: start weaviate run: | source ./ci/compose.sh diff --git a/ci/pyodide-e2e/units.mjs b/ci/pyodide-e2e/units.mjs index 61ba72c2c..f333080a7 100644 --- a/ci/pyodide-e2e/units.mjs +++ b/ci/pyodide-e2e/units.mjs @@ -1,17 +1,22 @@ -// Runs the weaviate_client_web unit suite (packages/web/tests) inside Pyodide (WASM) -// under Node, plus bootstrap scenarios that each need a fresh interpreter. No running -// Weaviate is needed — everything is driven through fake senders / a fake pyfetch. +// Runs the weaviate_client_web unit suite (packages/web/tests) with pytest inside +// Pyodide (WASM) under Node, plus a bootstrap scenario in a fresh interpreter. No +// running Weaviate is needed — everything is driven through fake senders / a fake +// pyfetch. // -// Usage: node units.mjs +// Usage: node --experimental-wasm-jspi units.mjs // must contain exactly the two locally-built pure wheels: // weaviate_client-*.whl and weaviate_client_web-*.whl (same layout as run.mjs). // +// The JSPI flag is required: pytest's runner is synchronous, so async tests execute +// through run_until_complete, which needs stack switching (enableRunUntilComplete + +// a callPromising() entrypoint). Without the flag the run fails loudly at startup — +// it can never produce a false green. +// // The package imports pyodide at module scope, so this harness is the only place its -// unit tests can run; scenarios needing a clean import state (bootstrap and -// install-hint semantics) get one loadPyodide() each. Deliberately untested: the -// fall-through in weaviate/__init__.py when a real grpc module exists — no grpcio -// wheel exists for Emscripten, so that branch is CPython-only defence and cannot be -// exercised here. +// unit tests can run. The base client's hook logic (missing companion, broken +// companion, grpc-present fall-through) is covered by subprocess tests in +// test/test_wasm_compat.py on CPython; the bootstrap scenario below covers the one +// path that needs real Pyodide, micropip and the wheels. import { readdirSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; @@ -39,40 +44,25 @@ if ( process.exit(2); } -// Fresh interpreter with micropip ready and the wheels dir mounted; `only` restricts -// which of the two wheels get installed (the install-hint scenarios need the base -// client without its companion). -async function freshPyodide(only = prefixes) { - const pyodide = await loadPyodide({}); +// Fresh interpreter with micropip ready and the wheels dir mounted. +async function freshPyodide() { + const pyodide = await loadPyodide({ enableRunUntilComplete: true }); await pyodide.loadPackage("micropip"); const micropip = pyodide.pyimport("micropip"); pyodide.FS.mkdirTree("/wheels"); pyodide.mountNodeFS("/wheels", wheelsDir); for (const wheel of wheels) { - if (only.some((p) => wheel.startsWith(p))) { - await micropip.install(`emfs:/wheels/${wheel}`); - } + await micropip.install(`emfs:/wheels/${wheel}`); } return pyodide; } -async function scenario(name, pyodide, code) { - try { - pyodide.runPython(code); - console.log(`OK scenario: ${name}`); - } catch (err) { - console.error(`FAIL scenario: ${name}`); - console.error(err); - process.exit(1); - } -} +// --- bootstrap scenario: needs a clean import state, so its own interpreter -------- -// --- fresh-interpreter bootstrap scenarios ----------------------------------------- - -await scenario( - "bare 'import weaviate' bootstraps the companion", - await freshPyodide(), - ` +{ + const pyodide = await freshPyodide(); + try { + pyodide.runPython(` import sys assert "weaviate_client_web" not in sys.modules import weaviate # the ONLY weaviate-side import: must bootstrap the companion @@ -86,69 +76,55 @@ assert getattr(grpc, "__weaviate_client_web_shim__", False) is True assert getattr( httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False ) is True -`, -); - -await scenario( - "missing companion raises the install hint", - await freshPyodide(["weaviate_client-"]), - ` -try: - import weaviate -except ImportError as e: - assert "weaviate-client-web" in str(e), str(e) - assert "weaviate-client[grpc-web]" in str(e), str(e) - assert "WebAssembly/Pyodide" in str(e), str(e) -else: - raise AssertionError("expected ImportError without the companion") -`, -); - -await scenario( - "a broken companion surfaces its own error, not the install hint", - await freshPyodide(["weaviate_client-"]), - ` -# An INSTALLED companion whose import fails (here: a missing dependency of its own) -# must raise that error — the "install weaviate-client[grpc-web]" hint would send the -# user to reinstall a package that is already there. -import pathlib, sys -pkg = pathlib.Path("/broken/weaviate_client_web") -pkg.mkdir(parents=True) -(pkg / "__init__.py").write_text( - "raise ModuleNotFoundError(\\"No module named 'anyio'\\", name='anyio')\\n" -) -sys.path.insert(0, "/broken") -try: - import weaviate -except ImportError as e: - assert e.name == "anyio", (e.name, str(e)) - assert "anyio" in str(e), str(e) - assert "grpc-web" not in str(e), str(e) -else: - raise AssertionError("expected the companion's own ImportError to surface") -`, -); +`); + console.log("OK scenario: bare 'import weaviate' bootstraps the companion"); + } catch (err) { + console.error("FAIL scenario: bare 'import weaviate' bootstraps the companion"); + console.error(err); + process.exit(1); + } +} -// --- the main unit suite ----------------------------------------------------------- +// --- the pytest suite -------------------------------------------------------------- const testsDir = resolve(here, "../../packages/web/tests"); const pyodide = await freshPyodide(); console.log( `pyodide ${pyodide.version} / python ${pyodide.runPython("import sys; sys.version.split()[0]")}`, ); +const micropip = pyodide.pyimport("micropip"); +await micropip.install(["pytest==9.0.2", "pytest-asyncio==0.25.3"]); pyodide.FS.mkdirTree("/units"); pyodide.mountNodeFS("/units", testsDir); -try { - await pyodide.runPythonAsync(` + +// pytest.main is synchronous; entering through callPromising() lets the async tests +// stack-switch (run_until_complete) instead of failing with "Cannot stack switch". +const runPytest = pyodide.runPython(` import sys -sys.path.insert(0, "/units") -import runner -await runner.main() +sys.dont_write_bytecode = True # /units is the host checkout: no __pycache__ in it + +import pytest + +def _run(): + return int(pytest.main([ + "-v", + "-p", "no:cacheprovider", # no .pytest_cache in the host checkout either + "-o", "asyncio_mode=auto", + "-o", "asyncio_default_fixture_loop_scope=function", + "/units", + ])) + +_run `); +let exitCode; +try { + exitCode = await runPytest.callPromising(); } catch (err) { console.error(err); process.exit(1); } +// Any nonzero pytest exit code fails the run — including 5, "no tests collected". +console.log(`pytest exit code: ${exitCode}`); // The interpreters loaded above keep live handles on the Node event loop, so the // process does not exit on its own. -process.exit(0); +process.exit(exitCode === 0 ? 0 : 1); diff --git a/packages/web/README.md b/packages/web/README.md index c17e7f8df..3c4fd46c1 100644 --- a/packages/web/README.md +++ b/packages/web/README.md @@ -23,7 +23,10 @@ await micropip.install("weaviate-client[grpc-web]") The extra carries a `sys_platform == "emscripten"` marker, so the same requirement is a no-op on CPython — one requirements list works everywhere. Installing the companion -directly (`micropip.install("weaviate-client-web")`) works too. +directly (`micropip.install("weaviate-client-web")`) works too: it pins +`weaviate-client` to its own exact version, so a mismatched pair can never resolve. +Both forms require the first `weaviate-client` release that ships the extra and the +Emscripten marker; against older releases the resolver fails on `grpcio`. This package is defined for its environment: it imports `pyodide` at module scope and is therefore only importable under Emscripten/Pyodide. On CPython the base client never @@ -124,7 +127,7 @@ Pass `headers={...}` / `auth_credentials=...` as usual for API keys, OIDC or WCD Importing the companion explicitly first also works and remains the explicit form: ```python -import weaviate_client_web # installs the grpc shim under Emscripten (no-op elsewhere) +import weaviate_client_web # installs the grpc shim (only importable under Emscripten) import weaviate ``` @@ -187,11 +190,13 @@ Pyodide. From the repository root: python -m build --wheel --outdir dist . python -m build --wheel --outdir dist packages/web npm install --prefix ci/pyodide-e2e -node ci/pyodide-e2e/units.mjs dist # unit suite + bootstrap scenarios, no Weaviate needed -node ci/pyodide-e2e/run.mjs dist # e2e suite, needs a running Weaviate (see ci/) +node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist # pytest unit suite, no Weaviate needed +node ci/pyodide-e2e/run.mjs dist # e2e suite, needs a running Weaviate (see ci/) ``` -`units.mjs` runs the unit suite in `packages/web/tests/` (driven entirely through fake -senders and a fake `pyfetch`) plus fresh-interpreter bootstrap scenarios; `run.mjs` -runs the e2e suite against a live Weaviate. A `conftest.py` keeps pytest from -collecting the test modules on CPython, where they cannot import. +`units.mjs` runs pytest over `packages/web/tests/` inside Pyodide — async tests execute +on Pyodide's event loop via JSPI stack switching, hence the Node flag — plus a +fresh-interpreter bootstrap scenario; everything is driven through fake senders and a +fake `pyfetch`. `run.mjs` runs the e2e suite against a live Weaviate. On CPython the +`conftest.py` keeps pytest from collecting these modules (they cannot import there); +the base client's import-hook branches are covered by `test/test_wasm_compat.py`. diff --git a/packages/web/tests/conftest.py b/packages/web/tests/conftest.py index 9bc903df3..a6491b8dc 100644 --- a/packages/web/tests/conftest.py +++ b/packages/web/tests/conftest.py @@ -1,8 +1,14 @@ -"""Keep pytest away from this directory. +"""Restrict this suite to Pyodide. The package under test imports ``pyodide`` at module scope, so the test modules here -are only importable under Emscripten/Pyodide. They are run by -``ci/pyodide-e2e/units.mjs`` (via ``runner.py``), not by pytest. +are only importable under Emscripten/Pyodide. There, pytest runs them via +``ci/pyodide-e2e/units.mjs`` (async tests execute on Pyodide's event loop through JSPI +stack switching). On CPython, keep pytest from collecting them: + + node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist """ -collect_ignore_glob = ["*.py"] +import sys + +if sys.platform != "emscripten": + collect_ignore_glob = ["*.py"] diff --git a/packages/web/tests/harness.py b/packages/web/tests/harness.py deleted file mode 100644 index c5c8c0ad2..000000000 --- a/packages/web/tests/harness.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Shared helpers for the in-Pyodide test modules (pytest is not available here).""" - -import contextlib -import sys -from typing import Any, Optional - - -class _Caught: - value: Any - - -@contextlib.contextmanager -def raises(exc_type, contains: Optional[str] = None): - """``pytest.raises`` stand-in; ``contains`` is a plain substring, not a regex.""" - caught = _Caught() - try: - yield caught - except exc_type as e: - caught.value = e - if contains is not None and contains not in str(e): - raise AssertionError(f"{e!r} does not contain {contains!r}") from e - else: - raise AssertionError(f"expected {exc_type.__name__} to be raised") - - -@contextlib.contextmanager -def patched(obj, name: str, value): - """``monkeypatch.setattr`` stand-in with restore-on-exit.""" - sentinel = object() - old = getattr(obj, name, sentinel) - setattr(obj, name, value) - try: - yield - finally: - if old is sentinel: - delattr(obj, name) - else: - setattr(obj, name, old) - - -@contextlib.contextmanager -def sys_module(name: str, module): - """Set ``sys.modules[name]`` (``None`` makes ``import name`` raise) and restore.""" - present = name in sys.modules - old = sys.modules.get(name) - sys.modules[name] = module - try: - yield - finally: - if present: - sys.modules[name] = old # type: ignore[assignment] - else: - del sys.modules[name] diff --git a/packages/web/tests/runner.py b/packages/web/tests/runner.py deleted file mode 100644 index e9a6ae1fc..000000000 --- a/packages/web/tests/runner.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Collects and runs the test modules in this directory on Pyodide's event loop. - -Invoked by ``ci/pyodide-e2e/units.mjs``, which mounts this directory into the -interpreter and awaits :func:`main` (``asyncio.run()`` cannot be used inside Pyodide). -Test functions are ``test_*`` callables — plain or ``async`` — collected per module, -so names are qualified and cannot shadow across files. -""" - -import inspect -import traceback -from typing import List - -import test_framing -import test_httpx_fetch -import test_shim_install -import test_transport - -_MODULES = (test_framing, test_transport, test_httpx_fetch, test_shim_install) - - -async def main() -> None: - tests = [] - for module in _MODULES: - for name in sorted(vars(module)): - fn = getattr(module, name) - if name.startswith("test_") and callable(fn): - tests.append((f"{module.__name__}.{name}", fn)) - failures: List[str] = [] - for name, fn in tests: - try: - result = fn() - if inspect.iscoroutine(result): - await result - except Exception: - failures.append(name) - traceback.print_exc() - print(f"FAIL {name}", flush=True) - else: - print(f"OK {name}", flush=True) - print(f"{len(tests) - len(failures)}/{len(tests)} unit tests passed", flush=True) - if failures: - raise SystemExit(f"{len(failures)} unit test(s) failed: {', '.join(failures)}") diff --git a/packages/web/tests/test_framing.py b/packages/web/tests/test_framing.py index 16aeebe69..8054dbf94 100644 --- a/packages/web/tests/test_framing.py +++ b/packages/web/tests/test_framing.py @@ -1,8 +1,8 @@ -"""grpc-web framing tests. Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``.""" +"""grpc-web framing tests. Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs``.""" import struct -from harness import raises +import pytest from weaviate_client_web._framing import ( FrameError, @@ -44,7 +44,7 @@ def test_split_response_multiple_messages(): def test_split_response_message_after_trailer_raises(): body = _frame(b"a") + _frame(b"grpc-status:0\r\n", 0x80) + _frame(b"late") - with raises(FrameError, contains="after the trailer"): + with pytest.raises(FrameError, match="after the trailer"): split_response(body) @@ -97,18 +97,18 @@ def test_parse_trailers_keeps_status_when_a_key_is_not_ascii(): def test_truncated_frame_raises(): framed = encode_message(b"hello")[:-2] - with raises(TruncatedFrameError): + with pytest.raises(TruncatedFrameError): list(iter_frames(framed)) - with raises(TruncatedFrameError): + with pytest.raises(TruncatedFrameError): list(iter_frames(b"\x00\x00\x00")) # shorter than one frame header -def test_unknown_frame_flag_raises(): +@pytest.mark.parametrize("first_byte", [b"{", b"<", b"\x02", b"\x40", b"\xff"]) +def test_unknown_frame_flag_raises(first_byte): # a JSON / HTML body, or a flag bit this transport does not know - for first_byte in (b"{", b"<", b"\x02", b"\x40", b"\xff"): - body = first_byte + b"\x00\x00\x00\x01x" - with raises(UnknownFrameFlagError, contains="unknown grpc-web frame flag"): - list(iter_frames(body)) + body = first_byte + b"\x00\x00\x00\x01x" + with pytest.raises(UnknownFrameFlagError, match="unknown grpc-web frame flag"): + list(iter_frames(body)) def test_frame_errors_are_value_errors(): @@ -118,5 +118,5 @@ def test_frame_errors_are_value_errors(): def test_compressed_message_frame_rejected(): body = _frame(b"x", 0x01) - with raises(FrameError, contains="compressed"): + with pytest.raises(FrameError, match="compressed"): split_response(body) diff --git a/packages/web/tests/test_httpx_fetch.py b/packages/web/tests/test_httpx_fetch.py index c12a29d5f..c1833d74c 100644 --- a/packages/web/tests/test_httpx_fetch.py +++ b/packages/web/tests/test_httpx_fetch.py @@ -1,20 +1,20 @@ """Tests for the fetch-based httpx transport (``_httpx_fetch.py``). -Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``. ``pyfetch`` is bound at import -time in ``_httpx_fetch``, so fakes patch that module attribute; ``from js import -AbortSignal`` is resolved per request, so a fake js module in ``sys.modules`` +Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs``. ``pyfetch`` is bound at +import time in ``_httpx_fetch``, so fakes patch that module attribute; ``from js +import AbortSignal`` is resolved per request, so a fake js module in ``sys.modules`` intercepts it even under real Pyodide. The install-semantics tests at the bottom run against this interpreter's real installation: importing ``weaviate_client_web`` bootstrapped the transport globally. """ -import contextlib +import sys import types from typing import Any, Dict, List, Optional import httpx -from harness import patched, raises, sys_module +import pytest import weaviate_client_web import weaviate_client_web._httpx_fetch as _httpx_fetch @@ -44,11 +44,11 @@ async def __call__(self, url: str, **kwargs: Any) -> FakeFetchResponse: return self.response -@contextlib.contextmanager -def fake_pyfetch(): +@pytest.fixture +def fake_pyfetch(monkeypatch) -> FakePyfetch: fetch = FakePyfetch() - with patched(_httpx_fetch, "pyfetch", fetch): - yield fetch + monkeypatch.setattr(_httpx_fetch, "pyfetch", fetch) + return fetch class _AbortSignalRecorder: @@ -60,21 +60,20 @@ def timeout(self, ms: int): return f"signal-{ms}" -@contextlib.contextmanager -def fake_abort_signal(): +@pytest.fixture +def fake_abort_signal(monkeypatch) -> _AbortSignalRecorder: recorder = _AbortSignalRecorder() js_mod = types.ModuleType("js") js_mod.AbortSignal = recorder # type: ignore[attr-defined] - with sys_module("js", js_mod): - yield recorder + monkeypatch.setitem(sys.modules, "js", js_mod) + return recorder -@contextlib.contextmanager -def missing_js_module(): - # sys.modules[name] = None makes ``import js`` raise ImportError — the closest - # in-process stand-in for an environment without the js bridge. - with sys_module("js", None): - yield +@pytest.fixture +def missing_js(monkeypatch): + # sys.modules["js"] = None makes ``import js`` raise ImportError — the closest + # in-process stand-in for an environment without the js bridge + monkeypatch.setitem(sys.modules, "js", None) async def _handle(request: httpx.Request) -> httpx.Response: @@ -97,72 +96,66 @@ async def _via_client(method: str, url: str, **kwargs: Any) -> httpx.Response: return await client.request(method, url, **kwargs) -async def test_basic_get_round_trip(): - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse( - status=200, headers={"content-type": "application/json"}, body=b'{"version": "1.30.0"}' - ) - response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) +async def test_basic_get_round_trip(fake_pyfetch): + fake_pyfetch.response = FakeFetchResponse( + status=200, headers={"content-type": "application/json"}, body=b'{"version": "1.30.0"}' + ) + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert response.status_code == 200 - assert response.json() == {"version": "1.30.0"} - assert response.headers["content-type"] == "application/json" - call = fetch.calls[0] - assert call["url"] == "http://h:8080/v1/meta" - assert call["method"] == "GET" + assert response.status_code == 200 + assert response.json() == {"version": "1.30.0"} + assert response.headers["content-type"] == "application/json" + call = fake_pyfetch.calls[0] + assert call["url"] == "http://h:8080/v1/meta" + assert call["method"] == "GET" -async def test_response_has_request_attached_for_raise_for_status(): - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse(status=404, body=b"") - response = await _handle(httpx.Request("GET", "http://h:8080/v1/schema/Nope")) - with raises(httpx.HTTPStatusError): - response.raise_for_status() +async def test_response_has_request_attached_for_raise_for_status(fake_pyfetch): + fake_pyfetch.response = FakeFetchResponse(status=404, body=b"") + response = await _handle(httpx.Request("GET", "http://h:8080/v1/schema/Nope")) + with pytest.raises(httpx.HTTPStatusError): + response.raise_for_status() -async def test_head_response_without_body_yields_empty_content(): +@pytest.mark.parametrize("status", [204, 404]) +async def test_head_response_without_body_yields_empty_content(fake_pyfetch, status): # data.exists() / tenants.exists() are HEAD requests answered 204/404 with a null # body; the transport must hand httpx an empty response, not fail on the missing body - for status in (204, 404): - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse(status=status, body=None) - response = await _handle(httpx.Request("HEAD", "http://h:8080/v1/objects/A/uuid")) - assert response.status_code == status - assert response.content == b"" - assert "body" not in fetch.calls[0] + fake_pyfetch.response = FakeFetchResponse(status=status, body=None) + response = await _handle(httpx.Request("HEAD", "http://h:8080/v1/objects/A/uuid")) + assert response.status_code == status + assert response.content == b"" + assert "body" not in fake_pyfetch.calls[0] -async def test_delete_204_without_body_yields_empty_content(): +async def test_delete_204_without_body_yields_empty_content(fake_pyfetch): # data.delete_by_id() / reference_delete() are answered 204 with a null body - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse(status=204, body=None) - response = await _handle(httpx.Request("DELETE", "http://h:8080/v1/objects/A/uuid")) - assert response.status_code == 204 - assert response.content == b"" + fake_pyfetch.response = FakeFetchResponse(status=204, body=None) + response = await _handle(httpx.Request("DELETE", "http://h:8080/v1/objects/A/uuid")) + assert response.status_code == 204 + assert response.content == b"" -async def test_body_less_response_through_async_client(): +async def test_body_less_response_through_async_client(fake_pyfetch): # the full httpx.AsyncClient path (stream wrapping + read) on a body-less response - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse(status=204, body=None) - response = await _via_client("HEAD", "http://h:8080/v1/objects/A/uuid") - assert response.status_code == 204 - assert response.content == b"" + fake_pyfetch.response = FakeFetchResponse(status=204, body=None) + response = await _via_client("HEAD", "http://h:8080/v1/objects/A/uuid") + assert response.status_code == 204 + assert response.content == b"" -async def test_response_through_async_client_exposes_elapsed_and_content(): +async def test_response_through_async_client_exposes_elapsed_and_content(fake_pyfetch): # the batch-references path reads ``res.elapsed``, which httpx only sets after it # has read/closed a stream-backed response; a pre-loaded body never gets one payload = b'[{"result": {"status": "SUCCESS"}}]' - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse(status=200, body=payload) - response = await _via_client("POST", "http://h:8080/v1/batch/references", content=b"[]") - assert response.content == payload - assert response.json() == [{"result": {"status": "SUCCESS"}}] - assert response.elapsed.total_seconds() >= 0 + fake_pyfetch.response = FakeFetchResponse(status=200, body=payload) + response = await _via_client("POST", "http://h:8080/v1/batch/references", content=b"[]") + assert response.content == payload + assert response.json() == [{"result": {"status": "SUCCESS"}}] + assert response.elapsed.total_seconds() >= 0 -async def test_fetch_managed_request_headers_stripped(): +async def test_fetch_managed_request_headers_stripped(fake_pyfetch): request = httpx.Request( "POST", "http://h:8080/v1/objects", @@ -176,76 +169,63 @@ async def test_fetch_managed_request_headers_stripped(): }, content=b"{}", ) - with fake_pyfetch() as fetch: - await _handle(request) - sent = fetch.calls[0]["headers"] - assert sent["authorization"] == "Bearer k" - assert sent["content-type"] == "application/json" - for managed in ( - "host", - "connection", - "accept-encoding", - "content-length", - "transfer-encoding", - ): - assert managed not in sent, managed - - -async def test_get_without_body_omits_body_kwarg(): + await _handle(request) + sent = fake_pyfetch.calls[0]["headers"] + assert sent["authorization"] == "Bearer k" + assert sent["content-type"] == "application/json" + for managed in ("host", "connection", "accept-encoding", "content-length", "transfer-encoding"): + assert managed not in sent + + +async def test_get_without_body_omits_body_kwarg(fake_pyfetch): # fetch rejects GET/HEAD requests that carry a body, so the kwarg must be absent - with fake_pyfetch() as fetch: - await _handle(httpx.Request("GET", "http://h:8080/v1/.well-known/ready")) - assert "body" not in fetch.calls[0] + await _handle(httpx.Request("GET", "http://h:8080/v1/.well-known/ready")) + assert "body" not in fake_pyfetch.calls[0] -async def test_post_body_passed(): - with fake_pyfetch() as fetch: - await _handle(httpx.Request("POST", "http://h:8080/v1/graphql", content=b'{"query": "x"}')) - assert fetch.calls[0]["body"] == b'{"query": "x"}' +async def test_post_body_passed(fake_pyfetch): + await _handle(httpx.Request("POST", "http://h:8080/v1/graphql", content=b'{"query": "x"}')) + assert fake_pyfetch.calls[0]["body"] == b'{"query": "x"}' -async def test_delete_with_body_passed(): +async def test_delete_with_body_passed(fake_pyfetch): # the REST batch-delete path sends DELETE with a JSON body - with fake_pyfetch() as fetch: - await _handle( - httpx.Request("DELETE", "http://h:8080/v1/batch/objects", content=b'{"match": {}}') - ) - assert fetch.calls[0]["body"] == b'{"match": {}}' + await _handle( + httpx.Request("DELETE", "http://h:8080/v1/batch/objects", content=b'{"match": {}}') + ) + assert fake_pyfetch.calls[0]["body"] == b'{"match": {}}' -async def test_query_string_preserved_in_url(): - with fake_pyfetch() as fetch: - await _handle(httpx.Request("GET", "http://h:8080/v1/objects?class=A&limit=10&after=a%20b")) - assert fetch.calls[0]["url"] == "http://h:8080/v1/objects?class=A&limit=10&after=a%20b" +async def test_query_string_preserved_in_url(fake_pyfetch): + await _handle(httpx.Request("GET", "http://h:8080/v1/objects?class=A&limit=10&after=a%20b")) + assert fake_pyfetch.calls[0]["url"] == "http://h:8080/v1/objects?class=A&limit=10&after=a%20b" -async def test_content_encoding_stripped_from_response(): +async def test_content_encoding_stripped_from_response(fake_pyfetch): # fetch hands back ALREADY-decompressed bytes; if the original content-encoding # header were passed through, httpx.Response would gunzip a second time and raise # DecodingError. content-length is stale for the same reason. - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse( - status=200, - headers={"content-encoding": "gzip", "content-length": "23", "x-other": "kept"}, - body=b'{"version": "1.30.0"}', - ) - response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert response.json() == {"version": "1.30.0"} - assert "content-encoding" not in response.headers - assert response.headers["x-other"] == "kept" + fake_pyfetch.response = FakeFetchResponse( + status=200, + headers={"content-encoding": "gzip", "content-length": "23", "x-other": "kept"}, + body=b'{"version": "1.30.0"}', + ) + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert response.json() == {"version": "1.30.0"} + assert "content-encoding" not in response.headers + assert response.headers["x-other"] == "kept" -async def test_unreadable_response_headers_tolerated(): +async def test_unreadable_response_headers_tolerated(fake_pyfetch): class BadHeaders: def keys(self): raise TypeError("header shape varies across Pyodide versions") - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse(status=200, body=b"ok") - fetch.response.headers = BadHeaders() - response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert response.status_code == 200 - assert response.content == b"ok" + fake_pyfetch.response = FakeFetchResponse(status=200, body=b"ok") + fake_pyfetch.response.headers = BadHeaders() + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert response.status_code == 200 + assert response.content == b"ok" def _request_with_timeout(timeouts: Dict[str, Optional[float]]) -> httpx.Request: @@ -254,68 +234,57 @@ def _request_with_timeout(timeouts: Dict[str, Optional[float]]) -> httpx.Request return request -async def test_read_timeout_maps_to_abort_signal_ms(): +async def test_read_timeout_maps_to_abort_signal_ms(fake_pyfetch, fake_abort_signal): # mirrors what weaviate's AsyncClient puts in extensions: connect/read/write/pool - with fake_pyfetch() as fetch, fake_abort_signal() as signals: - await _handle( - _request_with_timeout({"connect": 2.0, "read": 30.0, "write": 5.0, "pool": 9.0}) - ) - assert signals.timeouts == [30000] - assert fetch.calls[0]["signal"] == "signal-30000" + await _handle(_request_with_timeout({"connect": 2.0, "read": 30.0, "write": 5.0, "pool": 9.0})) + assert fake_abort_signal.timeouts == [30000] + assert fake_pyfetch.calls[0]["signal"] == "signal-30000" -async def test_read_none_means_no_deadline_even_with_pool_and_connect_set(): +async def test_read_none_means_no_deadline_even_with_pool_and_connect_set( + fake_pyfetch, fake_abort_signal +): # what the base client hands over for a non-finite request timeout: read=None with the # session pool timeout still set; falling back to pool/connect would abort a long # insert after 5 s - with fake_pyfetch() as fetch, fake_abort_signal() as signals: - await _handle( - _request_with_timeout({"connect": None, "read": None, "write": None, "pool": 5}) - ) - await _handle( - _request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0}) - ) - assert signals.timeouts == [] - assert all("signal" not in c for c in fetch.calls) + await _handle(_request_with_timeout({"connect": None, "read": None, "write": None, "pool": 5})) + await _handle(_request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0})) + assert fake_abort_signal.timeouts == [] + assert all("signal" not in c for c in fake_pyfetch.calls) -async def test_read_timeout_alone_sets_the_deadline(): - with fake_pyfetch() as fetch, fake_abort_signal() as signals: - await _handle(_request_with_timeout({"connect": None, "read": 7, "write": None, "pool": 5})) - assert signals.timeouts == [7000] - assert fetch.calls[0]["signal"] == "signal-7000" +async def test_read_timeout_alone_sets_the_deadline(fake_pyfetch, fake_abort_signal): + await _handle(_request_with_timeout({"connect": None, "read": 7, "write": None, "pool": 5})) + assert fake_abort_signal.timeouts == [7000] + assert fake_pyfetch.calls[0]["signal"] == "signal-7000" -async def test_no_timeout_extension_sends_no_signal(): - with fake_pyfetch() as fetch, fake_abort_signal() as signals: - await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert signals.timeouts == [] - assert "signal" not in fetch.calls[0] +async def test_no_timeout_extension_sends_no_signal(fake_pyfetch, fake_abort_signal): + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert fake_abort_signal.timeouts == [] + assert "signal" not in fake_pyfetch.calls[0] -async def test_missing_js_module_degrades_to_no_signal(): +async def test_missing_js_module_degrades_to_no_signal(fake_pyfetch, missing_js): # without the js bridge the AbortSignal import fails; the request must still go out - with fake_pyfetch() as fetch, missing_js_module(): - response = await _handle( - _request_with_timeout({"connect": 2.0, "read": 30.0, "write": None, "pool": None}) - ) - assert response.status_code == 200 - assert "signal" not in fetch.calls[0] + response = await _handle( + _request_with_timeout({"connect": 2.0, "read": 30.0, "write": None, "pool": None}) + ) + assert response.status_code == 200 + assert "signal" not in fake_pyfetch.calls[0] -async def test_zero_timeout_means_no_deadline(): +async def test_zero_timeout_means_no_deadline(fake_pyfetch, fake_abort_signal): # an explicit read=0 must not fall through to the 5s connect timeout, nor become an # immediate AbortSignal.timeout(0) - with fake_pyfetch() as fetch, fake_abort_signal() as signals: - await _handle( - _request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None}) - ) - assert signals.timeouts == [] - assert "signal" not in fetch.calls[0] + await _handle(_request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None})) + assert fake_abort_signal.timeouts == [] + assert "signal" not in fake_pyfetch.calls[0] -def test_abort_signal_ms_bounds(): - cases = [ +@pytest.mark.parametrize( + "timeout,expected_ms", + [ (None, None), (0, None), (-1, None), @@ -326,29 +295,28 @@ def test_abort_signal_ms_bounds(): (1e8, _MAX_ABORT_SIGNAL_MS), (1e10, _MAX_ABORT_SIGNAL_MS), (1e308, _MAX_ABORT_SIGNAL_MS), # finite, but *1000 overflows: capped, not an error - ] - for timeout, expected_ms in cases: - assert _abort_signal_ms(timeout) == expected_ms, timeout + ], +) +def test_abort_signal_ms_bounds(timeout, expected_ms): + assert _abort_signal_ms(timeout) == expected_ms -async def test_infinite_timeout_sends_no_signal(): +async def test_infinite_timeout_sends_no_signal(fake_pyfetch, fake_abort_signal): # an inf read deadline reaching the transport: no signal, not an OverflowError - with fake_pyfetch() as fetch, fake_abort_signal() as signals: - await _handle( - _request_with_timeout({"connect": None, "read": float("inf"), "write": None, "pool": 5}) - ) - assert signals.timeouts == [] - assert "signal" not in fetch.calls[0] + await _handle( + _request_with_timeout({"connect": None, "read": float("inf"), "write": None, "pool": 5}) + ) + assert fake_abort_signal.timeouts == [] + assert "signal" not in fake_pyfetch.calls[0] -async def test_huge_timeout_is_capped_to_int32_ms(): +async def test_huge_timeout_is_capped_to_int32_ms(fake_pyfetch, fake_abort_signal): # setTimeout delays above 2^31-1 ms overflow and fire at once, aborting the request - with fake_pyfetch() as fetch, fake_abort_signal() as signals: - await _handle( - _request_with_timeout({"connect": None, "read": 1e10, "write": None, "pool": None}) - ) - assert signals.timeouts == [_MAX_ABORT_SIGNAL_MS] - assert fetch.calls[0]["signal"] == f"signal-{_MAX_ABORT_SIGNAL_MS}" + await _handle( + _request_with_timeout({"connect": None, "read": 1e10, "write": None, "pool": None}) + ) + assert fake_abort_signal.timeouts == [_MAX_ABORT_SIGNAL_MS] + assert fake_pyfetch.calls[0]["signal"] == f"signal-{_MAX_ABORT_SIGNAL_MS}" class RaisingPyfetch: @@ -359,68 +327,67 @@ async def __call__(self, url: str, **kwargs: Any): raise self.exc -@contextlib.contextmanager -def raising_pyfetch(exc: BaseException): - with patched(_httpx_fetch, "pyfetch", RaisingPyfetch(exc)): - yield +def _install_raising_pyfetch(monkeypatch, exc: BaseException) -> None: + monkeypatch.setattr(_httpx_fetch, "pyfetch", RaisingPyfetch(exc)) -async def test_fetch_failure_maps_to_httpx_connect_error(): +async def test_fetch_failure_maps_to_httpx_connect_error(monkeypatch): # pyodide surfaces JS fetch rejections as OSError; the base client can only classify # httpx exceptions (WeaviateConnectionError etc.), so the shim must translate - with raising_pyfetch(OSError("TypeError: Failed to fetch")): - with raises(httpx.ConnectError, contains="Failed to fetch") as excinfo: - await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert isinstance(excinfo.value.__cause__, OSError) + _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) + with pytest.raises(httpx.ConnectError, match="Failed to fetch") as excinfo: + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert isinstance(excinfo.value.__cause__, OSError) -async def test_fetch_abort_with_deadline_maps_to_read_timeout(): +async def test_fetch_abort_with_deadline_maps_to_read_timeout(monkeypatch, fake_abort_signal): # AbortSignal.timeout firing surfaces as an OSError subclass mentioning the abort; # with a deadline set this must classify as a timeout, not a connection error - with raising_pyfetch(OSError("AbortError: signal timed out")), fake_abort_signal(): - with raises(httpx.ReadTimeout, contains="signal timed out"): - await _handle( - _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) - ) + _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) + with pytest.raises(httpx.ReadTimeout, match="signal timed out"): + await _handle( + _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) + ) -async def test_fetch_failure_with_deadline_but_no_timeout_message_stays_connect_error(): +async def test_fetch_failure_with_deadline_but_no_timeout_message_stays_connect_error( + monkeypatch, fake_abort_signal +): # nearly every weaviate request sets a read deadline; a plain network failure on # such a request must remain a connection error, not become a timeout - with raising_pyfetch(OSError("TypeError: Failed to fetch")), fake_abort_signal(): - with raises(httpx.ConnectError, contains="Failed to fetch"): - await _handle( - _request_with_timeout({"connect": None, "read": 30.0, "write": None, "pool": None}) - ) + _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) + with pytest.raises(httpx.ConnectError, match="Failed to fetch"): + await _handle( + _request_with_timeout({"connect": None, "read": 30.0, "write": None, "pool": None}) + ) -async def test_fetch_abort_without_deadline_stays_connect_error(): +async def test_fetch_abort_without_deadline_stays_connect_error(monkeypatch, missing_js): # the same message without a deadline set (no js bridge -> no signal) is not OUR # timeout, so it must stay a connection error - with raising_pyfetch(OSError("AbortError: signal timed out")), missing_js_module(): - with raises(httpx.ConnectError): - await _handle( - _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) - ) + _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) + with pytest.raises(httpx.ConnectError): + await _handle( + _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) + ) -async def test_empty_oserror_str_keeps_repr_detail(): - with raising_pyfetch(OSError()): - with raises(httpx.ConnectError) as excinfo: - await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) - assert "OSError" in str(excinfo.value) +async def test_empty_oserror_str_keeps_repr_detail(monkeypatch): + _install_raising_pyfetch(monkeypatch, OSError()) + with pytest.raises(httpx.ConnectError) as excinfo: + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert "OSError" in str(excinfo.value) -async def test_crlf_in_header_value_rejected(): +async def test_crlf_in_header_value_rejected(fake_pyfetch): # httpx.Request accepts CR/LF in header values and relies on h11 to reject them at # send time; this transport bypasses h11 and must keep that defence request = httpx.Request( "GET", "http://h:8080/v1/meta", headers={"x-key": "val\r\nx-injected: evil"} ) - with fake_pyfetch() as fetch: - with raises(httpx.LocalProtocolError): - await _handle(request) - assert fetch.calls == [] + with pytest.raises(httpx.LocalProtocolError): + await _handle(request) + assert fake_pyfetch.calls == [] # --------------------------------------------------------------------------- @@ -450,15 +417,18 @@ def test_sync_transport_left_untouched(): def test_uninstall_restores_original_transport(): + # the module keeps the pre-patch method while installed, so the restore can be + # checked by identity even though the install happened at import time + original = _httpx_fetch._original_handle_async_request + assert original is not None patched_method = httpx.AsyncHTTPTransport.handle_async_request try: weaviate_client_web.uninstall_fetch_transport() assert not weaviate_client_web.is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is original assert httpx.AsyncHTTPTransport.handle_async_request is not patched_method - assert not getattr( - httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False - ) weaviate_client_web.uninstall_fetch_transport() # no-op when not installed + assert httpx.AsyncHTTPTransport.handle_async_request is original finally: weaviate_client_web.install_fetch_transport() assert weaviate_client_web.is_fetch_transport_installed() @@ -468,14 +438,13 @@ def test_uninstall_restores_original_transport(): ) -async def test_installed_transport_routes_async_client_through_pyfetch(): +async def test_installed_transport_routes_async_client_through_pyfetch(fake_pyfetch): # the globally installed transport (no custom transport argument) must reach pyfetch - with fake_pyfetch() as fetch: - fetch.response = FakeFetchResponse( - status=200, headers={"content-type": "application/json"}, body=b'{"ok": true}' - ) - async with httpx.AsyncClient() as client: - response = await client.get("http://h:8080/v1/meta") - assert response.status_code == 200 - assert response.json() == {"ok": True} - assert fetch.calls and fetch.calls[0]["url"] == "http://h:8080/v1/meta" + fake_pyfetch.response = FakeFetchResponse( + status=200, headers={"content-type": "application/json"}, body=b'{"ok": true}' + ) + async with httpx.AsyncClient() as client: + response = await client.get("http://h:8080/v1/meta") + assert response.status_code == 200 + assert response.json() == {"ok": True} + assert fake_pyfetch.calls and fake_pyfetch.calls[0]["url"] == "http://h:8080/v1/meta" diff --git a/packages/web/tests/test_shim_install.py b/packages/web/tests/test_shim_install.py index d5dede428..0efab5973 100644 --- a/packages/web/tests/test_shim_install.py +++ b/packages/web/tests/test_shim_install.py @@ -1,14 +1,15 @@ """grpc shim tests, against this interpreter's real installation. -Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``: importing ``weaviate_client_web`` -bootstrapped the shim, so ``sys.modules['grpc']`` here IS the shim. Bootstrap scenarios -that need a clean import state (a bare ``import weaviate``, a missing or broken -companion) live in ``units.mjs``, one fresh interpreter each. +Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs``: importing +``weaviate_client_web`` bootstrapped the shim, so ``sys.modules['grpc']`` here IS the +shim. Bootstrap scenarios that need a clean import state live in ``units.mjs`` (one +fresh interpreter) and, for the base client's hook logic, in +``test/test_wasm_compat.py`` on CPython. """ import struct -from harness import raises +import pytest import weaviate_client_web from weaviate_client_web import GrpcWebChannel, set_sender @@ -24,8 +25,9 @@ def test_import_weaviate_under_shim(): import grpc assert weaviate_client_web.is_installed() + assert weaviate_client_web.install() is True # idempotent, reports the shim in place assert getattr(grpc, "__weaviate_client_web_shim__", False) is True - assert grpc.__version__ == FAKE_GRPC_VERSION + assert grpc.__version__ == "1.72.1" assert grpc._utilities.first_version_is_lower("1.0.0", "2.0.0") is False # type: ignore[attr-defined] from grpc.aio._typing import ChannelArgumentType # noqa: F401 @@ -43,9 +45,8 @@ def test_import_weaviate_under_shim(): def test_sync_channel_factory_raises_async_only(): import grpc - with raises(RuntimeError) as excinfo: + with pytest.raises(RuntimeError, match="async"): grpc.insecure_channel("localhost:50051") - assert "async" in str(excinfo.value).lower() async def test_real_proto_unary_round_trip_under_shim(): diff --git a/packages/web/tests/test_transport.py b/packages/web/tests/test_transport.py index 9bb118b97..b063767ee 100644 --- a/packages/web/tests/test_transport.py +++ b/packages/web/tests/test_transport.py @@ -1,8 +1,9 @@ -"""grpc-web channel / multicallable tests. Run inside Pyodide via ``ci/pyodide-e2e/units.mjs``. +"""grpc-web channel / multicallable tests. -These exercise the transport classes directly through fake senders — no network, no -running Weaviate. Tests are ``async def`` awaited on Pyodide's event loop by -``runner.py`` (``asyncio.run()`` cannot be used inside Pyodide). +Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs`` — async tests execute on +Pyodide's event loop through JSPI stack switching (pytest-asyncio in auto mode). These +exercise the transport classes directly through fake senders — no network, no running +Weaviate. """ import asyncio @@ -10,7 +11,7 @@ import sys from typing import Dict, List, Optional, Tuple -from harness import patched, raises +import pytest from weaviate_client_web import GrpcWebChannel, set_sender from weaviate_client_web._channel import _body_excerpt, _encode_timeout @@ -97,7 +98,7 @@ async def test_error_trailer_raises_aiorpcerror(): channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.PERMISSION_DENIED assert excinfo.value.code().name == "PERMISSION_DENIED" @@ -108,7 +109,7 @@ async def test_percent_encoded_grpc_message_decoded(): body = _frame(b"grpc-status:5\r\ngrpc-message:not%20found\r\n", 0x80) channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.details() == "not found" @@ -118,7 +119,7 @@ async def test_trailers_only_status_in_http_headers(): FakeSender(status=200, headers={"grpc-status": "16", "grpc-message": "auth"}, body=b"") ) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.UNAUTHENTICATED @@ -155,7 +156,7 @@ async def _details_of(status, body, headers=None, path="/grpc.health.v1.Health/C """Run one request against a canned HTTP response and return the AioRpcError.""" channel = _channel(FakeSender(status=status, headers=headers or {}, body=body)) mc = channel.unary_unary(path, lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") return excinfo.value @@ -185,10 +186,10 @@ async def test_nginx_502_maps_to_unavailable_so_the_client_retries(): assert "502 Bad Gateway" in err.details() -async def test_gateway_errors_are_unavailable(): - for status in (503, 504): - err = await _details_of(status, b"upstream down") - assert err.code() is StatusCode.UNAVAILABLE, status +@pytest.mark.parametrize("status", [503, 504]) +async def test_gateway_errors_are_unavailable(status): + err = await _details_of(status, b"upstream down") + assert err.code() is StatusCode.UNAVAILABLE async def test_nginx_404_html_is_reported_as_an_http_404(): @@ -337,7 +338,7 @@ async def test_binary_metadata_base64_encoded(): def test_stream_stream_raises_clear_error(): channel = _channel(FakeSender()) mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) - with raises(RuntimeError, contains="not supported over grpc-web"): + with pytest.raises(RuntimeError, match="not supported over grpc-web"): mc(request_iterator=iter([]), timeout=5, metadata=None) @@ -348,7 +349,7 @@ async def slow_sender(url, headers, body, timeout): channel = GrpcWebChannel("h:1", secure=False, sender=slow_sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q", timeout=0.01) assert excinfo.value.code() is StatusCode.DEADLINE_EXCEEDED @@ -359,7 +360,7 @@ async def boom(url, headers, body, timeout): channel = GrpcWebChannel("h:1", secure=False, sender=boom) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.UNAVAILABLE assert "ConnectionError: connection refused" in str(excinfo.value.details()) @@ -372,7 +373,7 @@ async def boom(url, headers, body, timeout): channel = GrpcWebChannel("h:1", secure=False, sender=boom) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert "ConnectionError" in str(excinfo.value.details()) @@ -382,7 +383,7 @@ async def test_empty_ok_response_hints_at_cors_expose_headers(): # whose grpc-status/grpc-message headers were stripped by CORS channel = _channel(FakeSender(status=200, headers={}, body=b"")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "Access-Control-Expose-Headers" in str(excinfo.value.details()) @@ -393,7 +394,7 @@ async def test_empty_ok_response_with_grpc_status_has_no_cors_hint(): # not a CORS problem — the hint must not appear channel = _channel(FakeSender(status=200, headers={"grpc-status": "0"}, body=b"")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "Access-Control-Expose-Headers" not in str(excinfo.value.details()) @@ -404,7 +405,7 @@ async def test_message_frame_without_grpc_status_is_internal_not_success(): # dropped the trailer frame) must be an error, never a fabricated success channel = _channel(FakeSender(status=200, headers={}, body=_frame(b"reply-bytes"))) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "missing grpc-status" in str(excinfo.value.details()) @@ -425,7 +426,7 @@ def test_stream_stream_error_recommends_insert_many_only(): # only one supported under WASM), so the error must not recommend them channel = _channel(FakeSender()) mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) - with raises(RuntimeError) as excinfo: + with pytest.raises(RuntimeError) as excinfo: mc(request_iterator=iter([]), timeout=5, metadata=None) assert "insert_many" in str(excinfo.value) for sync_only in ("dynamic", "fixed_size", "rate_limit"): @@ -436,7 +437,7 @@ async def test_malformed_frame_maps_to_internal(): # A 3-byte body cannot contain even a 5-byte frame header -> framing ValueError. channel = _channel(FakeSender(body=b"\x00\x00\x00")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL @@ -445,7 +446,7 @@ async def test_malformed_grpc_status_maps_to_internal(): body = _frame(b"grpc-status:notanint\r\n", 0x80) channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: + with pytest.raises(AioRpcError) as excinfo: await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL @@ -465,8 +466,9 @@ def test_body_excerpt_empty_and_non_printable(): assert _body_excerpt(b"ok\x00\x01") == "ok" -def test_encode_timeout_stays_within_eight_digits(): - cases = [ +@pytest.mark.parametrize( + "seconds,expected", + [ (None, None), (float("inf"), None), (float("nan"), None), @@ -481,13 +483,14 @@ def test_encode_timeout_stays_within_eight_digits(): (1e10, None), # would need hours, which transcoders reject above 8H: no deadline (1e15, None), (1e308, None), # finite, but *1000 overflows to infinity: must not raise - ] - for seconds, expected in cases: - encoded = _encode_timeout(seconds) - assert encoded == expected, (seconds, encoded, expected) - if encoded is not None: - assert len(encoded) <= 9, encoded # 8 digits + unit - assert not encoded.endswith("H"), encoded + ], +) +def test_encode_timeout_stays_within_eight_digits(seconds, expected): + encoded = _encode_timeout(seconds) + assert encoded == expected + if encoded is not None: + assert len(encoded) <= 9 # 8 digits + unit + assert not encoded.endswith("H") async def test_infinite_timeout_sends_no_deadline(): @@ -517,48 +520,52 @@ async def test_huge_timeout_uses_minutes_then_no_deadline(): assert sender.calls[2][3] is None # no client-side wait either -async def test_crlf_in_metadata_rejected(): +@pytest.mark.parametrize("bad", ["val\r\nx-injected: evil", "val\nx", "v\0"]) +async def test_crlf_in_metadata_rejected(bad): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - for bad in ("val\r\nx-injected: evil", "val\nx", "v\0"): - with raises(ValueError, contains="Illegal character"): - await mc(b"q", metadata=[("x-key", bad)]) - with raises(ValueError, contains="Illegal character"): - await mc(b"q", metadata=[("x-key\r\n", "v")]) + with pytest.raises(ValueError, match="Illegal character"): + await mc(b"q", metadata=[("x-key", bad)]) + with pytest.raises(ValueError, match="Illegal character"): + await mc(b"q", metadata=[("x-key\r\n", "v")]) assert sender.calls == [] -async def _unavailable_details(path_prefix, platform): +async def _unavailable_details(monkeypatch, path_prefix, platform): async def boom(url, headers, body, timeout): raise ConnectionError("Failed to fetch") - with patched(sys, "platform", platform): - channel = GrpcWebChannel("h:50051", secure=False, sender=boom, path_prefix=path_prefix) - mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - with raises(AioRpcError) as excinfo: - await mc(b"q") + monkeypatch.setattr(sys, "platform", platform) + channel = GrpcWebChannel("h:50051", secure=False, sender=boom, path_prefix=path_prefix) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q") assert excinfo.value.code() is StatusCode.UNAVAILABLE return excinfo.value.details() -async def test_unavailable_without_path_prefix_under_emscripten_hints_at_grpc_path_prefix(): +async def test_unavailable_without_path_prefix_under_emscripten_hints_at_grpc_path_prefix( + monkeypatch, +): # the connect helpers always set the prefix under Emscripten, so a prefix-less channel # here means hand-built ConnectionParams; the error must say what to do instead - details = await _unavailable_details(path_prefix="", platform="emscripten") + details = await _unavailable_details(monkeypatch, path_prefix="", platform="emscripten") assert "grpc_path_prefix='/v1/grpc-web'" in details assert "1.38.3" in details assert "connect helpers" in details -async def test_unavailable_with_path_prefix_has_no_prefix_hint(): - details = await _unavailable_details(path_prefix="/v1/grpc-web", platform="emscripten") +async def test_unavailable_with_path_prefix_has_no_prefix_hint(monkeypatch): + details = await _unavailable_details( + monkeypatch, path_prefix="/v1/grpc-web", platform="emscripten" + ) assert "no grpc_path_prefix" not in details -async def test_unavailable_without_path_prefix_off_emscripten_has_no_prefix_hint(): +async def test_unavailable_without_path_prefix_off_emscripten_has_no_prefix_hint(monkeypatch): # off Emscripten an empty prefix against a transcoder is the normal configuration - details = await _unavailable_details(path_prefix="", platform="linux") + details = await _unavailable_details(monkeypatch, path_prefix="", platform="linux") assert "no grpc_path_prefix" not in details @@ -577,19 +584,21 @@ async def test_path_prefix_prepended_to_url(): assert sender.calls[0][0] == "http://example.com:8090/grpc-web/weaviate.v1.Weaviate/Search" -async def test_path_prefix_normalized_in_url(): - cases = [ +@pytest.mark.parametrize( + "raw,expected_url", + [ ("grpc-web", "http://h:1/grpc-web/svc/M"), ("/grpc-web/", "http://h:1/grpc-web/svc/M"), ("/a/b", "http://h:1/a/b/svc/M"), ("", "http://h:1/svc/M"), - ] - for raw, expected_url in cases: - sender = FakeSender(body=_ok_response(b"r")) - channel = GrpcWebChannel("h:1", secure=False, sender=sender, path_prefix=raw) - mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - await mc(b"q") - assert sender.calls[0][0] == expected_url, raw + ], +) +async def test_path_prefix_normalized_in_url(raw, expected_url): + sender = FakeSender(body=_ok_response(b"r")) + channel = GrpcWebChannel("h:1", secure=False, sender=sender, path_prefix=raw) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + await mc(b"q") + assert sender.calls[0][0] == expected_url def test_shim_factory_extracts_path_prefix_option(): diff --git a/test/test_wasm_compat.py b/test/test_wasm_compat.py index 5398a825a..ab2da4bb7 100644 --- a/test/test_wasm_compat.py +++ b/test/test_wasm_compat.py @@ -6,7 +6,10 @@ """ import asyncio +import pathlib +import subprocess import sys +import textwrap import grpc import pytest @@ -288,3 +291,105 @@ def test_an_explicit_local_grpc_port_is_warned_about_but_the_default_is_not(emsc with pytest.warns(UserWarning, match="Con006"): client = weaviate.use_async_with_local(port=8080, grpc_port=8081) _assert_grpc_rides_rest(client) + + +# --- the single-import hook (weaviate/__init__.py) ------------------------------------ +# +# The hook fires on sys.platform == "emscripten" and (via the companion's bootstrap) +# replaces sys.modules['grpc'] process-wide, so each scenario runs in a fresh subprocess +# with the platform faked before `import weaviate`. The success path — a bare import +# that bootstraps the real companion — needs real Pyodide and runs in +# ci/pyodide-e2e/units.mjs; the hook's other branches are plain CPython logic and are +# pinned here. + +_REPO_ROOT = str(pathlib.Path(__file__).resolve().parents[1]) + +# CPython derives the _sysconfigdata module name from sys.platform on first use, so a +# faked platform breaks any later sysconfig lookup (pydantic imports zoneinfo, which +# calls sysconfig.get_config_var). Prime the cache before faking. +_PRIME_SYSCONFIG = """ +import sysconfig + +sysconfig.get_config_vars() +""" + + +def _run_hook_scenario( + body: str, *, prelude: str = "", path_entry: str = _REPO_ROOT, no_site: bool = False +) -> subprocess.CompletedProcess: + # -I -S: skip site-packages entirely (plain -I still processes the venv's .pth + # files), so nothing pip-installed is importable — only stdlib plus `path_entry`. + interp = [sys.executable, "-I", "-S"] if no_site else [sys.executable] + script = f"import sys\nsys.path.insert(0, {path_entry!r})\n" + prelude + textwrap.dedent(body) + return subprocess.run([*interp, "-c", script], capture_output=True, text=True) + + +def test_bare_import_without_companion_raises_clear_import_error() -> None: + # No site-packages, so neither weaviate_client_web nor grpcio is importable; the repo + # root goes on sys.path so the weaviate package itself is still found. + result = _run_hook_scenario( + """ + sys.platform = "emscripten" + try: + import weaviate + except ImportError as e: + assert "weaviate-client-web" in str(e), str(e) + assert "weaviate-client[grpc-web]" in str(e), str(e) + assert "WebAssembly/Pyodide" in str(e), str(e) + print("OK") + else: + raise AssertionError("expected ImportError without the companion") + """, + no_site=True, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_bare_import_with_grpc_present_falls_through_silently() -> None: + # Companion blocked but a real grpc IS importable (grpcio in the dev env): the hook + # must fall through and leave the normal import path untouched. + result = _run_hook_scenario( + prelude=_PRIME_SYSCONFIG, + body=""" + sys.platform = "emscripten" + sys.modules["weaviate_client_web"] = None # makes its import raise ImportError + + import weaviate + import grpc + + assert not getattr(grpc, "__weaviate_client_web_shim__", False) + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_bare_import_with_broken_companion_surfaces_its_own_error(tmp_path) -> None: + # An INSTALLED companion whose import fails (here: a missing dependency of its own) + # must raise that error, not the install hint — the hint would send the user to + # reinstall a package that is already there. + fake_pkg = tmp_path / "weaviate_client_web" + fake_pkg.mkdir() + (fake_pkg / "__init__.py").write_text( + "raise ModuleNotFoundError(\"No module named 'anyio'\", name='anyio')\n" + ) + result = _run_hook_scenario( + prelude=_PRIME_SYSCONFIG, + body=""" + sys.platform = "emscripten" + try: + import weaviate + except ImportError as e: + assert e.name == "anyio", (e.name, str(e)) + assert "anyio" in str(e), str(e) + assert "grpc-web" not in str(e), str(e) + print("OK") + else: + raise AssertionError("expected the companion's own ImportError to surface") + """, + path_entry=str(tmp_path), + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout From c98ca479bef1bb11335f207e9eb1e26434f6796a Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:04:34 +0200 Subject: [PATCH 5/6] ci(grpc-web): publish the companion wheel alongside the base client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release job builds the weaviate-client-web wheel into dist and asserts both packages carry the same version on the artifacts actually uploaded — the companion's weaviate-client== pin makes a base-only release leave the [grpc-web] extra unresolvable. The companion is wheel-only: its setup.py resolves the lockstep version from git tags, which an unpacked sdist would not have. The wheel artifact for the GitHub release includes it too. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Caze9m6PBSfYkt77mMKj2b --- .github/workflows/main.yaml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index da9ea3302..d5765e6ec 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -357,8 +357,10 @@ jobs: cache: 'pip' # caching pip dependencies - name: Install dependencies run: pip install -r requirements-test.txt -r requirements-devel.txt - - name: Build a binary wheel - run: python -m build + - name: Build binary wheels (base client + grpc-web companion) + run: | + python -m build + python -m build --wheel --outdir dist packages/web - name: Create Wheel Artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -429,9 +431,23 @@ jobs: cache: 'pip' # caching pip dependencies - name: Install dependencies run: pip install -r requirements-devel.txt - - name: Build a binary wheel - run: python -m build - - name: Publish distribution 📦 to PyPI on new tags + - name: Build distributions (base client + grpc-web companion) + # The companion is wheel-only: its setup.py resolves the lockstep version from + # the repository's git tags, which an unpacked sdist would not have — and + # micropip consumes wheels only anyway. + run: | + python -m build + python -m build --wheel --outdir dist packages/web + - name: Assert both packages carry the same version + # weaviate-client-web pins weaviate-client== at build time, so + # a base-only release would leave the [grpc-web] extra unresolvable. This gate + # runs on the artifacts that are actually uploaded. + run: | + base=$(basename dist/weaviate_client-*.whl); base=${base#weaviate_client-}; base=${base%%-*} + web=$(basename dist/weaviate_client_web-*.whl); web=${web#weaviate_client_web-}; web=${web%%-*} + echo "weaviate-client=$base weaviate-client-web=$web" + test "$base" = "$web" + - name: Publish distributions 📦 to PyPI on new tags if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 with: From 568d11b99874b4bf48537761e47df33dabf15037 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:04:36 +0200 Subject: [PATCH 6/6] ci(grpc-web): pin a metadata-coherent pytest / pytest-asyncio pair pytest-asyncio 0.25.3 declares pytest<9,>=8.2, so pin pytest 8.4.2 instead of the bundled 9.0.2 rather than depending on micropip tolerating the conflict. pytest-asyncio stays on 0.25.x: the asyncio.Runner-based 1.x fails under JSPI stack switching, while 0.25.x's run_until_complete-based execution works. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Caze9m6PBSfYkt77mMKj2b --- ci/pyodide-e2e/units.mjs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/pyodide-e2e/units.mjs b/ci/pyodide-e2e/units.mjs index f333080a7..68721f36b 100644 --- a/ci/pyodide-e2e/units.mjs +++ b/ci/pyodide-e2e/units.mjs @@ -93,7 +93,13 @@ console.log( `pyodide ${pyodide.version} / python ${pyodide.runPython("import sys; sys.version.split()[0]")}`, ); const micropip = pyodide.pyimport("micropip"); -await micropip.install(["pytest==9.0.2", "pytest-asyncio==0.25.3"]); +// A metadata-coherent pair: pytest-asyncio 0.25.3 declares pytest<9,>=8.2. Its +// run_until_complete-based execution stack-switches correctly under JSPI, unlike the +// asyncio.Runner-based pytest-asyncio 1.x, whose async tests fail here. (The Pyodide +// distribution bundles pytest 9 next to pytest-asyncio 0.25.3, contradicting that +// constraint — micropip tolerates it, but there is no reason to depend on its +// leniency, so both are pinned from PyPI.) +await micropip.install(["pytest==8.4.2", "pytest-asyncio==0.25.3"]); pyodide.FS.mkdirTree("/units"); pyodide.mountNodeFS("/units", testsDir);