diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 163c70afa..d5765e6ec 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -105,28 +105,8 @@ jobs: name: coverage-report-${{ matrix.folder }} path: coverage-${{ matrix.folder }}.xml - grpc-web-tests: - name: Run gRPC-Web Package Tests - runs-on: ubuntu-latest - timeout-minutes: 5 - strategy: - fail-fast: false - matrix: - version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: ${{ matrix.version }} - cache: 'pip' # caching pip dependencies - - run: | - pip install -r requirements-test.txt -r requirements-devel.txt - pip install -e . -e packages/web - - name: Run grpc-web package tests - run: pytest packages/web/tests - pyodide-e2e: - name: Run Pyodide (WASM) e2e Tests + name: Run Pyodide (WASM) Unit + e2e Tests runs-on: ubuntu-latest timeout-minutes: 15 # No Python matrix: the pinned Pyodide bundle fixes the interpreter (see @@ -154,6 +134,22 @@ jobs: pip install build python -m build --wheel --outdir dist . python -m build --wheel --outdir dist packages/web + - name: Assert both wheels carry the same version + # weaviate-client-web is versioned in lockstep with weaviate-client: both derive + # from the same git tag via setuptools_scm. + run: | + base=$(basename dist/weaviate_client-*.whl); base=${base#weaviate_client-}; base=${base%%-*} + web=$(basename dist/weaviate_client_web-*.whl); web=${web#weaviate_client_web-}; web=${web%%-*} + echo "weaviate-client=$base weaviate-client-web=$web" + test "$base" = "$web" + - name: Run the unit suite inside Pyodide under Node + # The grpc-web package imports pyodide at module scope, so its unit tests only + # run here. Needs no running Weaviate. The JSPI flag lets pytest's synchronous + # runner await async tests via stack switching; without it the run fails + # loudly at startup (never a false green). + run: | + npm install --prefix ci/pyodide-e2e + node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist - name: start weaviate run: | source ./ci/compose.sh @@ -361,8 +357,10 @@ jobs: cache: 'pip' # caching pip dependencies - name: Install dependencies run: pip install -r requirements-test.txt -r requirements-devel.txt - - name: Build a binary wheel - run: python -m build + - name: Build binary wheels (base client + grpc-web companion) + run: | + python -m build + python -m build --wheel --outdir dist packages/web - name: Create Wheel Artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -418,7 +416,7 @@ jobs: build-and-publish: name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI - needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package, proto-test, grpc-web-tests, pyodide-e2e] + needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package, proto-test, pyodide-e2e] runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -433,9 +431,23 @@ jobs: cache: 'pip' # caching pip dependencies - name: Install dependencies run: pip install -r requirements-devel.txt - - name: Build a binary wheel - run: python -m build - - name: Publish distribution 📦 to PyPI on new tags + - name: Build distributions (base client + grpc-web companion) + # The companion is wheel-only: its setup.py resolves the lockstep version from + # the repository's git tags, which an unpacked sdist would not have — and + # micropip consumes wheels only anyway. + run: | + python -m build + python -m build --wheel --outdir dist packages/web + - name: Assert both packages carry the same version + # weaviate-client-web pins weaviate-client== at build time, so + # a base-only release would leave the [grpc-web] extra unresolvable. This gate + # runs on the artifacts that are actually uploaded. + run: | + base=$(basename dist/weaviate_client-*.whl); base=${base#weaviate_client-}; base=${base%%-*} + web=$(basename dist/weaviate_client_web-*.whl); web=${web#weaviate_client_web-}; web=${web%%-*} + echo "weaviate-client=$base weaviate-client-web=$web" + test "$base" = "$web" + - name: Publish distributions 📦 to PyPI on new tags if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 with: diff --git a/ci/pyodide-e2e/units.mjs b/ci/pyodide-e2e/units.mjs new file mode 100644 index 000000000..68721f36b --- /dev/null +++ b/ci/pyodide-e2e/units.mjs @@ -0,0 +1,136 @@ +// Runs the weaviate_client_web unit suite (packages/web/tests) with pytest inside +// Pyodide (WASM) under Node, plus a bootstrap scenario in a fresh interpreter. No +// running Weaviate is needed — everything is driven through fake senders / a fake +// pyfetch. +// +// Usage: node --experimental-wasm-jspi units.mjs +// must contain exactly the two locally-built pure wheels: +// weaviate_client-*.whl and weaviate_client_web-*.whl (same layout as run.mjs). +// +// The JSPI flag is required: pytest's runner is synchronous, so async tests execute +// through run_until_complete, which needs stack switching (enableRunUntilComplete + +// a callPromising() entrypoint). Without the flag the run fails loudly at startup — +// it can never produce a false green. +// +// The package imports pyodide at module scope, so this harness is the only place its +// unit tests can run. The base client's hook logic (missing companion, broken +// companion, grpc-present fall-through) is covered by subprocess tests in +// test/test_wasm_compat.py on CPython; the bootstrap scenario below covers the one +// path that needs real Pyodide, micropip and the wheels. +import { readdirSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { loadPyodide } from "pyodide"; + +if (!process.argv[2]) { + console.error("usage: node units.mjs "); + process.exit(2); +} +const wheelsDir = resolve(process.argv[2]); +const here = dirname(fileURLToPath(import.meta.url)); + +const wheels = readdirSync(wheelsDir) + .filter((f) => f.endsWith(".whl")) + .sort(); // installs weaviate_client before weaviate_client_web, which depends on it +const prefixes = ["weaviate_client-", "weaviate_client_web-"]; +if ( + wheels.length !== 2 || + !prefixes.every((p) => wheels.some((w) => w.startsWith(p))) +) { + console.error( + `expected exactly one weaviate_client-*.whl and one weaviate_client_web-*.whl in ${wheelsDir}, found: ${JSON.stringify(wheels)}`, + ); + process.exit(2); +} + +// Fresh interpreter with micropip ready and the wheels dir mounted. +async function freshPyodide() { + const pyodide = await loadPyodide({ enableRunUntilComplete: true }); + await pyodide.loadPackage("micropip"); + const micropip = pyodide.pyimport("micropip"); + pyodide.FS.mkdirTree("/wheels"); + pyodide.mountNodeFS("/wheels", wheelsDir); + for (const wheel of wheels) { + await micropip.install(`emfs:/wheels/${wheel}`); + } + return pyodide; +} + +// --- bootstrap scenario: needs a clean import state, so its own interpreter -------- + +{ + const pyodide = await freshPyodide(); + try { + pyodide.runPython(` +import sys +assert "weaviate_client_web" not in sys.modules +import weaviate # the ONLY weaviate-side import: must bootstrap the companion +assert "weaviate_client_web" in sys.modules, "hook did not import the companion" +import weaviate_client_web +import grpc +import httpx +assert weaviate_client_web.is_installed() +assert weaviate_client_web.is_fetch_transport_installed() +assert getattr(grpc, "__weaviate_client_web_shim__", False) is True +assert getattr( + httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False +) is True +`); + console.log("OK scenario: bare 'import weaviate' bootstraps the companion"); + } catch (err) { + console.error("FAIL scenario: bare 'import weaviate' bootstraps the companion"); + console.error(err); + process.exit(1); + } +} + +// --- the pytest suite -------------------------------------------------------------- + +const testsDir = resolve(here, "../../packages/web/tests"); +const pyodide = await freshPyodide(); +console.log( + `pyodide ${pyodide.version} / python ${pyodide.runPython("import sys; sys.version.split()[0]")}`, +); +const micropip = pyodide.pyimport("micropip"); +// A metadata-coherent pair: pytest-asyncio 0.25.3 declares pytest<9,>=8.2. Its +// run_until_complete-based execution stack-switches correctly under JSPI, unlike the +// asyncio.Runner-based pytest-asyncio 1.x, whose async tests fail here. (The Pyodide +// distribution bundles pytest 9 next to pytest-asyncio 0.25.3, contradicting that +// constraint — micropip tolerates it, but there is no reason to depend on its +// leniency, so both are pinned from PyPI.) +await micropip.install(["pytest==8.4.2", "pytest-asyncio==0.25.3"]); +pyodide.FS.mkdirTree("/units"); +pyodide.mountNodeFS("/units", testsDir); + +// pytest.main is synchronous; entering through callPromising() lets the async tests +// stack-switch (run_until_complete) instead of failing with "Cannot stack switch". +const runPytest = pyodide.runPython(` +import sys +sys.dont_write_bytecode = True # /units is the host checkout: no __pycache__ in it + +import pytest + +def _run(): + return int(pytest.main([ + "-v", + "-p", "no:cacheprovider", # no .pytest_cache in the host checkout either + "-o", "asyncio_mode=auto", + "-o", "asyncio_default_fixture_loop_scope=function", + "/units", + ])) + +_run +`); +let exitCode; +try { + exitCode = await runPytest.callPromising(); +} catch (err) { + console.error(err); + process.exit(1); +} +// Any nonzero pytest exit code fails the run — including 5, "no tests collected". +console.log(`pytest exit code: ${exitCode}`); +// The interpreters loaded above keep live handles on the Node event loop, so the +// process does not exit on its own. +process.exit(exitCode === 0 ? 0 : 1); diff --git a/packages/web/README.md b/packages/web/README.md index 582ab2796..3c4fd46c1 100644 --- a/packages/web/README.md +++ b/packages/web/README.md @@ -12,6 +12,26 @@ Requires Weaviate ≥ 1.38.3 (the first release to serve grpc-web natively) or a transcoder in front of an older server. Pyodide ≥ 0.27 recommended; verified on Pyodide 314.0.4 (CPython 3.14). +## Installation + +Install through the base client's `grpc-web` extra: + +```python +import micropip +await micropip.install("weaviate-client[grpc-web]") +``` + +The extra carries a `sys_platform == "emscripten"` marker, so the same requirement is a +no-op on CPython — one requirements list works everywhere. Installing the companion +directly (`micropip.install("weaviate-client-web")`) works too: it pins +`weaviate-client` to its own exact version, so a mismatched pair can never resolve. +Both forms require the first `weaviate-client` release that ships the extra and the +Emscripten marker; against older releases the resolver fails on `grpcio`. + +This package is defined for its environment: it imports `pyodide` at module scope and is +therefore only importable under Emscripten/Pyodide. On CPython the base client never +imports it (and the extra never installs it). + ## How it works Under Pyodide there is no `grpcio` Emscripten wheel, and `import weaviate` hard-imports @@ -81,7 +101,7 @@ await collection.query.near_text("hello", limit=3) Nothing selects grpc-web: `use_async_with_local()`, `use_async_with_weaviate_cloud()` and `use_async_with_custom()` all route gRPC onto the REST endpoint under `/v1/grpc-web` when -they run under Emscripten, and behave exactly as before everywhere else. +they run under Emscripten, and use native gRPC everywhere else. ```python client = weaviate.use_async_with_weaviate_cloud( @@ -107,7 +127,7 @@ Pass `headers={...}` / `auth_credentials=...` as usual for API keys, OIDC or WCD Importing the companion explicitly first also works and remains the explicit form: ```python -import weaviate_client_web # installs the grpc shim under Emscripten (no-op elsewhere) +import weaviate_client_web # installs the grpc shim (only importable under Emscripten) import weaviate ``` @@ -161,11 +181,22 @@ deployments that go through a grpc-web transcoder or a proxy must configure CORS - note that a CORS-blocked request is indistinguishable from a network failure in the browser (`TypeError: Failed to fetch`), and is retried as UNAVAILABLE. -## Testing on CPython +## Testing + +Because the package imports `pyodide` at module scope, its unit tests run inside +Pyodide. From the repository root: + +```sh +python -m build --wheel --outdir dist . +python -m build --wheel --outdir dist packages/web +npm install --prefix ci/pyodide-e2e +node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist # pytest unit suite, no Weaviate needed +node ci/pyodide-e2e/run.mjs dist # e2e suite, needs a running Weaviate (see ci/) +``` -`weaviate_client_web.install(force=True)` installs the shim on a normal CPython -interpreter (run it in a fresh process, before importing `weaviate`). Inject a sender -with `weaviate_client_web.set_sender(...)` (e.g. `make_httpx_sender()`) to exercise the -transport against an Envoy/vanguard transcoder without a browser. -`install_fetch_transport(force=True)` likewise patches httpx on CPython, given an -importable `pyodide.http` stand-in. +`units.mjs` runs pytest over `packages/web/tests/` inside Pyodide — async tests execute +on Pyodide's event loop via JSPI stack switching, hence the Node flag — plus a +fresh-interpreter bootstrap scenario; everything is driven through fake senders and a +fake `pyfetch`. `run.mjs` runs the e2e suite against a live Weaviate. On CPython the +`conftest.py` keeps pytest from collecting these modules (they cannot import there); +the base client's import-hook branches are covered by `test/test_wasm_compat.py`. diff --git a/packages/web/pyproject.toml b/packages/web/pyproject.toml index 95792c315..0f4872e18 100644 --- a/packages/web/pyproject.toml +++ b/packages/web/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=65", "wheel"] +requires = ["setuptools>=65", "setuptools_scm[toml] >6.2", "wheel"] build-backend = "setuptools.build_meta" [project] @@ -10,21 +10,19 @@ requires-python = ">=3.10" license = { text = "BSD-3-Clause" } authors = [{ name = "Weaviate", email = "hello@weaviate.io" }] keywords = ["weaviate", "grpc-web", "pyodide", "wasm", "emscripten"] -# Version is kept in lockstep with weaviate-client. TODO(lockstep): derive from the same -# git tag via setuptools_scm and assert the built versions match in CI before publishing. -version = "0.0.1.dev0" -# Deliberately depends on weaviate-client WITHOUT grpcio (grpcio is excluded under -# Emscripten by the `sys_platform != "emscripten"` marker in the base package's deps). -dependencies = [ - "weaviate-client", - # Pyodide's bundled httpx build omits anyio, but authlib imports it directly. - 'anyio ; sys_platform == "emscripten"', -] +# Version comes from the repository's git tags via setuptools_scm (root below), so every +# build carries the same version as weaviate-client; CI asserts the built wheels match. +# Dependencies are computed in setup.py: the weaviate-client requirement is pinned to +# that same version at build time, so mismatched pairs cannot resolve at install time. +dynamic = ["version", "dependencies"] [project.urls] Source = "https://github.com/weaviate/weaviate-python-client" Tracker = "https://github.com/weaviate/weaviate-python-client/issues" +[tool.setuptools_scm] +root = "../.." + [tool.setuptools.packages.find] where = ["src"] diff --git a/packages/web/setup.py b/packages/web/setup.py new file mode 100644 index 000000000..e99c3f653 --- /dev/null +++ b/packages/web/setup.py @@ -0,0 +1,25 @@ +"""Injects the lockstep ``weaviate-client==`` pin at build time. + +Both packages derive their version from the repository's git tags (setuptools_scm), so +the version is only known when the wheel is built and a static ``dependencies`` list +cannot express the pin. The pin makes mismatched pairs unresolvable at install time: +the two packages share private contracts (the ``HTTP `` error-string markers, +the exception constants ``_channel`` imports), so a companion must only ever install +next to the base client it was built with. + +Consequence for releasing: every tag must publish BOTH packages — a base-only release +would leave the extra pointing at a companion whose pin no longer resolves. +""" + +from setuptools import setup +from setuptools_scm import get_version + +version = get_version(root="../..", relative_to=__file__) + +setup( + install_requires=[ + f"weaviate-client=={version}", + # Pyodide's bundled httpx build omits anyio, but authlib imports it directly. + 'anyio ; sys_platform == "emscripten"', + ] +) diff --git a/packages/web/src/weaviate_client_web/__init__.py b/packages/web/src/weaviate_client_web/__init__.py index 2e9217991..d573b7af3 100644 --- a/packages/web/src/weaviate_client_web/__init__.py +++ b/packages/web/src/weaviate_client_web/__init__.py @@ -23,14 +23,22 @@ warns that it overrode them. An explicit ``import weaviate_client_web`` before ``import weaviate`` also works and -remains the explicit form. The shim is installed automatically only under Emscripten, so -importing this package on a normal CPython install never clobbers a real, working -``grpcio``. Async clients only — the synchronous client is not supported in the browser. +remains the explicit form. This package is defined for its environment: it imports +``pyodide`` at module scope, so it is only importable under Emscripten/Pyodide — on +CPython the base client never imports it, and the ``weaviate-client[grpc-web]`` extra +does not install it there (platform marker). Async clients only — the synchronous client +is not supported in the browser. """ import os import sys +from ._channel import GrpcWebChannel, set_sender +from ._httpx_fetch import ( + install_fetch_transport, + is_fetch_transport_installed, + uninstall_fetch_transport, +) from ._shim import StatusCode, install, is_installed __all__ = [ @@ -40,7 +48,6 @@ "uninstall_fetch_transport", "is_fetch_transport_installed", "set_sender", - "make_httpx_sender", "GrpcWebChannel", "StatusCode", ] @@ -54,21 +61,8 @@ def _bootstrap() -> None: os.environ.setdefault("PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", "python") install() # The REST path needs fetch too: httpx/httpcore open raw sockets, which do - # not exist under WASM. Imported lazily so CPython imports stay light. - from ._httpx_fetch import install_fetch_transport - + # not exist under WASM. install_fetch_transport() _bootstrap() - -# Imported after the bootstrap. These modules pull their grpc base classes directly from -# ``._shim`` (not via ``sys.modules['grpc']``), so importing them is safe regardless of -# whether the shim was installed. -from ._channel import GrpcWebChannel, set_sender # noqa: E402 -from ._httpx_fetch import ( # noqa: E402 - install_fetch_transport, - is_fetch_transport_installed, - uninstall_fetch_transport, -) -from ._sender import make_httpx_sender # noqa: E402 diff --git a/packages/web/src/weaviate_client_web/_channel.py b/packages/web/src/weaviate_client_web/_channel.py index 428c4f8f8..64e6d06be 100644 --- a/packages/web/src/weaviate_client_web/_channel.py +++ b/packages/web/src/weaviate_client_web/_channel.py @@ -23,12 +23,12 @@ from ._sender import Sender, pyfetch_sender from ._shim import AioChannel, AioRpcError, StatusCode, status_from_int -# Module-level default sender; overridable for tests / non-browser runtimes. +# Module-level default sender; overridable for tests. _default_sender: Sender = pyfetch_sender def set_sender(sender: Sender) -> None: - """Override the default async sender used by new channels (tests/integration).""" + """Override the default async sender used by new channels (tests).""" global _default_sender _default_sender = sender diff --git a/packages/web/src/weaviate_client_web/_httpx_fetch.py b/packages/web/src/weaviate_client_web/_httpx_fetch.py index b8f4be69c..b9d400465 100644 --- a/packages/web/src/weaviate_client_web/_httpx_fetch.py +++ b/packages/web/src/weaviate_client_web/_httpx_fetch.py @@ -21,6 +21,10 @@ - the browser's fetch follows redirects internally, so httpx never sees a 3xx; - multi-value response headers (e.g. Set-Cookie) are folded into one value; - responses are fully buffered (no streaming). + +Like the rest of this package, this module imports ``pyodide`` at module scope and is +therefore only importable under Emscripten/Pyodide (or with a ``pyodide`` stand-in +pre-installed in ``sys.modules``). """ import math @@ -28,13 +32,14 @@ from typing import Callable, Dict, Optional import httpx +from pyodide.http import pyfetch # type: ignore[import-not-found] _installed = False _original_handle_async_request: Optional[Callable] = None # Hop-by-hop / connection-managed headers that the browser's fetch controls itself. -# Browsers silently drop forbidden headers, but Node's undici (used by the CPython/Node -# test path) rejects some of them outright, so strip them before handing off. +# Browsers silently drop forbidden headers, but Node's undici (behind pyfetch in the +# Node-based test harness) rejects some of them outright, so strip them before handing off. _FETCH_MANAGED_HEADERS = { "host", "connection", @@ -122,8 +127,6 @@ def _validate_header(name: str, value: str) -> None: async def _fetch_handle_async_request( self: httpx.AsyncHTTPTransport, request: httpx.Request ) -> httpx.Response: - from pyodide.http import pyfetch # type: ignore[import-not-found] - headers: Dict[str, str] = {} for k, v in request.headers.items(): if k.lower() in _FETCH_MANAGED_HEADERS: @@ -177,21 +180,17 @@ async def _fetch_handle_async_request( _fetch_handle_async_request.__weaviate_fetch_shim__ = True # type: ignore[attr-defined] -def install_fetch_transport(force: bool = False) -> None: +def install_fetch_transport() -> None: """Patch ``httpx.AsyncHTTPTransport`` to send requests through ``fetch``. - Installs only under Emscripten unless ``force=True`` (CPython testing, where a - ``pyodide`` stub must be importable). Idempotent. + Installs only under Emscripten (elsewhere httpx's own socket transports work and + must be left in place). Idempotent. """ global _installed, _original_handle_async_request if _installed: return - if not force and sys.platform != "emscripten": + if sys.platform != "emscripten": return - # Fail fast: the handler imports pyfetch per request, so a missing pyodide module - # would otherwise surface as a confusing ModuleNotFoundError on the first request. - from pyodide.http import pyfetch # type: ignore[import-not-found] # noqa: F401 - _original_handle_async_request = httpx.AsyncHTTPTransport.handle_async_request httpx.AsyncHTTPTransport.handle_async_request = _fetch_handle_async_request # type: ignore[method-assign] _installed = True diff --git a/packages/web/src/weaviate_client_web/_sender.py b/packages/web/src/weaviate_client_web/_sender.py index d41f9f6c0..fd2a3488a 100644 --- a/packages/web/src/weaviate_client_web/_sender.py +++ b/packages/web/src/weaviate_client_web/_sender.py @@ -2,11 +2,17 @@ A *sender* is ``async def sender(url, headers, body, timeout) -> (status, headers, body)``. The default uses ``pyodide.http.pyfetch`` (browser fetch); a sender can be injected for -testing or for non-browser runtimes via :func:`weaviate_client_web.set_sender`. +testing via :func:`weaviate_client_web.set_sender`. + +Like the rest of this package, this module imports ``pyodide`` at module scope and is +therefore only importable under Emscripten/Pyodide (or with a ``pyodide`` stand-in +pre-installed in ``sys.modules``). """ from typing import Awaitable, Callable, Dict, Optional, Tuple +from pyodide.http import pyfetch # type: ignore[import-not-found] + Sender = Callable[ [str, Dict[str, str], bytes, Optional[float]], Awaitable[Tuple[int, Dict[str, str], bytes]], @@ -18,12 +24,9 @@ async def pyfetch_sender( ) -> Tuple[int, Dict[str, str], bytes]: """Default browser sender. - Imports ``pyodide.http`` lazily so this module stays importable on CPython (where - ``pyodide`` does not exist). ``pyfetch`` has no timeout parameter of its own; the - call deadline is enforced by ``GrpcWebChannel._unary`` via ``asyncio.wait_for``. + ``pyfetch`` has no timeout parameter of its own; the call deadline is enforced by + ``GrpcWebChannel._unary`` via ``asyncio.wait_for``. """ - from pyodide.http import pyfetch # type: ignore[import-not-found] - response = await pyfetch(url, method="POST", headers=headers, body=body) data = await response.bytes() try: @@ -31,30 +34,3 @@ async def pyfetch_sender( except Exception: # pragma: no cover - header shape varies across Pyodide versions resp_headers = {} return int(response.status), resp_headers, data - - -def make_httpx_sender(client: Optional[object] = None) -> Sender: - """Build a sender backed by ``httpx.AsyncClient`` for CPython tests/integration. - - Targets a grpc-web transcoder (Envoy / connectrpc vanguard). - """ - import httpx - - async def _send( - url: str, headers: Dict[str, str], body: bytes, timeout: Optional[float] - ) -> Tuple[int, Dict[str, str], bytes]: - owns_client = client is None - active = client or httpx.AsyncClient() - assert isinstance(active, httpx.AsyncClient) - try: - response = await active.post(url, headers=headers, content=body, timeout=timeout) - return ( - response.status_code, - {k.lower(): v for k, v in response.headers.items()}, - response.content, - ) - finally: - if owns_client: - await active.aclose() - - return _send diff --git a/packages/web/src/weaviate_client_web/_shim.py b/packages/web/src/weaviate_client_web/_shim.py index b04b8837a..0624f5259 100644 --- a/packages/web/src/weaviate_client_web/_shim.py +++ b/packages/web/src/weaviate_client_web/_shim.py @@ -217,14 +217,14 @@ def is_installed() -> bool: return getattr(sys.modules.get("grpc"), _SHIM_MARKER, False) is True -def install(force: bool = False) -> bool: +def install() -> bool: """Install the shim into ``sys.modules`` as ``grpc`` and submodules. - On normal platforms this is a no-op unless ``force=True`` — we must never clobber a - real, working ``grpcio``. Under Emscripten the bootstrap calls this automatically. - Returns ``True`` if the shim is in place afterwards. + On normal platforms this is a no-op — a real, working ``grpcio`` must be left in + place. Under Emscripten the bootstrap calls this automatically. Returns ``True`` + if the shim is in place afterwards. """ - if not force and sys.platform != "emscripten": + if sys.platform != "emscripten": return False if is_installed(): return True diff --git a/packages/web/tests/conftest.py b/packages/web/tests/conftest.py index fe4afb09d..a6491b8dc 100644 --- a/packages/web/tests/conftest.py +++ b/packages/web/tests/conftest.py @@ -1,7 +1,14 @@ -import pathlib +"""Restrict this suite to Pyodide. + +The package under test imports ``pyodide`` at module scope, so the test modules here +are only importable under Emscripten/Pyodide. There, pytest runs them via +``ci/pyodide-e2e/units.mjs`` (async tests execute on Pyodide's event loop through JSPI +stack switching). On CPython, keep pytest from collecting them: + + node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist +""" + import sys -# Make the package importable without an editable install. -_SRC = pathlib.Path(__file__).resolve().parents[1] / "src" -if str(_SRC) not in sys.path: - sys.path.insert(0, str(_SRC)) +if sys.platform != "emscripten": + collect_ignore_glob = ["*.py"] diff --git a/packages/web/tests/test_framing.py b/packages/web/tests/test_framing.py index fd18a35a3..8054dbf94 100644 --- a/packages/web/tests/test_framing.py +++ b/packages/web/tests/test_framing.py @@ -1,3 +1,5 @@ +"""grpc-web framing tests. Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs``.""" + import struct import pytest @@ -88,9 +90,9 @@ def test_parse_trailers_accepts_lf_only_lines(): def test_parse_trailers_keeps_status_when_a_key_is_not_ascii(): # one odd key from a proxy must not throw away the whole block - parsed = parse_trailers("x-caf\u00e9:1\r\ngrpc-status:0\r\n".encode("utf-8")) + parsed = parse_trailers("x-café:1\r\ngrpc-status:0\r\n".encode("utf-8")) assert parsed["grpc-status"] == "0" - assert parsed["x-caf\u00e9"] == "1" + assert parsed["x-café"] == "1" def test_truncated_frame_raises(): diff --git a/packages/web/tests/test_httpx_fetch.py b/packages/web/tests/test_httpx_fetch.py index c34f8e35d..c1833d74c 100644 --- a/packages/web/tests/test_httpx_fetch.py +++ b/packages/web/tests/test_httpx_fetch.py @@ -1,30 +1,24 @@ -"""Tests for the fetch-based httpx transport (_httpx_fetch.py). +"""Tests for the fetch-based httpx transport (``_httpx_fetch.py``). -In-process tests call ``_fetch_handle_async_request`` directly with a fake -``pyodide.http`` module injected into ``sys.modules`` — no global monkeypatch of -``httpx.AsyncHTTPTransport`` is needed, so the real httpx in the dev environment is left -untouched. Install semantics (which DO patch the class globally) run in fresh -subprocesses, mirroring test_shim_install.py. +Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs``. ``pyfetch`` is bound at +import time in ``_httpx_fetch``, so fakes patch that module attribute; ``from js +import AbortSignal`` is resolved per request, so a fake js module in ``sys.modules`` +intercepts it even under real Pyodide. + +The install-semantics tests at the bottom run against this interpreter's real +installation: importing ``weaviate_client_web`` bootstrapped the transport globally. """ -import asyncio -import pathlib -import subprocess import sys -import textwrap import types from typing import Any, Dict, List, Optional import httpx import pytest -from weaviate_client_web._httpx_fetch import ( - _MAX_ABORT_SIGNAL_MS, - _abort_signal_ms, - _fetch_handle_async_request, -) - -_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") +import weaviate_client_web +import weaviate_client_web._httpx_fetch as _httpx_fetch +from weaviate_client_web._httpx_fetch import _MAX_ABORT_SIGNAL_MS, _abort_signal_ms class FakeFetchResponse: @@ -53,47 +47,60 @@ async def __call__(self, url: str, **kwargs: Any) -> FakeFetchResponse: @pytest.fixture def fake_pyfetch(monkeypatch) -> FakePyfetch: fetch = FakePyfetch() - pyodide_mod = types.ModuleType("pyodide") - http_mod = types.ModuleType("pyodide.http") - http_mod.pyfetch = fetch # type: ignore[attr-defined] - pyodide_mod.http = http_mod # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "pyodide", pyodide_mod) - monkeypatch.setitem(sys.modules, "pyodide.http", http_mod) + monkeypatch.setattr(_httpx_fetch, "pyfetch", fetch) return fetch -async def _handle_async(request: httpx.Request) -> httpx.Response: +class _AbortSignalRecorder: + def __init__(self): + self.timeouts: List[int] = [] + + def timeout(self, ms: int): + self.timeouts.append(ms) + return f"signal-{ms}" + + +@pytest.fixture +def fake_abort_signal(monkeypatch) -> _AbortSignalRecorder: + recorder = _AbortSignalRecorder() + js_mod = types.ModuleType("js") + js_mod.AbortSignal = recorder # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "js", js_mod) + return recorder + + +@pytest.fixture +def missing_js(monkeypatch): + # sys.modules["js"] = None makes ``import js`` raise ImportError — the closest + # in-process stand-in for an environment without the js bridge + monkeypatch.setitem(sys.modules, "js", None) + + +async def _handle(request: httpx.Request) -> httpx.Response: # self is unused by the handler implementation; a bare transport instance suffices transport = httpx.AsyncHTTPTransport.__new__(httpx.AsyncHTTPTransport) - response = await _fetch_handle_async_request(transport, request) + response = await _httpx_fetch._fetch_handle_async_request(transport, request) await response.aread() # httpx.AsyncClient reads non-streamed responses the same way return response -def _handle(request: httpx.Request) -> httpx.Response: - return asyncio.run(_handle_async(request)) - - class _FetchTransport(httpx.AsyncBaseTransport): - """Route an ``httpx.AsyncClient`` through the handler without patching httpx globally.""" + """Route an ``httpx.AsyncClient`` through the handler without touching global state.""" async def handle_async_request(self, request: httpx.Request) -> httpx.Response: - return await _fetch_handle_async_request(self, request) # type: ignore[arg-type] - + return await _httpx_fetch._fetch_handle_async_request(self, request) # type: ignore[arg-type] -def _via_client(method: str, url: str, **kwargs: Any) -> httpx.Response: - async def main() -> httpx.Response: - async with httpx.AsyncClient(transport=_FetchTransport()) as client: - return await client.request(method, url, **kwargs) - return asyncio.run(main()) +async def _via_client(method: str, url: str, **kwargs: Any) -> httpx.Response: + async with httpx.AsyncClient(transport=_FetchTransport()) as client: + return await client.request(method, url, **kwargs) -def test_basic_get_round_trip(fake_pyfetch): +async def test_basic_get_round_trip(fake_pyfetch): fake_pyfetch.response = FakeFetchResponse( status=200, headers={"content-type": "application/json"}, body=b'{"version": "1.30.0"}' ) - response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) assert response.status_code == 200 assert response.json() == {"version": "1.30.0"} @@ -103,52 +110,52 @@ def test_basic_get_round_trip(fake_pyfetch): assert call["method"] == "GET" -def test_response_has_request_attached_for_raise_for_status(fake_pyfetch): +async def test_response_has_request_attached_for_raise_for_status(fake_pyfetch): fake_pyfetch.response = FakeFetchResponse(status=404, body=b"") - response = _handle(httpx.Request("GET", "http://h:8080/v1/schema/Nope")) + response = await _handle(httpx.Request("GET", "http://h:8080/v1/schema/Nope")) with pytest.raises(httpx.HTTPStatusError): response.raise_for_status() @pytest.mark.parametrize("status", [204, 404]) -def test_head_response_without_body_yields_empty_content(fake_pyfetch, status): +async def test_head_response_without_body_yields_empty_content(fake_pyfetch, status): # data.exists() / tenants.exists() are HEAD requests answered 204/404 with a null # body; the transport must hand httpx an empty response, not fail on the missing body fake_pyfetch.response = FakeFetchResponse(status=status, body=None) - response = _handle(httpx.Request("HEAD", "http://h:8080/v1/objects/A/uuid")) + response = await _handle(httpx.Request("HEAD", "http://h:8080/v1/objects/A/uuid")) assert response.status_code == status assert response.content == b"" assert "body" not in fake_pyfetch.calls[0] -def test_delete_204_without_body_yields_empty_content(fake_pyfetch): +async def test_delete_204_without_body_yields_empty_content(fake_pyfetch): # data.delete_by_id() / reference_delete() are answered 204 with a null body fake_pyfetch.response = FakeFetchResponse(status=204, body=None) - response = _handle(httpx.Request("DELETE", "http://h:8080/v1/objects/A/uuid")) + response = await _handle(httpx.Request("DELETE", "http://h:8080/v1/objects/A/uuid")) assert response.status_code == 204 assert response.content == b"" -def test_body_less_response_through_async_client(fake_pyfetch): +async def test_body_less_response_through_async_client(fake_pyfetch): # the full httpx.AsyncClient path (stream wrapping + read) on a body-less response fake_pyfetch.response = FakeFetchResponse(status=204, body=None) - response = _via_client("HEAD", "http://h:8080/v1/objects/A/uuid") + response = await _via_client("HEAD", "http://h:8080/v1/objects/A/uuid") assert response.status_code == 204 assert response.content == b"" -def test_response_through_async_client_exposes_elapsed_and_content(fake_pyfetch): +async def test_response_through_async_client_exposes_elapsed_and_content(fake_pyfetch): # the batch-references path reads ``res.elapsed``, which httpx only sets after it # has read/closed a stream-backed response; a pre-loaded body never gets one payload = b'[{"result": {"status": "SUCCESS"}}]' fake_pyfetch.response = FakeFetchResponse(status=200, body=payload) - response = _via_client("POST", "http://h:8080/v1/batch/references", content=b"[]") + response = await _via_client("POST", "http://h:8080/v1/batch/references", content=b"[]") assert response.content == payload assert response.json() == [{"result": {"status": "SUCCESS"}}] assert response.elapsed.total_seconds() >= 0 -def test_fetch_managed_request_headers_stripped(fake_pyfetch): +async def test_fetch_managed_request_headers_stripped(fake_pyfetch): request = httpx.Request( "POST", "http://h:8080/v1/objects", @@ -162,7 +169,7 @@ def test_fetch_managed_request_headers_stripped(fake_pyfetch): }, content=b"{}", ) - _handle(request) + await _handle(request) sent = fake_pyfetch.calls[0]["headers"] assert sent["authorization"] == "Bearer k" assert sent["content-type"] == "application/json" @@ -170,29 +177,31 @@ def test_fetch_managed_request_headers_stripped(fake_pyfetch): assert managed not in sent -def test_get_without_body_omits_body_kwarg(fake_pyfetch): +async def test_get_without_body_omits_body_kwarg(fake_pyfetch): # fetch rejects GET/HEAD requests that carry a body, so the kwarg must be absent - _handle(httpx.Request("GET", "http://h:8080/v1/.well-known/ready")) + await _handle(httpx.Request("GET", "http://h:8080/v1/.well-known/ready")) assert "body" not in fake_pyfetch.calls[0] -def test_post_body_passed(fake_pyfetch): - _handle(httpx.Request("POST", "http://h:8080/v1/graphql", content=b'{"query": "x"}')) +async def test_post_body_passed(fake_pyfetch): + await _handle(httpx.Request("POST", "http://h:8080/v1/graphql", content=b'{"query": "x"}')) assert fake_pyfetch.calls[0]["body"] == b'{"query": "x"}' -def test_delete_with_body_passed(fake_pyfetch): +async def test_delete_with_body_passed(fake_pyfetch): # the REST batch-delete path sends DELETE with a JSON body - _handle(httpx.Request("DELETE", "http://h:8080/v1/batch/objects", content=b'{"match": {}}')) + await _handle( + httpx.Request("DELETE", "http://h:8080/v1/batch/objects", content=b'{"match": {}}') + ) assert fake_pyfetch.calls[0]["body"] == b'{"match": {}}' -def test_query_string_preserved_in_url(fake_pyfetch): - _handle(httpx.Request("GET", "http://h:8080/v1/objects?class=A&limit=10&after=a%20b")) +async def test_query_string_preserved_in_url(fake_pyfetch): + await _handle(httpx.Request("GET", "http://h:8080/v1/objects?class=A&limit=10&after=a%20b")) assert fake_pyfetch.calls[0]["url"] == "http://h:8080/v1/objects?class=A&limit=10&after=a%20b" -def test_content_encoding_stripped_from_response(fake_pyfetch): +async def test_content_encoding_stripped_from_response(fake_pyfetch): # fetch hands back ALREADY-decompressed bytes; if the original content-encoding # header were passed through, httpx.Response would gunzip a second time and raise # DecodingError. content-length is stale for the same reason. @@ -201,93 +210,74 @@ def test_content_encoding_stripped_from_response(fake_pyfetch): headers={"content-encoding": "gzip", "content-length": "23", "x-other": "kept"}, body=b'{"version": "1.30.0"}', ) - response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) assert response.json() == {"version": "1.30.0"} assert "content-encoding" not in response.headers assert response.headers["x-other"] == "kept" -def test_unreadable_response_headers_tolerated(fake_pyfetch): +async def test_unreadable_response_headers_tolerated(fake_pyfetch): class BadHeaders: def keys(self): raise TypeError("header shape varies across Pyodide versions") fake_pyfetch.response = FakeFetchResponse(status=200, body=b"ok") fake_pyfetch.response.headers = BadHeaders() - response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) assert response.status_code == 200 assert response.content == b"ok" -class _AbortSignalRecorder: - def __init__(self): - self.timeouts: List[int] = [] - - def timeout(self, ms: int): - self.timeouts.append(ms) - return f"signal-{ms}" - - -@pytest.fixture -def fake_abort_signal(monkeypatch) -> _AbortSignalRecorder: - recorder = _AbortSignalRecorder() - js_mod = types.ModuleType("js") - js_mod.AbortSignal = recorder # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "js", js_mod) - return recorder - - def _request_with_timeout(timeouts: Dict[str, Optional[float]]) -> httpx.Request: request = httpx.Request("GET", "http://h:8080/v1/meta") request.extensions["timeout"] = timeouts return request -def test_read_timeout_maps_to_abort_signal_ms(fake_pyfetch, fake_abort_signal): +async def test_read_timeout_maps_to_abort_signal_ms(fake_pyfetch, fake_abort_signal): # mirrors what weaviate's AsyncClient puts in extensions: connect/read/write/pool - _handle(_request_with_timeout({"connect": 2.0, "read": 30.0, "write": 5.0, "pool": 9.0})) + await _handle(_request_with_timeout({"connect": 2.0, "read": 30.0, "write": 5.0, "pool": 9.0})) assert fake_abort_signal.timeouts == [30000] assert fake_pyfetch.calls[0]["signal"] == "signal-30000" -def test_read_none_means_no_deadline_even_with_pool_and_connect_set( +async def test_read_none_means_no_deadline_even_with_pool_and_connect_set( fake_pyfetch, fake_abort_signal ): # what the base client hands over for a non-finite request timeout: read=None with the # session pool timeout still set; falling back to pool/connect would abort a long # insert after 5 s - _handle(_request_with_timeout({"connect": None, "read": None, "write": None, "pool": 5})) - _handle(_request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0})) + await _handle(_request_with_timeout({"connect": None, "read": None, "write": None, "pool": 5})) + await _handle(_request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0})) assert fake_abort_signal.timeouts == [] assert all("signal" not in c for c in fake_pyfetch.calls) -def test_read_timeout_alone_sets_the_deadline(fake_pyfetch, fake_abort_signal): - _handle(_request_with_timeout({"connect": None, "read": 7, "write": None, "pool": 5})) +async def test_read_timeout_alone_sets_the_deadline(fake_pyfetch, fake_abort_signal): + await _handle(_request_with_timeout({"connect": None, "read": 7, "write": None, "pool": 5})) assert fake_abort_signal.timeouts == [7000] assert fake_pyfetch.calls[0]["signal"] == "signal-7000" -def test_no_timeout_extension_sends_no_signal(fake_pyfetch, fake_abort_signal): - _handle(httpx.Request("GET", "http://h:8080/v1/meta")) +async def test_no_timeout_extension_sends_no_signal(fake_pyfetch, fake_abort_signal): + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) assert fake_abort_signal.timeouts == [] assert "signal" not in fake_pyfetch.calls[0] -def test_missing_js_module_degrades_to_no_signal(fake_pyfetch): - # off-browser (no js module) the AbortSignal import fails; the request must still go out - assert "js" not in sys.modules - response = _handle( +async def test_missing_js_module_degrades_to_no_signal(fake_pyfetch, missing_js): + # without the js bridge the AbortSignal import fails; the request must still go out + response = await _handle( _request_with_timeout({"connect": 2.0, "read": 30.0, "write": None, "pool": None}) ) assert response.status_code == 200 assert "signal" not in fake_pyfetch.calls[0] -def test_zero_timeout_means_no_deadline(fake_pyfetch, fake_abort_signal): +async def test_zero_timeout_means_no_deadline(fake_pyfetch, fake_abort_signal): # an explicit read=0 must not fall through to the 5s connect timeout, nor become an # immediate AbortSignal.timeout(0) - _handle(_request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None})) + await _handle(_request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None})) assert fake_abort_signal.timeouts == [] assert "signal" not in fake_pyfetch.calls[0] @@ -311,18 +301,20 @@ def test_abort_signal_ms_bounds(timeout, expected_ms): assert _abort_signal_ms(timeout) == expected_ms -def test_infinite_timeout_sends_no_signal(fake_pyfetch, fake_abort_signal): +async def test_infinite_timeout_sends_no_signal(fake_pyfetch, fake_abort_signal): # an inf read deadline reaching the transport: no signal, not an OverflowError - _handle( + await _handle( _request_with_timeout({"connect": None, "read": float("inf"), "write": None, "pool": 5}) ) assert fake_abort_signal.timeouts == [] assert "signal" not in fake_pyfetch.calls[0] -def test_huge_timeout_is_capped_to_int32_ms(fake_pyfetch, fake_abort_signal): +async def test_huge_timeout_is_capped_to_int32_ms(fake_pyfetch, fake_abort_signal): # setTimeout delays above 2^31-1 ms overflow and fire at once, aborting the request - _handle(_request_with_timeout({"connect": None, "read": 1e10, "write": None, "pool": None})) + await _handle( + _request_with_timeout({"connect": None, "read": 1e10, "write": None, "pool": None}) + ) assert fake_abort_signal.timeouts == [_MAX_ABORT_SIGNAL_MS] assert fake_pyfetch.calls[0]["signal"] == f"signal-{_MAX_ABORT_SIGNAL_MS}" @@ -336,270 +328,123 @@ async def __call__(self, url: str, **kwargs: Any): def _install_raising_pyfetch(monkeypatch, exc: BaseException) -> None: - pyodide_mod = types.ModuleType("pyodide") - http_mod = types.ModuleType("pyodide.http") - http_mod.pyfetch = RaisingPyfetch(exc) # type: ignore[attr-defined] - pyodide_mod.http = http_mod # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "pyodide", pyodide_mod) - monkeypatch.setitem(sys.modules, "pyodide.http", http_mod) + monkeypatch.setattr(_httpx_fetch, "pyfetch", RaisingPyfetch(exc)) -def test_fetch_failure_maps_to_httpx_connect_error(monkeypatch): +async def test_fetch_failure_maps_to_httpx_connect_error(monkeypatch): # pyodide surfaces JS fetch rejections as OSError; the base client can only classify # httpx exceptions (WeaviateConnectionError etc.), so the shim must translate _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) with pytest.raises(httpx.ConnectError, match="Failed to fetch") as excinfo: - _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) assert isinstance(excinfo.value.__cause__, OSError) -def test_fetch_abort_with_deadline_maps_to_read_timeout(monkeypatch, fake_abort_signal): +async def test_fetch_abort_with_deadline_maps_to_read_timeout(monkeypatch, fake_abort_signal): # AbortSignal.timeout firing surfaces as an OSError subclass mentioning the abort; # with a deadline set this must classify as a timeout, not a connection error _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) with pytest.raises(httpx.ReadTimeout, match="signal timed out"): - _handle(_request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None})) + await _handle( + _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) + ) -def test_fetch_failure_with_deadline_but_no_timeout_message_stays_connect_error( +async def test_fetch_failure_with_deadline_but_no_timeout_message_stays_connect_error( monkeypatch, fake_abort_signal ): # nearly every weaviate request sets a read deadline; a plain network failure on # such a request must remain a connection error, not become a timeout _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) with pytest.raises(httpx.ConnectError, match="Failed to fetch"): - _handle(_request_with_timeout({"connect": None, "read": 30.0, "write": None, "pool": None})) + await _handle( + _request_with_timeout({"connect": None, "read": 30.0, "write": None, "pool": None}) + ) -def test_fetch_abort_without_deadline_stays_connect_error(monkeypatch): - # the same message without a deadline set (no js module -> no signal) is not OUR +async def test_fetch_abort_without_deadline_stays_connect_error(monkeypatch, missing_js): + # the same message without a deadline set (no js bridge -> no signal) is not OUR # timeout, so it must stay a connection error _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) - assert "js" not in sys.modules with pytest.raises(httpx.ConnectError): - _handle(_request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None})) + await _handle( + _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) + ) -def test_empty_oserror_str_keeps_repr_detail(monkeypatch): +async def test_empty_oserror_str_keeps_repr_detail(monkeypatch): _install_raising_pyfetch(monkeypatch, OSError()) with pytest.raises(httpx.ConnectError) as excinfo: - _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) assert "OSError" in str(excinfo.value) -def test_crlf_in_header_value_rejected(fake_pyfetch): +async def test_crlf_in_header_value_rejected(fake_pyfetch): # httpx.Request accepts CR/LF in header values and relies on h11 to reject them at # send time; this transport bypasses h11 and must keep that defence request = httpx.Request( "GET", "http://h:8080/v1/meta", headers={"x-key": "val\r\nx-injected: evil"} ) with pytest.raises(httpx.LocalProtocolError): - _handle(request) + await _handle(request) assert fake_pyfetch.calls == [] # --------------------------------------------------------------------------- -# Install semantics: these patch httpx.AsyncHTTPTransport globally, so each -# scenario runs in a fresh subprocess (same pattern as test_shim_install.py). +# install semantics, against this interpreter's real installation # --------------------------------------------------------------------------- -_FAKE_PYODIDE_PRELUDE = """ -import sys, types - -class _FakeResponse: - status = 200 - headers = {"content-type": "application/json"} - async def bytes(self): - return b'{"ok": true}' - -CALLS = [] -async def pyfetch(url, **kwargs): - CALLS.append((url, kwargs)) - return _FakeResponse() - -_pyodide = types.ModuleType("pyodide") -_http = types.ModuleType("pyodide.http") -_http.pyfetch = pyfetch -_pyodide.http = _http -sys.modules["pyodide"] = _pyodide -sys.modules["pyodide.http"] = _http -""" - - -def _run(body: str, prelude: str = "") -> subprocess.CompletedProcess: - script = f"import sys\nsys.path.insert(0, {_SRC!r})\n" + prelude + textwrap.dedent(body) - return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) +def test_bootstrap_installed_fetch_transport(): + # This interpreter imported weaviate_client_web at the top of this file, so the real + # bootstrap ran — even though Pyodide's bundled httpx carries its own jsfetch + # transport, the package's transport must be the active one. + assert weaviate_client_web.is_fetch_transport_installed() + assert ( + getattr(httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False) + is True + ) -def test_force_install_routes_async_client_through_pyfetch(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import asyncio, httpx - from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed - install_fetch_transport(force=True) - assert is_fetch_transport_installed() - - async def main(): - async with httpx.AsyncClient() as client: - return await client.get("http://h:8080/v1/meta") - - resp = asyncio.run(main()) - assert resp.status_code == 200, resp.status_code - assert resp.json() == {"ok": True} - assert CALLS and CALLS[0][0] == "http://h:8080/v1/meta" - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_install_without_force_is_noop_off_emscripten(): - result = _run( - """ - import sys - assert sys.platform != "emscripten" - import httpx - before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed - install_fetch_transport() - assert not is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is before - print("OK") - """ - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_force_install_is_idempotent(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import httpx - from weaviate_client_web import install_fetch_transport - install_fetch_transport(force=True) - patched = httpx.AsyncHTTPTransport.handle_async_request - install_fetch_transport(force=True) - assert httpx.AsyncHTTPTransport.handle_async_request is patched - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout +def test_install_fetch_transport_is_idempotent(): + patched_method = httpx.AsyncHTTPTransport.handle_async_request + weaviate_client_web.install_fetch_transport() + assert httpx.AsyncHTTPTransport.handle_async_request is patched_method def test_sync_transport_left_untouched(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import httpx - sync_before = httpx.HTTPTransport.handle_request - from weaviate_client_web import install_fetch_transport - install_fetch_transport(force=True) - assert httpx.HTTPTransport.handle_request is sync_before - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout + assert not getattr(httpx.HTTPTransport.handle_request, "__weaviate_fetch_shim__", False) def test_uninstall_restores_original_transport(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import httpx - before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_client_web import ( - install_fetch_transport, - is_fetch_transport_installed, - uninstall_fetch_transport, - ) - uninstall_fetch_transport() # no-op when not installed - install_fetch_transport(force=True) - assert is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is not before - uninstall_fetch_transport() - assert not is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is before - print("OK") - """, + # the module keeps the pre-patch method while installed, so the restore can be + # checked by identity even though the install happened at import time + original = _httpx_fetch._original_handle_async_request + assert original is not None + patched_method = httpx.AsyncHTTPTransport.handle_async_request + try: + weaviate_client_web.uninstall_fetch_transport() + assert not weaviate_client_web.is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is original + assert httpx.AsyncHTTPTransport.handle_async_request is not patched_method + weaviate_client_web.uninstall_fetch_transport() # no-op when not installed + assert httpx.AsyncHTTPTransport.handle_async_request is original + finally: + weaviate_client_web.install_fetch_transport() + assert weaviate_client_web.is_fetch_transport_installed() + assert ( + getattr(httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False) + is True ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_patched_method_carries_sentinel(): - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import httpx - from weaviate_client_web import install_fetch_transport - assert not getattr( - httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False - ) - install_fetch_transport(force=True) - assert getattr( - httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False - ) is True - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_force_install_without_pyodide_fails_fast(): - # without a pyodide module the install must raise immediately, not let every later - # request die with a lazy ModuleNotFoundError - result = _run( - """ - import httpx - before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed - try: - install_fetch_transport(force=True) - except ModuleNotFoundError: - assert not is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is before - print("OK") - else: - raise AssertionError("expected install to fail fast without pyodide") - """ - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_emscripten_installs_even_when_platform_httpx_has_jsfetch(): - # Pyodide's bundled httpx ships a jsfetch transport that crashes on body-less - # responses (HEAD / 204); the shim must take over regardless of the httpx build - result = _run( - prelude=_FAKE_PYODIDE_PRELUDE, - body=""" - import importlib.machinery, sys, types - - sys.platform = "emscripten" - fake = types.ModuleType("httpx._transports.jsfetch") - fake.__spec__ = importlib.machinery.ModuleSpec( - "httpx._transports.jsfetch", loader=None - ) - sys.modules["httpx._transports.jsfetch"] = fake - - import httpx - before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed - install_fetch_transport() # no force: the platform alone must trigger it - assert is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is not before - assert getattr( - httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False - ) is True - print("OK") - """, + + +async def test_installed_transport_routes_async_client_through_pyfetch(fake_pyfetch): + # the globally installed transport (no custom transport argument) must reach pyfetch + fake_pyfetch.response = FakeFetchResponse( + status=200, headers={"content-type": "application/json"}, body=b'{"ok": true}' ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout + async with httpx.AsyncClient() as client: + response = await client.get("http://h:8080/v1/meta") + assert response.status_code == 200 + assert response.json() == {"ok": True} + assert fake_pyfetch.calls and fake_pyfetch.calls[0]["url"] == "http://h:8080/v1/meta" diff --git a/packages/web/tests/test_shim_install.py b/packages/web/tests/test_shim_install.py index cdcbed56d..0efab5973 100644 --- a/packages/web/tests/test_shim_install.py +++ b/packages/web/tests/test_shim_install.py @@ -1,123 +1,84 @@ -"""Shim/import tests. +"""grpc shim tests, against this interpreter's real installation. -Installing the shim replaces ``sys.modules['grpc']`` process-wide, so each scenario runs -in a fresh subprocess to avoid clobbering the real ``grpc`` used by the rest of the suite. +Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs``: importing +``weaviate_client_web`` bootstrapped the shim, so ``sys.modules['grpc']`` here IS the +shim. Bootstrap scenarios that need a clean import state live in ``units.mjs`` (one +fresh interpreter) and, for the base client's hook logic, in +``test/test_wasm_compat.py`` on CPython. """ -import pathlib -import subprocess -import sys -import textwrap +import struct -_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") +import pytest +import weaviate_client_web +from weaviate_client_web import GrpcWebChannel, set_sender +from weaviate_client_web._sender import pyfetch_sender +from weaviate_client_web._shim import FAKE_GRPC_VERSION -def _run(body: str) -> subprocess.CompletedProcess: - script = f"import sys\nsys.path.insert(0, {_SRC!r})\n" + textwrap.dedent(body) - return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + +def _frame(payload: bytes, flag: int = 0x00) -> bytes: + return struct.pack(">BI", flag, len(payload)) + payload def test_import_weaviate_under_shim(): - result = _run( - """ - import weaviate_client_web - assert weaviate_client_web.install(force=True) is True - assert weaviate_client_web.is_installed() - - import grpc - assert getattr(grpc, "__weaviate_client_web_shim__", False) is True - assert grpc.__version__ == "1.72.1" - assert grpc._utilities.first_version_is_lower("1.0.0", "2.0.0") is False - from grpc.aio._typing import ChannelArgumentType # noqa: F401 - - import weaviate # must not raise even though grpcio is shimmed - from weaviate.proto.v1 import weaviate_pb2_grpc - from weaviate_client_web import GrpcWebChannel + import grpc - ch = GrpcWebChannel("localhost:50051", secure=False) - stub = weaviate_pb2_grpc.WeaviateStub(ch) - assert stub.Search is not None - assert stub.BatchObjects is not None - assert stub.BatchDelete is not None - assert isinstance(ch, grpc.aio.Channel) - print("OK") - """ - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout + assert weaviate_client_web.is_installed() + assert weaviate_client_web.install() is True # idempotent, reports the shim in place + assert getattr(grpc, "__weaviate_client_web_shim__", False) is True + assert grpc.__version__ == "1.72.1" + assert grpc._utilities.first_version_is_lower("1.0.0", "2.0.0") is False # type: ignore[attr-defined] + from grpc.aio._typing import ChannelArgumentType # noqa: F401 + + import weaviate # noqa: F401 # must not raise even though grpcio is shimmed + from weaviate.proto.v1 import weaviate_pb2_grpc + + ch = GrpcWebChannel("localhost:50051", secure=False) + stub = weaviate_pb2_grpc.WeaviateStub(ch) + assert stub.Search is not None + assert stub.BatchObjects is not None + assert stub.BatchDelete is not None + assert isinstance(ch, grpc.aio.Channel) def test_sync_channel_factory_raises_async_only(): - result = _run( - """ - import weaviate_client_web - weaviate_client_web.install(force=True) - import grpc - try: - grpc.insecure_channel("localhost:50051") - except RuntimeError as exc: - assert "async" in str(exc).lower() - print("OK") - else: - raise AssertionError("expected sync channel factory to raise") - """ - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_real_proto_unary_round_trip_under_shim(): - result = _run( - """ - import asyncio - import struct - import weaviate_client_web - weaviate_client_web.install(force=True) - - import weaviate # noqa: F401 - from weaviate.proto.v1 import tenants_pb2, weaviate_pb2_grpc - - reply = tenants_pb2.TenantsGetReply() - payload = reply.SerializeToString() - - def frame(p, flag=0x00): - return struct.pack(">BI", flag, len(p)) + p - - body = frame(payload) + frame(b"grpc-status:0\\r\\n", 0x80) - - async def sender(url, headers, body_in, timeout): - assert headers["authorization"] == "Bearer k" - assert url.endswith("/weaviate.v1.Weaviate/TenantsGet") - return 200, {}, body - - weaviate_client_web.set_sender(sender) - from weaviate_client_web import GrpcWebChannel - ch = GrpcWebChannel("localhost:50051", secure=False) - stub = weaviate_pb2_grpc.WeaviateStub(ch) + import grpc - async def main(): - res = await stub.TenantsGet( - tenants_pb2.TenantsGetRequest(), - metadata=[("authorization", "Bearer k")], - timeout=5, - ) - assert isinstance(res, tenants_pb2.TenantsGetReply) - print("OK") + with pytest.raises(RuntimeError, match="async"): + grpc.insecure_channel("localhost:50051") - asyncio.run(main()) - """ - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout + +async def test_real_proto_unary_round_trip_under_shim(): + from weaviate.proto.v1 import tenants_pb2, weaviate_pb2_grpc + + reply = tenants_pb2.TenantsGetReply() + payload = reply.SerializeToString() + body = _frame(payload) + _frame(b"grpc-status:0\r\n", 0x80) + + async def sender(url, headers, body_in, timeout): + assert headers["authorization"] == "Bearer k" + assert url.endswith("/weaviate.v1.Weaviate/TenantsGet") + return 200, {}, body + + set_sender(sender) + try: + ch = GrpcWebChannel("localhost:50051", secure=False) + stub = weaviate_pb2_grpc.WeaviateStub(ch) + res = await stub.TenantsGet( + tenants_pb2.TenantsGetRequest(), + metadata=[("authorization", "Bearer k")], + timeout=5, + ) + assert isinstance(res, tenants_pb2.TenantsGetReply) + finally: + set_sender(pyfetch_sender) def test_fake_grpc_version_matches_base_fallback(): - # In-process on purpose: nothing here installs the shim, we only compare the two - # copies of the pinned version. The shim advertises FAKE_GRPC_VERSION as - # grpc.__version__ and the base package falls back to _GRPCIO_FALLBACK_VERSION - # under Emscripten — the vendored stubs' version gates see both, so they must - # never drift apart. + # The shim advertises FAKE_GRPC_VERSION as grpc.__version__ and the base package + # falls back to _GRPCIO_FALLBACK_VERSION under Emscripten — the vendored stubs' + # version gates see both, so they must never drift apart. from weaviate.proto.v1 import _GRPCIO_FALLBACK_VERSION - from weaviate_client_web._shim import FAKE_GRPC_VERSION assert FAKE_GRPC_VERSION == _GRPCIO_FALLBACK_VERSION diff --git a/packages/web/tests/test_single_import.py b/packages/web/tests/test_single_import.py deleted file mode 100644 index 562f31e0a..000000000 --- a/packages/web/tests/test_single_import.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Tests for the single-import hook in the base client (``weaviate/__init__.py``). - -The hook fires on ``sys.platform == "emscripten"`` and (via the companion's bootstrap) -replaces ``sys.modules['grpc']`` process-wide, so each scenario runs in a fresh -subprocess with the platform faked before ``import weaviate`` — the same pattern as -test_shim_install.py / test_httpx_fetch.py's install tests. -""" - -import pathlib -import subprocess -import sys -import textwrap - -_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") -_REPO_ROOT = str(pathlib.Path(__file__).resolve().parents[3]) - -# CPython derives the _sysconfigdata module name from sys.platform on first use, so a -# faked platform breaks any later sysconfig lookup (pydantic imports zoneinfo, which -# calls sysconfig.get_config_var). Prime the cache before faking. -_PRIME_SYSCONFIG = """ -import sysconfig - -sysconfig.get_config_vars() -""" - -# The companion's bootstrap installs the fetch transport under Emscripten and fails fast -# if pyodide.http cannot be imported, so a faked platform needs a stand-in module. -_FAKE_PYODIDE = """ -import types - -_pyodide = types.ModuleType("pyodide") -_http = types.ModuleType("pyodide.http") -_http.pyfetch = None -_pyodide.http = _http -sys.modules["pyodide"] = _pyodide -sys.modules["pyodide.http"] = _http -""" - - -def _run( - body: str, *, prelude: str = "", path_entry: str = _SRC, no_site: bool = False -) -> subprocess.CompletedProcess: - # -I -S: skip site-packages entirely (plain -I still processes the venv's .pth - # files), so nothing pip-installed is importable — only stdlib plus `path_entry`. - interp = [sys.executable, "-I", "-S"] if no_site else [sys.executable] - script = f"import sys\nsys.path.insert(0, {path_entry!r})\n" + prelude + textwrap.dedent(body) - return subprocess.run([*interp, "-c", script], capture_output=True, text=True) - - -def test_bare_import_weaviate_installs_shim_under_emscripten(): - result = _run( - prelude=_PRIME_SYSCONFIG + _FAKE_PYODIDE, - body=""" - sys.platform = "emscripten" - - import weaviate # the ONLY weaviate-side import: must bootstrap the companion - - assert "weaviate_client_web" in sys.modules, "hook did not import the companion" - import weaviate_client_web - assert weaviate_client_web.is_installed() - assert weaviate_client_web.is_fetch_transport_installed() - import grpc - assert getattr(grpc, "__weaviate_client_web_shim__", False) is True - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_bare_import_without_companion_raises_clear_import_error(): - # No site-packages, so neither weaviate_client_web nor grpcio is importable; the repo - # root goes on sys.path so the weaviate package itself is still found. - result = _run( - """ - sys.platform = "emscripten" - try: - import weaviate - except ImportError as e: - assert "weaviate-client-web" in str(e), str(e) - assert "WebAssembly/Pyodide" in str(e), str(e) - print("OK") - else: - raise AssertionError("expected ImportError without the companion") - """, - path_entry=_REPO_ROOT, - no_site=True, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_bare_import_with_grpc_present_falls_through_silently(): - # Companion blocked but a real grpc IS importable (grpcio in the dev env): the hook - # must fall through and leave the normal import path untouched. - result = _run( - prelude=_PRIME_SYSCONFIG, - body=""" - sys.platform = "emscripten" - sys.modules["weaviate_client_web"] = None # makes its import raise ImportError - - import weaviate - import grpc - - assert not getattr(grpc, "__weaviate_client_web_shim__", False) - print("OK") - """, - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout - - -def test_bare_import_with_broken_companion_surfaces_its_own_error(tmp_path): - # An INSTALLED companion whose import fails (here: a missing dependency of its own) - # must raise that error, not the "install weaviate-client-web" hint — the hint would - # send the user to reinstall a package that is already there. - fake_pkg = tmp_path / "weaviate_client_web" - fake_pkg.mkdir() - (fake_pkg / "__init__.py").write_text( - "raise ModuleNotFoundError(\"No module named 'anyio'\", name='anyio')\n" - ) - result = _run( - prelude=_PRIME_SYSCONFIG, - body=""" - sys.platform = "emscripten" - try: - import weaviate - except ImportError as e: - assert e.name == "anyio", (e.name, str(e)) - assert "anyio" in str(e), str(e) - assert "weaviate-client-web" not in str(e), str(e) - print("OK") - else: - raise AssertionError("expected the companion's own ImportError to surface") - """, - path_entry=str(tmp_path), - ) - assert result.returncode == 0, result.stderr - assert "OK" in result.stdout diff --git a/packages/web/tests/test_transport.py b/packages/web/tests/test_transport.py index e1a43492b..b063767ee 100644 --- a/packages/web/tests/test_transport.py +++ b/packages/web/tests/test_transport.py @@ -1,8 +1,9 @@ -"""In-process tests for the grpc-web channel/multicallable. +"""grpc-web channel / multicallable tests. -These exercise the transport classes directly (they import their grpc base classes from -``weaviate_client_web._shim``, not from ``sys.modules['grpc']``), so no shim install is -needed and the real ``grpc`` in the dev environment is left untouched. +Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs`` — async tests execute on +Pyodide's event loop through JSPI stack switching (pytest-asyncio in auto mode). These +exercise the transport classes directly through fake senders — no network, no running +Weaviate. """ import asyncio @@ -12,13 +13,15 @@ import pytest -from weaviate_client_web._channel import ( - GrpcWebChannel, - _body_excerpt, - _encode_timeout, - set_sender, +from weaviate_client_web import GrpcWebChannel, set_sender +from weaviate_client_web._channel import _body_excerpt, _encode_timeout +from weaviate_client_web._sender import pyfetch_sender +from weaviate_client_web._shim import ( + AioChannel, + AioRpcError, + StatusCode, + _aio_insecure_channel, ) -from weaviate_client_web._shim import AioChannel, AioRpcError, StatusCode def _frame(payload: bytes, flag: int = 0x00) -> bytes: @@ -52,7 +55,7 @@ def test_grpcwebchannel_is_grpc_aio_channel(): assert isinstance(_channel(FakeSender()), AioChannel) -def test_unary_success_round_trip(): +async def test_unary_success_round_trip(): sender = FakeSender(body=_ok_response(b"reply-bytes")) channel = _channel(sender) mc = channel.unary_unary( @@ -62,7 +65,7 @@ def test_unary_success_round_trip(): _registered_method=True, ) - result = asyncio.run(mc(b"request-bytes", metadata=[("authorization", "Bearer k")], timeout=5)) + result = await mc(b"request-bytes", metadata=[("authorization", "Bearer k")], timeout=5) assert result == b"reply-bytes" url, headers, body, timeout = sender.calls[0] @@ -74,50 +77,50 @@ def test_unary_success_round_trip(): assert timeout == 5 -def test_secure_channel_uses_https(): +async def test_secure_channel_uses_https(): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender, secure=True) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - asyncio.run(mc(b"q")) + await mc(b"q") assert sender.calls[0][0].startswith("https://example.com:443/") -def test_health_call_without_metadata(): +async def test_health_call_without_metadata(): sender = FakeSender(body=_ok_response(b"pong")) channel = _channel(sender) mc = channel.unary_unary("/grpc.health.v1.Health/Check", lambda x: x, lambda b: b) # mirrors the health check in connect/v4.py — request + timeout, no metadata - assert asyncio.run(mc(b"ping", timeout=2)) == b"pong" + assert await mc(b"ping", timeout=2) == b"pong" -def test_error_trailer_raises_aiorpcerror(): +async def test_error_trailer_raises_aiorpcerror(): body = _frame(b"grpc-status:7\r\ngrpc-message:nope\r\n", 0x80) channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + await mc(b"q") assert excinfo.value.code() is StatusCode.PERMISSION_DENIED assert excinfo.value.code().name == "PERMISSION_DENIED" assert excinfo.value.details() == "nope" -def test_percent_encoded_grpc_message_decoded(): +async def test_percent_encoded_grpc_message_decoded(): body = _frame(b"grpc-status:5\r\ngrpc-message:not%20found\r\n", 0x80) channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + await mc(b"q") assert excinfo.value.details() == "not found" -def test_trailers_only_status_in_http_headers(): +async def test_trailers_only_status_in_http_headers(): channel = _channel( FakeSender(status=200, headers={"grpc-status": "16", "grpc-message": "auth"}, body=b"") ) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + await mc(b"q") assert excinfo.value.code() is StatusCode.UNAUTHENTICATED @@ -149,19 +152,19 @@ def test_trailers_only_status_in_http_headers(): ) -def _details_of(status, body, headers=None, path="/grpc.health.v1.Health/Check"): +async def _details_of(status, body, headers=None, path="/grpc.health.v1.Health/Check"): """Run one request against a canned HTTP response and return the AioRpcError.""" channel = _channel(FakeSender(status=status, headers=headers or {}, body=body)) mc = channel.unary_unary(path, lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + await mc(b"q") return excinfo.value -def test_weaviate_404_json_names_both_candidate_causes(): +async def test_weaviate_404_json_names_both_candidate_causes(): # A 404 means EITHER the server predates the native /v1/grpc-web endpoint OR the # configured path prefix is wrong. The channel cannot tell which, so it must say both. - err = _details_of(404, WEAVIATE_404_JSON, {"content-type": "application/json"}) + err = await _details_of(404, WEAVIATE_404_JSON, {"content-type": "application/json"}) details = err.details() assert err.code() is StatusCode.UNIMPLEMENTED @@ -174,44 +177,46 @@ def test_weaviate_404_json_names_both_candidate_causes(): assert "malformed grpc-web response" not in details -def test_nginx_502_maps_to_unavailable_so_the_client_retries(): +async def test_nginx_502_maps_to_unavailable_so_the_client_retries(): # weaviate/retry.py retries UNAVAILABLE and nothing else; a gateway error arriving # as INTERNAL is silently un-retried, which is the regression this pins. - err = _details_of(502, NGINX_502_HTML) + err = await _details_of(502, NGINX_502_HTML) assert err.code() is StatusCode.UNAVAILABLE assert err.details().startswith("HTTP 502 ") assert "502 Bad Gateway" in err.details() @pytest.mark.parametrize("status", [503, 504]) -def test_gateway_errors_are_unavailable(status): - err = _details_of(status, b"upstream down") +async def test_gateway_errors_are_unavailable(status): + err = await _details_of(status, b"upstream down") assert err.code() is StatusCode.UNAVAILABLE -def test_nginx_404_html_is_reported_as_an_http_404(): - err = _details_of(404, NGINX_404_HTML) +async def test_nginx_404_html_is_reported_as_an_http_404(): + err = await _details_of(404, NGINX_404_HTML) assert err.code() is StatusCode.UNIMPLEMENTED assert err.details().startswith("HTTP 404 ") assert "404 Not Found" in err.details() assert "malformed grpc-web response" not in err.details() -def test_405_names_the_wrong_prefix(): +async def test_405_names_the_wrong_prefix(): # a 405 can only come from an existing HTTP route (measured live: a prefix pointing # at /v1/objects answers "method POST is not allowed"), so the prefix is wrong - err = _details_of(405, b'{"code":405,"message":"method POST is not allowed, but [GET] are"}') + err = await _details_of( + 405, b'{"code":405,"message":"method POST is not allowed, but [GET] are"}' + ) assert err.details().startswith("HTTP 405 ") assert "path prefix" in err.details() assert "/v1/grpc-web" in err.details() assert "method POST is not allowed" in err.details() -def test_truncated_grpc_web_body_is_reported_as_truncated_not_as_wrong_prefix(): +async def test_truncated_grpc_web_body_is_reported_as_truncated_not_as_wrong_prefix(): # a valid frame header whose payload was cut short: the endpoint IS grpc-web, so the # SPA / path-prefix hint would send the user the wrong way body = _ok_response(b"reply-bytes")[:-6] - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.INTERNAL assert "truncated" in err.details() assert "cut short" in err.details() @@ -219,43 +224,43 @@ def test_truncated_grpc_web_body_is_reported_as_truncated_not_as_wrong_prefix(): assert "path prefix" not in err.details() -def test_spa_html_body_is_not_reported_as_truncated(): +async def test_spa_html_body_is_not_reported_as_truncated(): # text bodies decode to an unknown flag byte and a garbage length; they must read as # "not grpc-web framing", never as a truncated grpc-web body - err = _details_of(200, SPA_INDEX_HTML) + err = await _details_of(200, SPA_INDEX_HTML) assert "truncated" not in err.details() assert "not grpc-web framing" in err.details() -def test_message_frame_after_trailer_is_internal(): +async def test_message_frame_after_trailer_is_internal(): body = _frame(b"a") + _frame(b"grpc-status:0\r\n", 0x80) + _frame(b"late") - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.INTERNAL assert "malformed grpc-web response" in err.details() assert "after the trailer" in err.details() assert "single-page-app" not in err.details() -def test_multiple_message_frames_in_unary_response_is_internal(): +async def test_multiple_message_frames_in_unary_response_is_internal(): # a unary RPC has exactly one message; silently taking the first would hide a proxy # or server that streams several body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:0\r\n", 0x80) - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.INTERNAL assert "2 message frames" in err.details() -def test_error_status_wins_over_multiple_message_frames(): +async def test_error_status_wins_over_multiple_message_frames(): body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:5\r\ngrpc-message:gone\r\n", 0x80) - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.NOT_FOUND assert err.details() == "gone" -def test_spa_fallback_html_200_is_distinguishable_from_a_404(): +async def test_spa_fallback_html_200_is_distinguishable_from_a_404(): # An HTTP 200 serving index.html is the other half of a wrong path prefix: the app's # catch-all route answers instead of Weaviate. It must not read as malformed framing. - err = _details_of(200, SPA_INDEX_HTML) + err = await _details_of(200, SPA_INDEX_HTML) details = err.details() assert details.startswith("HTTP 200 ") @@ -263,82 +268,81 @@ def test_spa_fallback_html_200_is_distinguishable_from_a_404(): assert "single-page-app" in details # names the actual cause assert "malformed grpc-web response" not in details # distinguishable from the 404 case, not the same generic message - assert details != _details_of(404, NGINX_404_HTML).details() + assert details != (await _details_of(404, NGINX_404_HTML)).details() -def test_401_json_body_maps_to_unauthenticated(): - err = _details_of(401, b'{"error":[{"message":"anonymous access not enabled"}]}') +async def test_401_json_body_maps_to_unauthenticated(): + err = await _details_of(401, b'{"error":[{"message":"anonymous access not enabled"}]}') assert err.code() is StatusCode.UNAUTHENTICATED assert err.details().startswith("HTTP 401 ") assert "anonymous access not enabled" in err.details() -def test_403_error_body_reaches_details(): +async def test_403_error_body_reaches_details(): # regression: the response body is the most actionable part of the error and must # survive into details() rather than being parsed as frames and discarded - err = _details_of(403, b'{"code":403,"message":"forbidden: rbac denied"}') + err = await _details_of(403, b'{"code":403,"message":"forbidden: rbac denied"}') assert err.code() is StatusCode.PERMISSION_DENIED assert "forbidden: rbac denied" in err.details() -def test_error_body_excerpt_is_capped(): - err = _details_of(500, b"E" * 5000) +async def test_error_body_excerpt_is_capped(): + err = await _details_of(500, b"E" * 5000) details = err.details() assert "EEEE" in details assert details.endswith("...") assert len(details) < 600 # the 5000-byte body is excerpted, not pasted in -def test_binary_error_body_does_not_break_the_error(): +async def test_binary_error_body_does_not_break_the_error(): # a proxy answering with a binary payload must not raise UnicodeDecodeError while # the error message is being built - err = _details_of(502, b"\xff\xfe\x00\x01\x02") + err = await _details_of(502, b"\xff\xfe\x00\x01\x02") assert err.code() is StatusCode.UNAVAILABLE assert err.details().startswith("HTTP 502 ") -def test_non_200_with_valid_grpc_web_trailers_still_uses_grpc_status(): +async def test_non_200_with_valid_grpc_web_trailers_still_uses_grpc_status(): # guard on the fix's shape: the HTTP status must not shadow a real grpc-status that # a proxy shipped alongside a non-200 - err = _details_of(500, _frame(b"grpc-status:7\r\ngrpc-message:denied\r\n", 0x80)) + err = await _details_of(500, _frame(b"grpc-status:7\r\ngrpc-message:denied\r\n", 0x80)) assert err.code() is StatusCode.PERMISSION_DENIED assert err.details() == "denied" -def test_non_ascii_grpc_message_preserves_the_status(): +async def test_non_ascii_grpc_message_preserves_the_status(): # a trailer carrying raw UTF-8 (an un-percent-encoded proxy, or an error quoting a # collection name) must not degrade to INTERNAL and lose grpc-status body = _frame("grpc-status:5\r\ngrpc-message:collection Café not found\r\n".encode(), 0x80) - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.NOT_FOUND assert "Caf" in err.details() -def test_invalid_utf8_grpc_message_preserves_the_status(): +async def test_invalid_utf8_grpc_message_preserves_the_status(): # latin-1 bytes are not valid UTF-8; the status must still survive body = _frame(b"grpc-status:9\r\ngrpc-message:tenant caf\xe9 is COLD\r\n", 0x80) - err = _details_of(200, body) + err = await _details_of(200, body) assert err.code() is StatusCode.FAILED_PRECONDITION assert "tenant caf" in err.details() -def test_binary_metadata_base64_encoded(): +async def test_binary_metadata_base64_encoded(): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - asyncio.run(mc(b"q", metadata=[("trace-bin", b"\x00\x01\x02")])) + await mc(b"q", metadata=[("trace-bin", b"\x00\x01\x02")]) assert sender.calls[0][1]["trace-bin"] == "AAEC" def test_stream_stream_raises_clear_error(): channel = _channel(FakeSender()) mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) - with pytest.raises(RuntimeError) as excinfo: + with pytest.raises(RuntimeError, match="not supported over grpc-web"): mc(request_iterator=iter([]), timeout=5, metadata=None) - assert "not supported over grpc-web" in str(excinfo.value) -def test_timeout_maps_to_deadline_exceeded(): +async def test_timeout_maps_to_deadline_exceeded(): async def slow_sender(url, headers, body, timeout): await asyncio.sleep(0.5) return 200, {}, _ok_response(b"x") @@ -346,23 +350,23 @@ async def slow_sender(url, headers, body, timeout): channel = GrpcWebChannel("h:1", secure=False, sender=slow_sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q", timeout=0.01)) + await mc(b"q", timeout=0.01) assert excinfo.value.code() is StatusCode.DEADLINE_EXCEEDED -def test_transport_exception_maps_to_unavailable(): +async def test_transport_exception_maps_to_unavailable(): async def boom(url, headers, body, timeout): raise ConnectionError("connection refused") channel = GrpcWebChannel("h:1", secure=False, sender=boom) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + await mc(b"q") assert excinfo.value.code() is StatusCode.UNAVAILABLE assert "ConnectionError: connection refused" in str(excinfo.value.details()) -def test_transport_exception_with_empty_str_keeps_type(): +async def test_transport_exception_with_empty_str_keeps_type(): # httpx transport errors commonly stringify to '' — the detail must still name them async def boom(url, headers, body, timeout): raise ConnectionError() @@ -370,51 +374,51 @@ async def boom(url, headers, body, timeout): channel = GrpcWebChannel("h:1", secure=False, sender=boom) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + await mc(b"q") assert "ConnectionError" in str(excinfo.value.details()) -def test_empty_ok_response_hints_at_cors_expose_headers(): +async def test_empty_ok_response_hints_at_cors_expose_headers(): # HTTP 200, empty body, no grpc-status anywhere: the shape of a trailers-only error # whose grpc-status/grpc-message headers were stripped by CORS channel = _channel(FakeSender(status=200, headers={}, body=b"")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "Access-Control-Expose-Headers" in str(excinfo.value.details()) -def test_empty_ok_response_with_grpc_status_has_no_cors_hint(): +async def test_empty_ok_response_with_grpc_status_has_no_cors_hint(): # when grpc-status WAS visible (status 0, no frames), it is a malformed response, # not a CORS problem — the hint must not appear channel = _channel(FakeSender(status=200, headers={"grpc-status": "0"}, body=b"")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "Access-Control-Expose-Headers" not in str(excinfo.value.details()) -def test_message_frame_without_grpc_status_is_internal_not_success(): +async def test_message_frame_without_grpc_status_is_internal_not_success(): # HTTP 200 with a valid message frame but no grpc-status anywhere (e.g. a proxy # dropped the trailer frame) must be an error, never a fabricated success channel = _channel(FakeSender(status=200, headers={}, body=_frame(b"reply-bytes"))) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL assert "missing grpc-status" in str(excinfo.value.details()) -def test_message_frame_with_grpc_status_header_still_succeeds(): +async def test_message_frame_with_grpc_status_header_still_succeeds(): # trailers-only-in-headers responses (grpc-status as an HTTP header, no trailer # frame) remain valid per the grpc-web contract channel = _channel( FakeSender(status=200, headers={"grpc-status": "0"}, body=_frame(b"reply-bytes")) ) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - assert asyncio.run(mc(b"q")) == b"reply-bytes" + assert await mc(b"q") == b"reply-bytes" def test_stream_stream_error_recommends_insert_many_only(): @@ -429,30 +433,30 @@ def test_stream_stream_error_recommends_insert_many_only(): assert sync_only not in str(excinfo.value) -def test_malformed_frame_maps_to_internal(): +async def test_malformed_frame_maps_to_internal(): # A 3-byte body cannot contain even a 5-byte frame header -> framing ValueError. channel = _channel(FakeSender(body=b"\x00\x00\x00")) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL -def test_malformed_grpc_status_maps_to_internal(): +async def test_malformed_grpc_status_maps_to_internal(): body = _frame(b"grpc-status:notanint\r\n", 0x80) channel = _channel(FakeSender(body=body)) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + await mc(b"q") assert excinfo.value.code() is StatusCode.INTERNAL -def test_grpc_timeout_header_rounds_up(): +async def test_grpc_timeout_header_rounds_up(): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) # 123.4ms must round UP to 124ms (never advertise a shorter deadline than requested). - asyncio.run(mc(b"q", timeout=0.1234)) + await mc(b"q", timeout=0.1234) assert sender.calls[0][1]["grpc-timeout"] == "124m" @@ -489,27 +493,27 @@ def test_encode_timeout_stays_within_eight_digits(seconds, expected): assert not encoded.endswith("H") -def test_infinite_timeout_sends_no_deadline(): +async def test_infinite_timeout_sends_no_deadline(): # Timeout(query=inf): neither a grpc-timeout header nor a client-side wait sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - assert asyncio.run(mc(b"q", timeout=float("inf"))) == b"x" + assert await mc(b"q", timeout=float("inf")) == b"x" _, headers, _, timeout = sender.calls[0] assert "grpc-timeout" not in headers assert timeout is None -def test_huge_timeout_uses_minutes_then_no_deadline(): +async def test_huge_timeout_uses_minutes_then_no_deadline(): # 1e8 s in milliseconds is 12 digits; the server rejects more than 8 ("timeout is # too long", HTTP 400) and transcoders reject hour values above 8H, so past the # minute range the request carries no deadline at all sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - asyncio.run(mc(b"q", timeout=1e8)) - asyncio.run(mc(b"q", timeout=1e9)) - asyncio.run(mc(b"q", timeout=1e10)) + await mc(b"q", timeout=1e8) + await mc(b"q", timeout=1e9) + await mc(b"q", timeout=1e10) assert sender.calls[0][1]["grpc-timeout"] == "1666667M" assert sender.calls[1][1]["grpc-timeout"] == "16666667M" assert "grpc-timeout" not in sender.calls[2][1] @@ -517,18 +521,18 @@ def test_huge_timeout_uses_minutes_then_no_deadline(): @pytest.mark.parametrize("bad", ["val\r\nx-injected: evil", "val\nx", "v\0"]) -def test_crlf_in_metadata_rejected(bad): +async def test_crlf_in_metadata_rejected(bad): sender = FakeSender(body=_ok_response(b"x")) channel = _channel(sender) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(ValueError, match="Illegal character"): - asyncio.run(mc(b"q", metadata=[("x-key", bad)])) + await mc(b"q", metadata=[("x-key", bad)]) with pytest.raises(ValueError, match="Illegal character"): - asyncio.run(mc(b"q", metadata=[("x-key\r\n", "v")])) + await mc(b"q", metadata=[("x-key\r\n", "v")]) assert sender.calls == [] -def _unavailable_details(monkeypatch, path_prefix, platform): +async def _unavailable_details(monkeypatch, path_prefix, platform): async def boom(url, headers, body, timeout): raise ConnectionError("Failed to fetch") @@ -536,43 +540,47 @@ async def boom(url, headers, body, timeout): channel = GrpcWebChannel("h:50051", secure=False, sender=boom, path_prefix=path_prefix) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: - asyncio.run(mc(b"q")) + await mc(b"q") assert excinfo.value.code() is StatusCode.UNAVAILABLE return excinfo.value.details() -def test_unavailable_without_path_prefix_under_emscripten_hints_at_grpc_path_prefix(monkeypatch): +async def test_unavailable_without_path_prefix_under_emscripten_hints_at_grpc_path_prefix( + monkeypatch, +): # the connect helpers always set the prefix under Emscripten, so a prefix-less channel # here means hand-built ConnectionParams; the error must say what to do instead - details = _unavailable_details(monkeypatch, path_prefix="", platform="emscripten") + details = await _unavailable_details(monkeypatch, path_prefix="", platform="emscripten") assert "grpc_path_prefix='/v1/grpc-web'" in details assert "1.38.3" in details assert "connect helpers" in details -def test_unavailable_with_path_prefix_has_no_prefix_hint(monkeypatch): - details = _unavailable_details(monkeypatch, path_prefix="/v1/grpc-web", platform="emscripten") +async def test_unavailable_with_path_prefix_has_no_prefix_hint(monkeypatch): + details = await _unavailable_details( + monkeypatch, path_prefix="/v1/grpc-web", platform="emscripten" + ) assert "no grpc_path_prefix" not in details -def test_unavailable_without_path_prefix_off_emscripten_has_no_prefix_hint(monkeypatch): - # on CPython an empty prefix against a transcoder is the normal configuration - details = _unavailable_details(monkeypatch, path_prefix="", platform="linux") +async def test_unavailable_without_path_prefix_off_emscripten_has_no_prefix_hint(monkeypatch): + # off Emscripten an empty prefix against a transcoder is the normal configuration + details = await _unavailable_details(monkeypatch, path_prefix="", platform="linux") assert "no grpc_path_prefix" not in details -def test_close_is_awaitable_noop(): +async def test_close_is_awaitable_noop(): channel = _channel(FakeSender()) - assert asyncio.run(channel.close()) is None + assert await channel.close() is None -def test_path_prefix_prepended_to_url(): +async def test_path_prefix_prepended_to_url(): sender = FakeSender(body=_ok_response(b"r")) channel = GrpcWebChannel( "example.com:8090", secure=False, sender=sender, path_prefix="/grpc-web" ) mc = channel.unary_unary("/weaviate.v1.Weaviate/Search", lambda x: x, lambda b: b) - asyncio.run(mc(b"q")) + await mc(b"q") assert sender.calls[0][0] == "http://example.com:8090/grpc-web/weaviate.v1.Weaviate/Search" @@ -585,17 +593,15 @@ def test_path_prefix_prepended_to_url(): ("", "http://h:1/svc/M"), ], ) -def test_path_prefix_normalized_in_url(raw, expected_url): +async def test_path_prefix_normalized_in_url(raw, expected_url): sender = FakeSender(body=_ok_response(b"r")) channel = GrpcWebChannel("h:1", secure=False, sender=sender, path_prefix=raw) mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - asyncio.run(mc(b"q")) + await mc(b"q") assert sender.calls[0][0] == expected_url def test_shim_factory_extracts_path_prefix_option(): - from weaviate_client_web._shim import _aio_insecure_channel - with_prefix = _aio_insecure_channel( target="h:1", options=[("grpc.max_send_message_length", 1), ("grpc-web.path_prefix", "/grpc-web")], @@ -608,15 +614,13 @@ def test_shim_factory_extracts_path_prefix_option(): assert without_prefix._path_prefix == "" -def test_set_sender_overrides_default(): +async def test_set_sender_overrides_default(): sender = FakeSender(body=_ok_response(b"y")) set_sender(sender) try: channel = GrpcWebChannel("h:1", secure=False) # no explicit sender mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) - assert asyncio.run(mc(b"q")) == b"y" + assert await mc(b"q") == b"y" finally: - # restore the real default so other tests/processes are unaffected - from weaviate_client_web._sender import pyfetch_sender - + # restore the real default so other tests are unaffected set_sender(pyfetch_sender) diff --git a/setup.cfg b/setup.cfg index 7343116a5..26b397613 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,12 @@ python_requires = >=3.10 [options.extras_require] agents = weaviate-agents >=1.0.0, <2.0.0 +grpc-web = + # The Pyodide/WASM grpc-web companion, built from packages/web in this repo. Both + # packages derive their version from the same git tag (setuptools_scm; CI asserts + # the built wheels match). The marker makes the extra a no-op on CPython: the + # companion imports pyodide at module scope and only makes sense under Emscripten. + weaviate-client-web; sys_platform == "emscripten" [options.package_data] # If any package or subpackage contains *.txt, *.rst or *.md files, include them: diff --git a/test/test_wasm_compat.py b/test/test_wasm_compat.py index 5398a825a..ab2da4bb7 100644 --- a/test/test_wasm_compat.py +++ b/test/test_wasm_compat.py @@ -6,7 +6,10 @@ """ import asyncio +import pathlib +import subprocess import sys +import textwrap import grpc import pytest @@ -288,3 +291,105 @@ def test_an_explicit_local_grpc_port_is_warned_about_but_the_default_is_not(emsc with pytest.warns(UserWarning, match="Con006"): client = weaviate.use_async_with_local(port=8080, grpc_port=8081) _assert_grpc_rides_rest(client) + + +# --- the single-import hook (weaviate/__init__.py) ------------------------------------ +# +# The hook fires on sys.platform == "emscripten" and (via the companion's bootstrap) +# replaces sys.modules['grpc'] process-wide, so each scenario runs in a fresh subprocess +# with the platform faked before `import weaviate`. The success path — a bare import +# that bootstraps the real companion — needs real Pyodide and runs in +# ci/pyodide-e2e/units.mjs; the hook's other branches are plain CPython logic and are +# pinned here. + +_REPO_ROOT = str(pathlib.Path(__file__).resolve().parents[1]) + +# CPython derives the _sysconfigdata module name from sys.platform on first use, so a +# faked platform breaks any later sysconfig lookup (pydantic imports zoneinfo, which +# calls sysconfig.get_config_var). Prime the cache before faking. +_PRIME_SYSCONFIG = """ +import sysconfig + +sysconfig.get_config_vars() +""" + + +def _run_hook_scenario( + body: str, *, prelude: str = "", path_entry: str = _REPO_ROOT, no_site: bool = False +) -> subprocess.CompletedProcess: + # -I -S: skip site-packages entirely (plain -I still processes the venv's .pth + # files), so nothing pip-installed is importable — only stdlib plus `path_entry`. + interp = [sys.executable, "-I", "-S"] if no_site else [sys.executable] + script = f"import sys\nsys.path.insert(0, {path_entry!r})\n" + prelude + textwrap.dedent(body) + return subprocess.run([*interp, "-c", script], capture_output=True, text=True) + + +def test_bare_import_without_companion_raises_clear_import_error() -> None: + # No site-packages, so neither weaviate_client_web nor grpcio is importable; the repo + # root goes on sys.path so the weaviate package itself is still found. + result = _run_hook_scenario( + """ + sys.platform = "emscripten" + try: + import weaviate + except ImportError as e: + assert "weaviate-client-web" in str(e), str(e) + assert "weaviate-client[grpc-web]" in str(e), str(e) + assert "WebAssembly/Pyodide" in str(e), str(e) + print("OK") + else: + raise AssertionError("expected ImportError without the companion") + """, + no_site=True, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_bare_import_with_grpc_present_falls_through_silently() -> None: + # Companion blocked but a real grpc IS importable (grpcio in the dev env): the hook + # must fall through and leave the normal import path untouched. + result = _run_hook_scenario( + prelude=_PRIME_SYSCONFIG, + body=""" + sys.platform = "emscripten" + sys.modules["weaviate_client_web"] = None # makes its import raise ImportError + + import weaviate + import grpc + + assert not getattr(grpc, "__weaviate_client_web_shim__", False) + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_bare_import_with_broken_companion_surfaces_its_own_error(tmp_path) -> None: + # An INSTALLED companion whose import fails (here: a missing dependency of its own) + # must raise that error, not the install hint — the hint would send the user to + # reinstall a package that is already there. + fake_pkg = tmp_path / "weaviate_client_web" + fake_pkg.mkdir() + (fake_pkg / "__init__.py").write_text( + "raise ModuleNotFoundError(\"No module named 'anyio'\", name='anyio')\n" + ) + result = _run_hook_scenario( + prelude=_PRIME_SYSCONFIG, + body=""" + sys.platform = "emscripten" + try: + import weaviate + except ImportError as e: + assert e.name == "anyio", (e.name, str(e)) + assert "anyio" in str(e), str(e) + assert "grpc-web" not in str(e), str(e) + print("OK") + else: + raise AssertionError("expected the companion's own ImportError to surface") + """, + path_entry=str(tmp_path), + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout diff --git a/weaviate/__init__.py b/weaviate/__init__.py index fbd0fdabf..ef40e7a96 100644 --- a/weaviate/__init__.py +++ b/weaviate/__init__.py @@ -19,8 +19,9 @@ "weaviate requires the weaviate-client-web package under " "WebAssembly/Pyodide: there is no grpcio wheel for Emscripten, and " "weaviate-client-web provides the grpc-web (fetch) transport in its " - "place. Install it (e.g. micropip.install('weaviate-client-web')) and " - "import weaviate again." + "place. Install it via the extra (e.g. " + "micropip.install('weaviate-client[grpc-web]')) and import weaviate " + "again." ) from exc import os diff --git a/weaviate/connect/base.py b/weaviate/connect/base.py index 0f7d40bd1..82c8b3851 100644 --- a/weaviate/connect/base.py +++ b/weaviate/connect/base.py @@ -165,10 +165,11 @@ def _check_grpc_web_usable(self, is_async: bool) -> None: raise WeaviateInvalidInputError( "grpc_path_prefix enables grpc-web, which requires the " "'weaviate-client-web' package (it installs a grpc shim before " - "'import weaviate'); it is not active in this environment. Under Pyodide a " - "plain `import weaviate` activates it; on CPython call " - "weaviate_client_web.install(force=True) and set_sender(make_httpx_sender()) " - "before importing weaviate (intended for integration testing)." + "'import weaviate'); it is not active in this environment. grpc-web is " + "only available under WebAssembly/Pyodide, where a plain `import " + "weaviate` activates it (install the companion with " + "micropip.install('weaviate-client[grpc-web]')); on CPython use native " + "gRPC instead." ) def _grpc_channel(