diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 489ff9504..266224164 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -45,11 +45,11 @@ jobs: cache: 'pip' # caching pip dependencies - run: pip install -r requirements-devel.txt - name: "Ruff lint" - run: ruff check weaviate test mock_tests integration + run: ruff check weaviate test mock_tests integration packages/web - name: "Ruff format" - run: ruff format --diff weaviate test mock_tests integration + run: ruff format --diff weaviate test mock_tests integration packages/web - name: "Flake 8" - run: flake8 weaviate test mock_tests integration + run: flake8 weaviate test mock_tests integration packages/web - name: "Check release for pypi" run: | python -m build @@ -105,6 +105,65 @@ jobs: name: coverage-report-${{ matrix.folder }} path: coverage-${{ matrix.folder }}.xml + pyodide-e2e: + name: Run Pyodide (WASM) Unit + e2e Tests + runs-on: ubuntu-latest + timeout-minutes: 15 + # The interpreter comes from the pyodide pin in ci/pyodide-e2e/package.json. + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + fetch-tags: true + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.11" + cache: 'pip' # caching pip dependencies + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: "22" + - name: Login to Docker Hub + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 + if: ${{ !github.event.pull_request.head.repo.fork && github.triggering_actor != 'dependabot[bot]' }} + with: + username: ${{secrets.DOCKER_USERNAME}} + password: ${{secrets.DOCKER_PASSWORD}} + - name: Build pure wheels (base client + grpc-web) + run: | + pip install build + python -m build --wheel --outdir dist . + python -m build --wheel --outdir dist packages/web + - name: Assert both wheels carry the same version + # weaviate-client-web is versioned in lockstep with weaviate-client: both derive + # from the same git tag via setuptools_scm. + run: | + base=$(basename dist/weaviate_client-*.whl); base=${base#weaviate_client-}; base=${base%%-*} + web=$(basename dist/weaviate_client_web-*.whl); web=${web#weaviate_client_web-}; web=${web%%-*} + echo "weaviate-client=$base weaviate-client-web=$web" + test "$base" = "$web" + - name: Run the unit suite inside Pyodide under Node + # weaviate-client-web imports pyodide, so its unit tests run only here. + # JSPI lets pytest run async tests. + run: | + npm install --prefix ci/pyodide-e2e + node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist + - name: start weaviate + run: | + source ./ci/compose.sh + export WEAVIATE_VERSION=$WEAVIATE_139 + docker compose -f ci/docker-compose-async.yml up -d + wait "http://localhost:8090" + - name: Run the e2e suite inside Pyodide under Node + env: + WEAVIATE_HOST: localhost + WEAVIATE_PORT: "8090" + run: | + npm install --prefix ci/pyodide-e2e + node ci/pyodide-e2e/run.mjs dist + - name: stop weaviate + if: always() + run: docker compose -f ci/docker-compose-async.yml down --remove-orphans + proto-test: name: Run importing protos test runs-on: ubuntu-latest @@ -295,8 +354,10 @@ jobs: cache: 'pip' # caching pip dependencies - name: Install dependencies run: pip install -r requirements-test.txt -r requirements-devel.txt - - name: Build a binary wheel - run: python -m build + - name: Build binary wheels (base client + grpc-web companion) + run: | + python -m build + python -m build --wheel --outdir dist packages/web - name: Create Wheel Artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -352,7 +413,7 @@ jobs: build-and-publish: name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI - needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package, proto-test] + needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package, proto-test, pyodide-e2e] runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -367,9 +428,21 @@ jobs: cache: 'pip' # caching pip dependencies - name: Install dependencies run: pip install -r requirements-devel.txt - - name: Build a binary wheel - run: python -m build - - name: Publish distribution 📦 to PyPI on new tags + - name: Build distributions (base client + grpc-web companion) + # weaviate-client-web is wheel-only: its version comes from git tags, which an + # sdist lacks. + run: | + python -m build + python -m build --wheel --outdir dist packages/web + - name: Assert both packages carry the same version + # weaviate-client-web pins weaviate-client==; a mismatch leaves the + # [grpc-web] extra unresolvable. + run: | + base=$(basename dist/weaviate_client-*.whl); base=${base#weaviate_client-}; base=${base%%-*} + web=$(basename dist/weaviate_client_web-*.whl); web=${web#weaviate_client_web-}; web=${web%%-*} + echo "weaviate-client=$base weaviate-client-web=$web" + test "$base" = "$web" + - name: Publish distributions 📦 to PyPI on new tags if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 with: diff --git a/.gitignore b/.gitignore index b4ba50e1b..395b51d6c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ venv .idea dist/ -weaviate_client.egg-info +*.egg-info/ **/__pycache__ tmp build/ @@ -27,3 +27,5 @@ scratch/ *-test.sh *.hdf5 *.jsonl +ci/pyodide-e2e/node_modules/ +ci/pyodide-e2e/package-lock.json diff --git a/ci/pyodide-e2e/e2e.py b/ci/pyodide-e2e/e2e.py new file mode 100644 index 000000000..b37eacca6 --- /dev/null +++ b/ci/pyodide-e2e/e2e.py @@ -0,0 +1,199 @@ +"""e2e suite run inside Pyodide by run.mjs, which awaits main(); prints one OK line per step. + +Not covered: browser CORS (runs under Node) and OIDC (anonymous access only). +""" + +import os +import uuid +import warnings + +import weaviate_client_web # bootstraps the grpc shim + fetch transport under Emscripten + +import grpc +import httpx +import weaviate +import weaviate.classes as wvc +from weaviate.classes.config import DataType, Property, ReferenceProperty +from weaviate.classes.data import DataReference +from weaviate.classes.query import Filter +from weaviate.classes.tenants import Tenant +from weaviate.exceptions import WeaviateBatchStreamError, WeaviateQueryError + +COLL = "PyodideE2E" +MT_COLL = "PyodideE2ETenants" +# Weaviate's grpc-web path prefix; the connect helpers select it under Emscripten. +GRPC_WEB_PREFIX = "/v1/grpc-web" + + +def ok(step: str) -> None: + print(f"OK {step}", flush=True) + + +async def main() -> None: + assert weaviate_client_web.is_installed(), "grpc shim did not install under Emscripten" + assert getattr(grpc, "__weaviate_client_web_shim__", False), ( + "sys.modules['grpc'] is not the shim" + ) + # REST must use weaviate-client-web's fetch transport, not Pyodide's bundled one. + assert weaviate_client_web.is_fetch_transport_installed(), "fetch transport not installed" + assert getattr( + httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False + ), "httpx.AsyncHTTPTransport is not the package's fetch transport" + ok("self-check: package fetch transport is the active httpx transport") + + host = os.environ.get("WEAVIATE_HOST", "localhost") + port = int(os.environ.get("WEAVIATE_PORT", "8090")) + client = weaviate.use_async_with_custom( + http_host=host, + http_port=port, + http_secure=False, + grpc_host=host, + grpc_port=port, + grpc_secure=False, + ) + params = client._connection._connection_params + assert params._grpc_web_path_prefix == GRPC_WEB_PREFIX, params + assert params._grpc_target == f"{host}:{port}", params + ok("connect helper routed gRPC onto the REST endpoint under /v1/grpc-web") + + # No skip_init_checks: connect() performs the gRPC health check over grpc-web. + await client.connect() + ok("connect (health check over grpc-web)") + + try: + for name in (COLL, MT_COLL): + if await client.collections.exists(name): + await client.collections.delete(name) + + await client.collections.create( + COLL, + vector_config=wvc.config.Configure.Vectors.self_provided(), + properties=[ + Property(name="title", data_type=DataType.TEXT), + Property(name="idx", data_type=DataType.INT), + ], + references=[ReferenceProperty(name="related", target_collection=COLL)], + ) + ok("collections.create") + + coll = client.collections.get(COLL) + ret = await coll.data.insert_many([{"title": f"article {i}", "idx": i} for i in range(50)]) + assert not ret.has_errors and len(ret.uuids) == 50, f"insert_many errors: {ret.errors}" + ok("insert_many (BatchObjects) = 50") + + res = await coll.query.fetch_objects(limit=100) + assert len(res.objects) == 50, f"fetch_objects got {len(res.objects)}" + ok("query.fetch_objects = 50") + + res = await coll.query.bm25("article", limit=5) + assert len(res.objects) == 5, f"bm25 got {len(res.objects)}" + ok("query.bm25 limit=5 = 5") + + res = await coll.query.fetch_objects( + filters=Filter.by_property("idx").less_than(10), limit=100 + ) + assert len(res.objects) == 10, f"filtered fetch_objects got {len(res.objects)}" + ok("query.fetch_objects filtered idx<10 = 10") + + agg = await coll.aggregate.over_all(total_count=True) + assert agg.total_count == 50, f"aggregate total_count {agg.total_count}" + agg = await coll.aggregate.over_all( + return_metrics=[wvc.query.Metrics("idx").integer(minimum=True, maximum=True)] + ) + idx = agg.properties["idx"] + assert idx.minimum == 0 and idx.maximum == 49, agg.properties + ok("aggregate count=50 min=0 max=49") + + # REST calls answered without a body (HEAD 204/404, PATCH/DELETE 204) and the + # batch-references path, which reads httpx's response.elapsed. + first, second, last = ret.uuids[0], ret.uuids[1], ret.uuids[49] + assert await coll.data.exists(first) is True + assert await coll.data.exists(uuid.uuid4()) is False + ok("data.exists (HEAD 204 / 404) = True / False") + + await coll.data.update(uuid=first, properties={"title": "article 0 (updated)"}) + obj = await coll.query.fetch_object_by_id(first) + assert obj is not None and obj.properties["title"] == "article 0 (updated)", obj + ok("data.update (PATCH 204) -> fetch_object_by_id sees the update") + + refs = await coll.data.reference_add_many( + [ + DataReference( + from_property="related", from_uuid=ret.uuids[i], to_uuid=ret.uuids[i + 1] + ) + for i in range(5) + ] + ) + assert not refs.has_errors, f"reference_add_many errors: {refs.errors}" + assert refs.elapsed_seconds >= 0, refs + ok("data.reference_add_many (REST /batch/references) = 5") + + await coll.data.reference_delete(from_uuid=first, from_property="related", to=second) + ok("data.reference_delete (DELETE 204)") + + assert await coll.data.delete_by_id(last) is True + assert await coll.data.exists(last) is False + # a repeat delete answers 204 or 404 depending on topology; both are body-less + assert isinstance(await coll.data.delete_by_id(last), bool) + ok("data.delete_by_id (DELETE 204; repeat -> 204/404) = True, then bool") + + await client.collections.create( + MT_COLL, + vector_config=wvc.config.Configure.Vectors.self_provided(), + properties=[Property(name="title", data_type=DataType.TEXT)], + multi_tenancy_config=wvc.config.Configure.multi_tenancy(enabled=True), + ) + mt = client.collections.get(MT_COLL) + await mt.tenants.create([Tenant(name="t1"), Tenant(name="t2")]) + tenants = await mt.tenants.get() + assert set(tenants.keys()) == {"t1", "t2"}, f"TenantsGet: {set(tenants.keys())}" + ok("multi-tenant create + TenantsGet = {t1, t2}") + + assert await mt.tenants.exists("t1") is True + assert await mt.tenants.exists("t404") is False + ok("tenants.exists (HEAD 200 / 404) = True / False") + + t1 = mt.with_tenant("t1") + ret = await t1.data.insert_many([{"title": f"tenant doc {i}"} for i in range(10)]) + assert not ret.has_errors and len(ret.uuids) == 10, f"tenant insert_many: {ret.errors}" + agg = await t1.aggregate.over_all(total_count=True) + assert agg.total_count == 10, f"tenant aggregate {agg.total_count}" + ok("per-tenant insert_many = 10, aggregate = 10") + + dm = await t1.data.delete_many(where=Filter.by_property("title").like("tenant*")) + assert dm.successful == 10, f"delete_many successful={dm.successful}" + agg = await t1.aggregate.over_all(total_count=True) + assert agg.total_count == 0, f"post-delete aggregate {agg.total_count}" + ok("per-tenant delete_many (BatchDelete) = 10 -> aggregate = 0") + + try: + await client.collections.get("DoesNotExistXyz").query.fetch_objects(limit=1) + raise AssertionError("expected WeaviateQueryError for nonexistent collection") + except WeaviateQueryError as e: + assert "DoesNotExistXyz" in str(e), str(e) + ok("error mapping: nonexistent collection -> WeaviateQueryError names the collection") + + try: + async with client.batch.stream() as batch: + await batch.add_object(collection=COLL, properties={"title": "x", "idx": 999}) + raise AssertionError("batch.stream() did not raise under grpc-web") + except WeaviateBatchStreamError as e: + assert "grpc-web" in str(e) and "insert_many" in str(e), str(e) + ok("batch.stream() -> WeaviateBatchStreamError (clear message)") + + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + async with client.batch.experimental() as batch: + await batch.add_object(collection=COLL, properties={"title": "x", "idx": 999}) + raise AssertionError("batch.experimental() did not raise under grpc-web") + except WeaviateBatchStreamError: + ok("batch.experimental() -> WeaviateBatchStreamError") + + for name in (COLL, MT_COLL): + await client.collections.delete(name) + ok("cleanup") + finally: + await client.close() + + print("PYODIDE E2E: ALL STEPS OK", flush=True) diff --git a/ci/pyodide-e2e/package.json b/ci/pyodide-e2e/package.json new file mode 100644 index 000000000..fbf78f303 --- /dev/null +++ b/ci/pyodide-e2e/package.json @@ -0,0 +1,8 @@ +{ + "name": "weaviate-pyodide-e2e", + "private": true, + "description": "Runs the weaviate-client e2e suite inside Pyodide (WASM) under Node", + "dependencies": { + "pyodide": "314.0.4" + } +} diff --git a/ci/pyodide-e2e/run.mjs b/ci/pyodide-e2e/run.mjs new file mode 100644 index 000000000..703d0f05b --- /dev/null +++ b/ci/pyodide-e2e/run.mjs @@ -0,0 +1,71 @@ +// Runs e2e.py inside Pyodide under Node against a live Weaviate. +// Usage: node run.mjs (one weaviate_client-*.whl, one weaviate_client_web-*.whl) +// Env: WEAVIATE_HOST (default localhost), WEAVIATE_PORT (default 8090). +// The pyodide npm pin in package.json fixes the interpreter. +import { readdirSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { loadPyodide } from "pyodide"; + +if (!process.argv[2]) { + console.error("usage: node run.mjs "); + process.exit(2); +} +const wheelsDir = resolve(process.argv[2]); +const here = dirname(fileURLToPath(import.meta.url)); + +const wheels = readdirSync(wheelsDir) + .filter((f) => f.endsWith(".whl")) + .sort(); // installs weaviate_client before weaviate_client_web, which depends on it +const prefixes = ["weaviate_client-", "weaviate_client_web-"]; +if ( + wheels.length !== 2 || + !prefixes.every((p) => wheels.some((w) => w.startsWith(p))) +) { + console.error( + `expected exactly one weaviate_client-*.whl and one weaviate_client_web-*.whl in ${wheelsDir}, found: ${JSON.stringify(wheels)}`, + ); + process.exit(2); +} + +const pyodide = await loadPyodide({ + env: { + WEAVIATE_HOST: process.env.WEAVIATE_HOST ?? "localhost", + WEAVIATE_PORT: process.env.WEAVIATE_PORT ?? "8090", + }, +}); +console.log( + `pyodide ${pyodide.version} / python ${pyodide.runPython("import sys; sys.version.split()[0]")}`, +); + +await pyodide.loadPackage("micropip"); +const micropip = pyodide.pyimport("micropip"); +// anyio comes from weaviate-client-web's emscripten marker; installing it here would +// hide a broken marker. + +pyodide.FS.mkdirTree("/wheels"); +pyodide.mountNodeFS("/wheels", wheelsDir); +for (const wheel of wheels) { + console.log(`micropip install ${wheel}`); + await micropip.install(`emfs:/wheels/${wheel}`); +} + +// The first import is a bare `import weaviate`: it must install the grpc shim itself. +pyodide.runPython(` +import sys +assert "weaviate_client_web" not in sys.modules +import weaviate +assert getattr(sys.modules.get("grpc"), "__weaviate_client_web_shim__", False), \\ + "bare 'import weaviate' did not install the grpc shim" +print("OK bare 'import weaviate' bootstrapped the grpc shim") +`); + +// asyncio.run() is unavailable in Pyodide: load e2e.py, then await main() on Pyodide's loop. +pyodide.runPython(readFileSync(resolve(here, "e2e.py"), "utf8")); +try { + await pyodide.runPythonAsync("await main()"); +} catch (err) { + console.error(err); + process.exit(1); +} diff --git a/ci/pyodide-e2e/units.mjs b/ci/pyodide-e2e/units.mjs new file mode 100644 index 000000000..469a3ccee --- /dev/null +++ b/ci/pyodide-e2e/units.mjs @@ -0,0 +1,117 @@ +// Runs packages/web/tests with pytest inside Pyodide under Node, plus a fresh-interpreter +// bootstrap check. No Weaviate needed. +// Usage: node --experimental-wasm-jspi units.mjs (same wheels as run.mjs) +// JSPI is required: pytest runs async tests through run_until_complete (stack switching). +import { readdirSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { loadPyodide } from "pyodide"; + +if (!process.argv[2]) { + console.error("usage: node units.mjs "); + process.exit(2); +} +const wheelsDir = resolve(process.argv[2]); +const here = dirname(fileURLToPath(import.meta.url)); + +const wheels = readdirSync(wheelsDir) + .filter((f) => f.endsWith(".whl")) + .sort(); // installs weaviate_client before weaviate_client_web, which depends on it +const prefixes = ["weaviate_client-", "weaviate_client_web-"]; +if ( + wheels.length !== 2 || + !prefixes.every((p) => wheels.some((w) => w.startsWith(p))) +) { + console.error( + `expected exactly one weaviate_client-*.whl and one weaviate_client_web-*.whl in ${wheelsDir}, found: ${JSON.stringify(wheels)}`, + ); + process.exit(2); +} + +// Fresh interpreter with micropip ready and the wheels dir mounted. +async function freshPyodide() { + const pyodide = await loadPyodide({ enableRunUntilComplete: true }); + await pyodide.loadPackage("micropip"); + const micropip = pyodide.pyimport("micropip"); + pyodide.FS.mkdirTree("/wheels"); + pyodide.mountNodeFS("/wheels", wheelsDir); + for (const wheel of wheels) { + await micropip.install(`emfs:/wheels/${wheel}`); + } + return pyodide; +} + +// --- bootstrap scenario: needs a clean import state, so its own interpreter -------- + +{ + const pyodide = await freshPyodide(); + try { + pyodide.runPython(` +import sys +assert "weaviate_client_web" not in sys.modules +import weaviate # the ONLY weaviate-side import: must bootstrap the companion +assert "weaviate_client_web" in sys.modules, "hook did not import the companion" +import weaviate_client_web +import grpc +import httpx +assert weaviate_client_web.is_installed() +assert weaviate_client_web.is_fetch_transport_installed() +assert getattr(grpc, "__weaviate_client_web_shim__", False) is True +assert getattr( + httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False +) is True +`); + console.log("OK scenario: bare 'import weaviate' bootstraps the companion"); + } catch (err) { + console.error("FAIL scenario: bare 'import weaviate' bootstraps the companion"); + console.error(err); + process.exit(1); + } +} + +// --- the pytest suite -------------------------------------------------------------- + +const testsDir = resolve(here, "../../packages/web/tests"); +const pyodide = await freshPyodide(); +console.log( + `pyodide ${pyodide.version} / python ${pyodide.runPython("import sys; sys.version.split()[0]")}`, +); +const micropip = pyodide.pyimport("micropip"); +// pytest-asyncio 1.x (asyncio.Runner) fails under JSPI; 0.25.3 needs pytest<9, so pin +// both from PyPI. +await micropip.install(["pytest==8.4.2", "pytest-asyncio==0.25.3"]); +pyodide.FS.mkdirTree("/units"); +pyodide.mountNodeFS("/units", testsDir); + +// pytest.main is synchronous; entering through callPromising() lets the async tests +// stack-switch (run_until_complete) instead of failing with "Cannot stack switch". +const runPytest = pyodide.runPython(` +import sys +sys.dont_write_bytecode = True # /units is the host checkout: no __pycache__ in it + +import pytest + +def _run(): + return int(pytest.main([ + "-v", + "-p", "no:cacheprovider", # no .pytest_cache in the host checkout either + "-o", "asyncio_mode=auto", + "-o", "asyncio_default_fixture_loop_scope=function", + "/units", + ])) + +_run +`); +let exitCode; +try { + exitCode = await runPytest.callPromising(); +} catch (err) { + console.error(err); + process.exit(1); +} +// Any nonzero pytest exit code fails the run — including 5, "no tests collected". +console.log(`pytest exit code: ${exitCode}`); +// The interpreters loaded above keep live handles on the Node event loop, so the +// process does not exit on its own. +process.exit(exitCode === 0 ? 0 : 1); diff --git a/packages/web/README.md b/packages/web/README.md new file mode 100644 index 000000000..2331127b2 --- /dev/null +++ b/packages/web/README.md @@ -0,0 +1,161 @@ +# weaviate-client-web + +Runs the async [Weaviate Python client](https://github.com/weaviate/weaviate-python-client) +under Pyodide (browser pages, marimo notebooks, Web Workers): gRPC goes over grpc-web and +REST over `fetch`. + +Requires Weaviate ≥ 1.38.3 (the first release serving grpc-web) or a grpc-web transcoder +in front of an older server. Tested on Pyodide 314.0.4 (CPython 3.14). + +## Installation + +Install through the base client's `grpc-web` extra: + +```python +import micropip +await micropip.install("weaviate-client[grpc-web]") +``` + +The extra has a `sys_platform == "emscripten"` marker, so it installs nothing on CPython. +`micropip.install("weaviate-client-web")` also works; it pins `weaviate-client` to its own +version. Both need a `weaviate-client` release that ships the extra; older releases fail to +resolve `grpcio`. + +The package imports `pyodide` at module scope, so it imports only under Pyodide. + +## How it works + +Under Pyodide there is no `grpcio` wheel. Importing this package puts a pure-Python `grpc` +shim in `sys.modules`, and `GrpcWebChannel` sends unary RPCs as grpc-web POSTs through +`pyodide.http.pyfetch`. Call metadata (API key, OIDC bearer) becomes fetch headers. + +Under Pyodide the connect helpers send gRPC to the REST endpoint (same host, port and TLS) +under `/v1/grpc-web`. As in the TypeScript `@weaviate/web` client, native gRPC is not +available there, so this endpoint is not configurable through the helpers. + +To use a grpc-web transcoder on another endpoint (Envoy, +[vanguard](https://github.com/connectrpc/vanguard-go)), for example in front of +Weaviate < 1.38.3, build the connection parameters yourself: + +```python +from weaviate import WeaviateAsyncClient +from weaviate.connect import ConnectionParams + +client = WeaviateAsyncClient( + ConnectionParams.from_params( + http_host="weaviate.example.com", http_port=443, http_secure=True, + grpc_host="transcoder.example.com", grpc_port=443, grpc_secure=True, + # add grpc_path_prefix="/prefix" if the transcoder is not at the root + ) +) +``` + +REST goes through the package's own `fetch`-based httpx transport, including on Pyodide +builds whose httpx has one (that one fails on body-less HEAD/204 responses). + +## Usage + +With this package installed, `import weaviate` imports it first, under Pyodide only. If it +is missing, `import weaviate` raises an ImportError naming the extra. Against +Weaviate ≥ 1.38.3: + +```python +import weaviate + +client = weaviate.use_async_with_local(port=8080) +await client.connect() # runs the gRPC health check over grpc-web +collection = client.collections.get("Article") +await collection.query.near_text("hello", limit=3) +``` + +```python +client = weaviate.use_async_with_weaviate_cloud( + cluster_url="rAnD0mD1g1t5.something.weaviate.cloud", + auth_credentials=weaviate.classes.init.Auth.api_key("my-api-key"), +) +``` + +Weaviate Cloud: browser use requires **Allow all CORS origins** in the cluster's settings +in the Weaviate Cloud console (takes a few minutes to apply). Until it applies, the client +fails at its first REST call with a `Failed to fetch` connection error. + +`use_async_with_custom()` still requires `grpc_host`/`grpc_port`/`grpc_secure` — Python +cannot drop required parameters on one platform the way TypeScript drops them from a +type. Pass the HTTP values; anything else is overridden with them and warned about +(`Con006`), so a browser client never silently points somewhere it cannot reach. + +```python +client = weaviate.use_async_with_custom( + http_host="localhost", http_port=8080, http_secure=False, + grpc_host="localhost", grpc_port=8080, grpc_secure=False, # = the HTTP endpoint +) +``` + +Pass `headers={...}` / `auth_credentials=...` as usual for API keys, OIDC or Weaviate Cloud. +Importing `weaviate_client_web` before `weaviate` is equivalent. + +## Supported / unsupported + +| Feature | Kind | Status | +|----------------------------------------------------------|-----------------|--------| +| Search, Aggregate, TenantsGet, BatchObjects, BatchDelete | unary gRPC | Yes, over grpc-web | +| Health check (`/grpc.health.v1.Health/Check`) | unary gRPC | Yes, on `connect()` over grpc-web | +| REST (`is_ready`, config, `/batch/references`, …) | REST | Yes, via the package's own fetch transport | +| API-key auth (`Auth.api_key`) | header | Yes | +| OIDC auth (`client_credentials` / `client_password` / `bearer_token`) | REST | Untested: token refresh runs on an asyncio task (no threads) | +| Bulk insert: `collection.data.insert_many()` | unary gRPC | Yes, the bulk path under Pyodide | +| `batch.stream()` / `batch.experimental()` (BatchStream) | bidi streaming | No: grpc-web has no bidirectional streaming; raises at once, use `insert_many()` | +| `batch.dynamic()` / `fixed_size()` / `rate_limit()` | sync-client API | No: sync client only | +| Embedded Weaviate (`use_async_with_embedded`) | subprocess | No: raises "not supported under WebAssembly/Pyodide" | +| Synchronous client | — | No: async only | + +## Configuration not honored in the browser + +`fetch` manages connections itself, so several knobs are accepted but have no effect +under Pyodide: + +- `AdditionalConfig.proxies` / `trust_env` proxy environment variables (the browser + cannot proxy fetch requests per-client), +- connection-pool sizing and `session_pool_max_retries`, +- `GrpcConfig.credentials` (custom CA bundles — the browser's trust store decides TLS), +- `GrpcConfig.channel_options`, including `grpc.max_send_message_length` / + `grpc.max_receive_message_length` (only `grpc-web.path_prefix` is consumed). The + practical message-size ceiling is the server's `grpcMaxMessageSize` (reported by + `/v1/meta`); exceeding it surfaces as `RESOURCE_EXHAUSTED`, +- `Proxies.grpc` / `GRPC_PROXY`. + +## CORS requirements (browsers) + +Self-hosted Weaviate ≥ 1.38.3 (unchanged through 1.40.0-rc.1) answers CORS for its +`/v1/grpc-web` endpoint as follows (for Weaviate Cloud, see [Usage](#usage)): + +- allowed origins come from `CORS_ALLOW_ORIGIN` (default `*`); +- allowed request headers are `X-Grpc-Web`, `X-User-Agent`, `Grpc-Timeout`, + `Connect-Protocol-Version`, `Connect-Timeout-Ms` and `X-Weaviate-Client`, plus everything + in `CORS_ALLOW_HEADERS`, whose default already covers `Content-Type`, `Authorization`, + `X-Weaviate-Cluster-Url` and the vendor `*-Api-Key` headers; +- exposed response headers are `Grpc-Status`, `Grpc-Message` and + `Grpc-Status-Details-Bin`. + +A custom `headers={...}` entry outside that list must be added to `CORS_ALLOW_HEADERS`, +or the browser's preflight rejects the request. A proxy or grpc-web transcoder in front of +Weaviate must allow the same origins and request headers and expose the same response +headers; without `grpc-status, grpc-message` exposed, trailers-only error responses (e.g. +a bad API key) are reported as `INTERNAL: grpc-web response contained no message frame` +instead of the real error. + +In the browser a CORS-blocked request is indistinguishable from a network failure +(`TypeError: Failed to fetch`), and is retried as UNAVAILABLE. + +## Testing + +Because the package imports `pyodide` at module scope, its unit tests run inside +Pyodide. From the repository root: + +```sh +python -m build --wheel --outdir dist . +python -m build --wheel --outdir dist packages/web +npm install --prefix ci/pyodide-e2e +node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist # pytest unit suite, no Weaviate needed +node ci/pyodide-e2e/run.mjs dist # e2e suite, needs a running Weaviate (see ci/) +``` diff --git a/packages/web/pyproject.toml b/packages/web/pyproject.toml new file mode 100644 index 000000000..0f4872e18 --- /dev/null +++ b/packages/web/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=65", "setuptools_scm[toml] >6.2", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "weaviate-client-web" +description = "grpc-web / WASM (Pyodide) transport for the Weaviate Python client" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "BSD-3-Clause" } +authors = [{ name = "Weaviate", email = "hello@weaviate.io" }] +keywords = ["weaviate", "grpc-web", "pyodide", "wasm", "emscripten"] +# Version comes from the repository's git tags via setuptools_scm (root below), so every +# build carries the same version as weaviate-client; CI asserts the built wheels match. +# Dependencies are computed in setup.py: the weaviate-client requirement is pinned to +# that same version at build time, so mismatched pairs cannot resolve at install time. +dynamic = ["version", "dependencies"] + +[project.urls] +Source = "https://github.com/weaviate/weaviate-python-client" +Tracker = "https://github.com/weaviate/weaviate-python-client/issues" + +[tool.setuptools_scm] +root = "../.." + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +weaviate_client_web = ["py.typed"] diff --git a/packages/web/setup.py b/packages/web/setup.py new file mode 100644 index 000000000..181715c9e --- /dev/null +++ b/packages/web/setup.py @@ -0,0 +1,19 @@ +"""Pins weaviate-client== at build time (the version comes from git tags). + +The packages share private contracts (the "HTTP " details that weaviate.exceptions +matches, and the grpc-web constants _channel imports from weaviate.connect.base), so every +release tag must publish both. +""" + +from setuptools import setup +from setuptools_scm import get_version + +version = get_version(root="../..", relative_to=__file__) + +setup( + install_requires=[ + f"weaviate-client=={version}", + # Pyodide's bundled httpx build omits anyio, but authlib imports it directly. + 'anyio ; sys_platform == "emscripten"', + ] +) diff --git a/packages/web/src/weaviate_client_web/__init__.py b/packages/web/src/weaviate_client_web/__init__.py new file mode 100644 index 000000000..1ef01eedf --- /dev/null +++ b/packages/web/src/weaviate_client_web/__init__.py @@ -0,0 +1,41 @@ +"""grpc-web and fetch transports for the Weaviate Python client under Pyodide. + +Importing this package (``import weaviate`` does so under Emscripten) installs a pure-Python +``grpc`` shim and a fetch-based httpx transport. It imports only under Pyodide and supports +async clients only. +""" + +import os +import sys + +from ._channel import GrpcWebChannel, set_sender +from ._httpx_fetch import ( + install_fetch_transport, + is_fetch_transport_installed, + uninstall_fetch_transport, +) +from ._shim import StatusCode, install, is_installed + +__all__ = [ + "install", + "is_installed", + "install_fetch_transport", + "uninstall_fetch_transport", + "is_fetch_transport_installed", + "set_sender", + "GrpcWebChannel", + "StatusCode", +] + + +def _bootstrap() -> None: + if sys.platform == "emscripten": + # upb may be missing under Pyodide; set before protobuf is imported (setdefault + # keeps a user override). + os.environ.setdefault("PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", "python") + install() + # httpcore needs sockets, so REST also goes through fetch. + install_fetch_transport() + + +_bootstrap() diff --git a/packages/web/src/weaviate_client_web/_channel.py b/packages/web/src/weaviate_client_web/_channel.py new file mode 100644 index 000000000..e5fbbe852 --- /dev/null +++ b/packages/web/src/weaviate_client_web/_channel.py @@ -0,0 +1,428 @@ +"""grpc-web channel: the grpc.aio channel methods the client uses. + +unary_unary POSTs grpc-web through a pluggable sender; stream_stream (BatchStream) raises. +""" + +import asyncio +import base64 +import math +import sys +import urllib.parse +from typing import Any, Callable, Dict, List, Optional + +from ._framing import TruncatedFrameError, UnknownFrameFlagError, encode_message, split_response +from ._sender import Sender, pyfetch_sender +from ._shim import AioChannel, AioRpcError, StatusCode, status_from_int + +_default_sender: Sender = pyfetch_sender + + +def set_sender(sender: Sender) -> None: + """Override the default async sender used by new channels (tests).""" + global _default_sender + _default_sender = sender + + +def get_sender() -> Sender: + return _default_sender + + +# grpc-timeout is at most 8 digits plus a unit; anything longer is rejected by the server. +_GRPC_TIMEOUT_MAX = 100_000_000 + + +def _encode_timeout(seconds: Optional[float]) -> Optional[str]: + """Encode seconds as a grpc-timeout value: at most 8 digits, unit m/S/M, rounded up. + + Returns None (no deadline) for None, non-finite values and anything beyond 99,999,999 + minutes. Never uses H: vanguard rejects values above 8H. + """ + if seconds is None or not math.isfinite(seconds): + return None + if seconds / 60 >= _GRPC_TIMEOUT_MAX: + # past the minute range there is no encodable deadline; checked before the + # multiplication below, which overflows to infinity for huge finite values + return None + for amount, unit in ((seconds * 1000, "m"), (seconds, "S"), (seconds / 60, "M")): + value = max(1, math.ceil(amount)) + if value < _GRPC_TIMEOUT_MAX: + return f"{value}{unit}" + return None + + +def _fold_metadata(headers: Dict[str, str], metadata: Any) -> None: + """Fold gRPC call metadata (``[(key, value), ...]``) into fetch headers. + + Binary ``-bin`` keys are base64-encoded as grpc-web requires. + """ + if not metadata: + return + for key, value in metadata: + name = key.lower() + if name.endswith("-bin"): + raw = value if isinstance(value, (bytes, bytearray)) else str(value).encode() + text = base64.b64encode(raw).decode("ascii") + else: + text = value if isinstance(value, str) else str(value) + # This path bypasses h11/grpcio's header validation, so keep their defence here. + if any(c in name or c in text for c in ("\r", "\n", "\0")): + raise ValueError(f"Illegal character in gRPC metadata {name!r}") + headers[name] = text + + +def _header_lookup(headers: Dict[str, str], name: str) -> Optional[str]: + target = name.lower() + for key, value in headers.items(): + if key.lower() == target: + return value + return None + + +class _UnaryUnaryMultiCallable: + """Awaitable multicallable bound by ``WeaviateStub.__init__``. + + Called as ``await mc(request, metadata=..., timeout=...)`` (and, for the health + check, as ``mc(request, timeout=...)`` with no metadata). + """ + + def __init__( + self, + channel: "GrpcWebChannel", + path: str, + request_serializer: Callable[[Any], bytes], + response_deserializer: Callable[[bytes], Any], + ) -> None: + self._channel = channel + self._path = path + self._serialize = request_serializer + self._deserialize = response_deserializer + + async def __call__( + self, + request: Any, + *, + metadata: Any = None, + timeout: Optional[float] = None, + credentials: Any = None, + wait_for_ready: Any = None, + compression: Any = None, + ) -> Any: + payload = self._serialize(request) + return await self._channel._unary(self._path, payload, self._deserialize, metadata, timeout) + + +class _UnsupportedStreamMultiCallable: + """Placeholder for ``stream_stream`` (bidirectional streaming). + + Calling it raises immediately, before the ``async for`` in ``connect/v4.py`` begins + iterating. + """ + + def __init__(self, path: str) -> None: + self._path = path + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + # batch.dynamic()/fixed_size()/rate_limit() are sync-only, so not suggested here. + raise RuntimeError( + f"Bidirectional streaming RPC {self._path!r} (server-side batching / " + "BatchStream) is not supported over grpc-web/fetch. Use " + "collection.data.insert_many() instead of batch.stream()." + ) + + +class GrpcWebChannel(AioChannel): + """grpc.aio channel that sends unary RPCs as grpc-web.""" + + def __init__( + self, + target: Optional[str], + secure: bool, + options: Any = None, + path_prefix: str = "", + sender: Optional[Sender] = None, + ) -> None: + if not target: + raise ValueError("GrpcWebChannel requires a target (host:port)") + scheme = "https" if secure else "http" + self._base_url = f"{scheme}://{target}" + # Normalize to a single leading slash and no trailing slash; "" == native path. + cleaned = (path_prefix or "").strip("/") + self._path_prefix = f"/{cleaned}" if cleaned else "" + self._sender: Sender = sender or get_sender() + + def unary_unary( + self, + method: str, + request_serializer: Callable[[Any], bytes], + response_deserializer: Callable[[bytes], Any], + _registered_method: bool = False, + ) -> _UnaryUnaryMultiCallable: + return _UnaryUnaryMultiCallable(self, method, request_serializer, response_deserializer) + + def stream_stream( + self, + method: str, + request_serializer: Callable[[Any], bytes], + response_deserializer: Callable[[bytes], Any], + _registered_method: bool = False, + ) -> _UnsupportedStreamMultiCallable: + return _UnsupportedStreamMultiCallable(method) + + async def close(self, grace: Optional[float] = None) -> None: + # Nothing to tear down: each call is an independent fetch. + return None + + async def _unary( + self, + path: str, + payload: bytes, + deserialize: Callable[[bytes], Any], + metadata: Any, + timeout: Optional[float], + ) -> Any: + headers: Dict[str, str] = {} + _fold_metadata(headers, metadata) + # Set after folding: additional_headers reaches RPCs as call metadata, and a + # caller's Content-Type/accept must never replace the grpc-web protocol fields. + headers.update( + { + "content-type": "application/grpc-web+proto", + "accept": "application/grpc-web+proto", + "x-grpc-web": "1", + "x-user-agent": "weaviate-client-web", + } + ) + grpc_timeout = _encode_timeout(timeout) + if grpc_timeout is None: + timeout = None # None / non-finite: no deadline, server- or client-side + else: + headers["grpc-timeout"] = grpc_timeout + + url = self._base_url + self._path_prefix + path + framed = encode_message(payload) + + # Send. Enforce a client-side deadline (the grpc-timeout header is server-side + # only; pyfetch ignores its timeout arg, so without this a stalled request could + # hang forever). Any transport/parse failure is surfaced as AioRpcError; the only + # non-gRPC error a caller can see is the ValueError from metadata validation + # above, raised before any I/O (as native grpcio does). + try: + send = self._sender(url, headers, framed, timeout) + if timeout is not None: + status, resp_headers, body = await asyncio.wait_for(send, timeout) + else: + status, resp_headers, body = await send + except AioRpcError: + raise + except asyncio.TimeoutError as exc: + raise AioRpcError( + code=StatusCode.DEADLINE_EXCEEDED, + details=f"grpc-web request to {path} timed out after {timeout}s", + ) from exc + except Exception as exc: # network/transport failure -> retryable UNAVAILABLE + # str() of transport errors can be empty (e.g. httpx.ConnectError) — always + # include the exception type so failures stay diagnosable + detail = f"{type(exc).__name__}: {exc}" if str(exc) else repr(exc) + details = f"grpc-web transport error for {path}: {detail}" + if not self._path_prefix and sys.platform == "emscripten": + details += " " + _no_path_prefix_hint() + raise AioRpcError(code=StatusCode.UNAVAILABLE, details=details) from exc + + try: + return self._handle_response(status, resp_headers, body, deserialize, url) + except AioRpcError: + raise + except Exception as exc: # malformed framing / status / payload + raise AioRpcError( + code=StatusCode.INTERNAL, + details=f"malformed grpc-web response for {path}: {exc}", + ) from exc + + @staticmethod + def _handle_response( + http_status: int, + resp_headers: Dict[str, str], + body: bytes, + deserialize: Callable[[bytes], Any], + url: str = "", + ) -> Any: + # Real error responses carry non-grpc-web bodies (404 JSON, proxy HTML); keep + # status, URL and body in the error. + messages: List[bytes] = [] + trailers: Dict[str, str] = {} + frame_error: Optional[BaseException] = None + if body: + try: + messages, trailers = split_response(body) + except Exception as exc: + frame_error = exc + + raw_status = trailers.get("grpc-status") + if raw_status is None: + raw_status = _header_lookup(resp_headers, "grpc-status") + raw_message = ( + trailers.get("grpc-message") or _header_lookup(resp_headers, "grpc-message") or "" + ) + message = urllib.parse.unquote(raw_message) + + if raw_status is None: + # no grpc-status anywhere and either a non-200 or a body that is not + # grpc-web framing: a gRPC service did not answer this request + if http_status != 200 or frame_error is not None: + raise _frame_error_to_rpc(http_status, url, body, frame_error) + if messages: + # Every grpc-web unary response must carry a grpc-status (trailer frame + # or header); a proxy that drops the trailer must not read as success. + raise AioRpcError( + code=StatusCode.INTERNAL, + details="grpc-web response missing grpc-status trailers", + ) + code = StatusCode.OK + else: + code = status_from_int(int(raw_status)) + + if code is not StatusCode.OK: + raise AioRpcError(code=code, details=message) + if frame_error is not None: + # grpc-status said OK but the body will not parse — report what actually + # came back rather than a bare "no message frame". + raise _frame_error_to_rpc(http_status, url, body, frame_error) + if len(messages) > 1: + raise AioRpcError( + code=StatusCode.INTERNAL, + details=f"unary grpc-web response carried {len(messages)} message frames", + ) + if not messages: + details = "grpc-web response contained no message frame" + if raw_status is None: + # HTTP 200, no body frames, and no grpc-status anywhere: the classic + # signature of a trailers-only error response whose grpc-status / + # grpc-message headers were stripped by CORS in the browser. + details += ( + " and no grpc-status was visible. If this is a cross-origin browser " + "request, configure the grpc-web proxy to send " + "'Access-Control-Expose-Headers: grpc-status, grpc-message' so " + "trailers-only error responses are readable." + ) + raise AioRpcError(code=StatusCode.INTERNAL, details=details) + return deserialize(messages[0]) + + +_BODY_EXCERPT_LIMIT = 200 + + +def _body_excerpt(body: bytes, limit: int = _BODY_EXCERPT_LIMIT) -> str: + """Render a short, printable, one-line excerpt of a response body for error details. + + The body here is whatever a server or proxy sent — JSON, HTML, or binary — so decode + leniently and drop non-printables: building an error detail must never itself raise. + """ + if not body: + return "" + text = body[:limit].decode("utf-8", "replace") + text = " ".join("".join(ch if ch.isprintable() else " " for ch in text).split()) + if not text: + return f"<{len(body)} non-printable bytes>" + return text + ("..." if len(body) > limit else "") + + +def _no_path_prefix_hint() -> str: + # Lazy import: this module is imported while ``weaviate/__init__`` is still + # bootstrapping the shim under Emscripten. + from weaviate.connect.base import GRPC_WEB_MIN_SERVER_VERSION, GRPC_WEB_SERVER_PATH_PREFIX + + return ( + "(no grpc_path_prefix set — under WebAssembly the connect helpers route gRPC to " + f"the REST endpoint under '{GRPC_WEB_SERVER_PATH_PREFIX}' by themselves, so use " + "one of them; hand-built ConnectionParams must set " + f"grpc_path_prefix='{GRPC_WEB_SERVER_PATH_PREFIX}' for Weaviate >= " + f"{GRPC_WEB_MIN_SERVER_VERSION}, or point grpc_host/grpc_port at a grpc-web " + "transcoder)" + ) + + +def _frame_error_to_rpc( + http_status: int, url: str, body: bytes, frame_error: Optional[BaseException] +) -> AioRpcError: + """Choose the error for a body that did not parse as grpc-web frames.""" + if http_status != 200 or isinstance(frame_error, (UnknownFrameFlagError, TruncatedFrameError)): + return _non_grpc_web_error(http_status, url, body, frame_error) + # Well-formed grpc-web up to the point of failure: a grpc-web endpoint answered but + # broke the protocol (compressed frame, message after trailer, …). + return AioRpcError( + code=StatusCode.INTERNAL, + details=f"malformed grpc-web response from {url or ''}: {frame_error}", + ) + + +def _non_grpc_web_error( + http_status: int, + url: str, + body: bytes, + frame_error: Optional[BaseException] = None, +) -> AioRpcError: + """Error for a response that is not usable grpc-web. + + Details start with "HTTP ", then the URL and a body excerpt. + """ + truncated = isinstance(frame_error, TruncatedFrameError) + what = "not a grpc-web response" + if http_status == 200 and frame_error is not None: + if truncated: + what = f"the grpc-web body is truncated ({frame_error})" + else: + what = f"the body is not grpc-web framing ({frame_error})" + # the base client keys its "wrong path / server too old" diagnosis on this + # "HTTP " text (weaviate.exceptions.WeaviateGRPCUnavailableError) + parts = [f"HTTP {http_status} from {url or ''}: {what}."] + + if http_status == 404: + # either cause is possible; the channel does not know the server version + parts.append( + "The grpc-web endpoint does not exist at that path: either this Weaviate " + "server predates 1.38.3, the first release to serve grpc-web natively, or " + "the configured grpc-web path prefix is wrong for the proxy in front of it. " + "Weaviate's native prefix is '/v1/grpc-web'." + ) + elif http_status == 405: + # a 405 comes only from an existing HTTP route: the prefix points at one + parts.append( + "An HTTP route answered instead of the grpc-web endpoint (method not " + "allowed): the configured grpc-web path prefix is wrong. Weaviate's native " + "prefix is '/v1/grpc-web'." + ) + elif http_status in (502, 503, 504): + parts.append("Weaviate or the proxy in front of it is unavailable.") + elif http_status == 200 and truncated: + parts.append( + "The response was cut short — a proxy or browser buffering limit, or the " + "connection dropped mid-response." + ) + elif http_status == 200: + parts.append( + "Something other than a grpc-web endpoint answered — typically a proxy " + "error page or a single-page-app catch-all route serving index.html. Check " + "the grpc-web path prefix (Weaviate's native prefix is '/v1/grpc-web')." + ) + parts.append(f"Response body: {_body_excerpt(body)}") + + code = StatusCode.INTERNAL if http_status == 200 else _status_from_http(http_status) + return AioRpcError(code=code, details=" ".join(parts)) + + +def _status_from_http(http_status: int) -> StatusCode: + """Map an HTTP status to gRPC per the grpc-web spec. + + Adds 405 -> UNIMPLEMENTED: an HTTP route answered, so the path is wrong, as for 404. + """ + return { + 400: StatusCode.INTERNAL, + 401: StatusCode.UNAUTHENTICATED, + 403: StatusCode.PERMISSION_DENIED, + 404: StatusCode.UNIMPLEMENTED, + 405: StatusCode.UNIMPLEMENTED, + 429: StatusCode.UNAVAILABLE, + 502: StatusCode.UNAVAILABLE, + 503: StatusCode.UNAVAILABLE, + 504: StatusCode.UNAVAILABLE, + }.get(http_status, StatusCode.UNKNOWN) diff --git a/packages/web/src/weaviate_client_web/_framing.py b/packages/web/src/weaviate_client_web/_framing.py new file mode 100644 index 000000000..8ee2e61b9 --- /dev/null +++ b/packages/web/src/weaviate_client_web/_framing.py @@ -0,0 +1,96 @@ +"""grpc-web binary framing (application/grpc-web+proto). + +Each frame is a 1-byte flag, a 4-byte big-endian length and the payload. Flag 0x80 marks +the trailer frame (an HTTP/1-style header block); 0x01 (compressed) is not supported. +""" + +import struct +from typing import Dict, Iterator, List, Tuple + +_FLAG_TRAILER = 0x80 +_FLAG_COMPRESSED = 0x01 +_KNOWN_FLAGS = _FLAG_TRAILER | _FLAG_COMPRESSED +_HEADER = struct.Struct(">BI") # 1 flag byte + 4-byte big-endian length + + +class FrameError(ValueError): + """The body is not a well-formed grpc-web response.""" + + +class UnknownFrameFlagError(FrameError): + """A flag byte outside the grpc-web set: the body is not grpc-web framing (JSON, HTML, …).""" + + +class TruncatedFrameError(FrameError): + """The body ends before the length its frame header announces.""" + + +def encode_message(payload: bytes) -> bytes: + """Frame a single (uncompressed) protobuf payload for sending.""" + return _HEADER.pack(0x00, len(payload)) + payload + + +def iter_frames(buf: bytes) -> Iterator[Tuple[int, bytes]]: + """Yield ``(flag, payload)`` for each frame in a grpc-web response body.""" + off, n = 0, len(buf) + while off < n: + # Validate the flag before the length so a text body ('{', '<') is reported as + # non-grpc-web rather than as a truncated frame with a garbage length. + flag = buf[off] + if flag & ~_KNOWN_FLAGS: + raise UnknownFrameFlagError(f"unknown grpc-web frame flag 0x{flag:02x} at byte {off}") + if off + 5 > n: + raise TruncatedFrameError(f"truncated grpc-web frame header at byte {off}") + _, length = _HEADER.unpack_from(buf, off) + off += 5 + if off + length > n: + raise TruncatedFrameError( + f"truncated grpc-web frame: header announces {length} bytes, {n - off} remain" + ) + yield flag, buf[off : off + length] + off += length + + +def parse_trailers(raw: bytes) -> Dict[str, str]: + """Parse a trailer payload into a lower-cased dict; accepts CRLF or LF line ends. + + Undecodable bytes are replaced, so an odd grpc-message never drops grpc-status. + """ + out: Dict[str, str] = {} + for line in raw.split(b"\n"): + line = line.rstrip(b"\r") + if not line: + continue + key, _, value = line.partition(b":") + name = key.strip().decode("utf-8", "replace").lower() + out[name] = value.strip().decode("utf-8", "replace") + return out + + +def split_response(body: bytes) -> Tuple[List[bytes], Dict[str, str]]: + """Split a grpc-web body into message payloads and trailers. + + Rejects a second trailer frame, which could overwrite an error grpc-status. + """ + messages: List[bytes] = [] + trailers: Dict[str, str] = {} + seen_trailer = False + for flag, payload in iter_frames(body): + if flag & _FLAG_TRAILER: + if flag & _FLAG_COMPRESSED: + raise FrameError( + "compressed grpc-web trailer frames are not supported by this transport" + ) + if seen_trailer: + raise FrameError("second trailer frame in a grpc-web response") + trailers = parse_trailers(payload) + seen_trailer = True + elif flag & _FLAG_COMPRESSED: + raise FrameError( + "compressed grpc-web message frames are not supported by this transport" + ) + elif seen_trailer: + raise FrameError("message frame after the trailer frame") + else: + messages.append(payload) + return messages, trailers diff --git a/packages/web/src/weaviate_client_web/_httpx_fetch.py b/packages/web/src/weaviate_client_web/_httpx_fetch.py new file mode 100644 index 000000000..dcf337f7f --- /dev/null +++ b/packages/web/src/weaviate_client_web/_httpx_fetch.py @@ -0,0 +1,176 @@ +"""fetch-based httpx.AsyncHTTPTransport for Pyodide, where httpcore has no sockets. + +Replaces Pyodide's bundled jsfetch transport, which fails on body-less HEAD/204 responses +and ignores read timeouts. Differs from native httpx: fetch follows redirects itself, +multi-value headers are folded, and bodies are fully buffered. +""" + +import math +import sys +from typing import Callable, Dict, Optional + +import httpx +from pyodide.http import pyfetch # type: ignore[import-not-found] + +_installed = False +_original_handle_async_request: Optional[Callable] = None + +# Hop-by-hop / connection-managed headers that the browser's fetch controls itself. +# Browsers silently drop forbidden headers, but Node's undici (behind pyfetch in the +# Node-based test harness) rejects some of them outright, so strip them before handing off. +_FETCH_MANAGED_HEADERS = { + "host", + "connection", + "accept-encoding", + "content-length", + "transfer-encoding", +} + +# fetch has already decoded the body; passing content-encoding/length through would make +# httpx decode it again. +_FETCH_DECODED_RESPONSE_HEADERS = { + "content-encoding", + "content-length", +} + +_TIMEOUT_HINTS = ("timeout", "timed out", "abort") + +# JS timers take a signed 32-bit millisecond delay; anything larger overflows and fires +# immediately, so a huge timeout would abort every request at once. +_MAX_ABORT_SIGNAL_MS = 2**31 - 1 + + +async def _read_request_body(request: httpx.Request) -> bytes: + try: + return request.content + except httpx.RequestNotRead: + return await request.aread() + + +def _pick_timeout(request: httpx.Request) -> Optional[float]: + """The request deadline: httpx's ``read`` timeout only (None = no deadline). + + ``connect`` and ``pool`` have no meaning under fetch. + """ + timeouts = request.extensions.get("timeout") or {} + return timeouts.get("read") + + +def _abort_signal_ms(timeout: Optional[float]) -> Optional[int]: + """Milliseconds for an abort timer, or None for no deadline (None, negative, non-finite). + + Zero is immediate, as in httpx; positive values round up; capped at 2**31-1. + """ + if timeout is None or not math.isfinite(timeout) or timeout < 0: + return None + if timeout >= _MAX_ABORT_SIGNAL_MS / 1000: + # compared before the multiplication below, which overflows to infinity for + # huge finite values + return _MAX_ABORT_SIGNAL_MS + return min(math.ceil(timeout * 1000), _MAX_ABORT_SIGNAL_MS) + + +def _map_fetch_error( + e: BaseException, request: httpx.Request, deadline_set: bool +) -> httpx.TransportError: + """Map a pyfetch OSError (network, DNS, CORS, CSP, abort) to an httpx error. + + httpx.ReadTimeout if our deadline fired, else httpx.ConnectError. + """ + msg = str(e) or repr(e) + if deadline_set and any(hint in msg.lower() for hint in _TIMEOUT_HINTS): + return httpx.ReadTimeout(msg, request=request) + return httpx.ConnectError(msg, request=request) + + +def _validate_header(name: str, value: str) -> None: + # h11 normally rejects CR/LF/NUL in headers; this transport bypasses h11. + if any(c in name or c in value for c in ("\r", "\n", "\0")): + raise httpx.LocalProtocolError(f"Illegal character in header {name!r}") + + +async def _fetch_handle_async_request( + self: httpx.AsyncHTTPTransport, request: httpx.Request +) -> httpx.Response: + headers: Dict[str, str] = {} + for k, v in request.headers.items(): + if k.lower() in _FETCH_MANAGED_HEADERS: + continue + _validate_header(k, v) + headers[k] = v + kwargs: Dict[str, object] = {} + body = await _read_request_body(request) + if body: + # fetch rejects GET/HEAD requests that carry a body + kwargs["body"] = body + + deadline_set = False + deadline_ms = _abort_signal_ms(_pick_timeout(request)) + if deadline_ms is not None: + try: + from js import AbortSignal # type: ignore[import-not-found] + + kwargs["signal"] = AbortSignal.timeout(deadline_ms) + deadline_set = True + except Exception: # pragma: no cover - AbortSignal.timeout availability varies + pass + + try: + response = await pyfetch(str(request.url), method=request.method, headers=headers, **kwargs) + # A body-less response (HEAD, 204) reads as b"": fetch resolves a null body to + # an empty ArrayBuffer. + data = await response.bytes() + except OSError as e: # incl. pyodide.http.AbortError + raise _map_fetch_error(e, request, deadline_set) from e + + try: + resp_headers = { + k: v + for k, v in dict(response.headers).items() + if k.lower() not in _FETCH_DECODED_RESPONSE_HEADERS + } + except Exception: # pragma: no cover - header shape varies across Pyodide versions + resp_headers = {} + # Hand httpx an unread stream, as its own transports do: the client reads it and + # only then stamps ``response.elapsed``, which the batch-references path relies on. + return httpx.Response( + status_code=int(response.status), + headers=resp_headers, + stream=httpx.ByteStream(data), + request=request, + ) + + +# marker for detecting the patched method (tests, ci/pyodide-e2e) +_fetch_handle_async_request.__weaviate_fetch_shim__ = True # type: ignore[attr-defined] + + +def install_fetch_transport() -> None: + """Patch ``httpx.AsyncHTTPTransport`` to send requests through ``fetch``. + + Installs only under Emscripten (elsewhere httpx's own socket transports work and + must be left in place). Idempotent. + """ + global _installed, _original_handle_async_request + if _installed: + return + if sys.platform != "emscripten": + return + _original_handle_async_request = httpx.AsyncHTTPTransport.handle_async_request + httpx.AsyncHTTPTransport.handle_async_request = _fetch_handle_async_request # type: ignore[method-assign] + _installed = True + + +def uninstall_fetch_transport() -> None: + """Restore the original ``httpx.AsyncHTTPTransport`` behaviour. No-op if not installed.""" + global _installed, _original_handle_async_request + if not _installed: + return + assert _original_handle_async_request is not None + httpx.AsyncHTTPTransport.handle_async_request = _original_handle_async_request # type: ignore[method-assign] + _original_handle_async_request = None + _installed = False + + +def is_fetch_transport_installed() -> bool: + return _installed diff --git a/packages/web/src/weaviate_client_web/_sender.py b/packages/web/src/weaviate_client_web/_sender.py new file mode 100644 index 000000000..472ada397 --- /dev/null +++ b/packages/web/src/weaviate_client_web/_sender.py @@ -0,0 +1,32 @@ +"""HTTP senders for the grpc-web transport. + +A *sender* is ``async def sender(url, headers, body, timeout) -> (status, headers, body)``. +The default uses ``pyodide.http.pyfetch``; tests inject one with +:func:`weaviate_client_web.set_sender`. +""" + +from typing import Awaitable, Callable, Dict, Optional, Tuple + +from pyodide.http import pyfetch # type: ignore[import-not-found] + +Sender = Callable[ + [str, Dict[str, str], bytes, Optional[float]], + Awaitable[Tuple[int, Dict[str, str], bytes]], +] + + +async def pyfetch_sender( + url: str, headers: Dict[str, str], body: bytes, timeout: Optional[float] +) -> Tuple[int, Dict[str, str], bytes]: + """Default browser sender. + + ``pyfetch`` has no timeout parameter of its own; the call deadline is enforced by + ``GrpcWebChannel._unary`` via ``asyncio.wait_for``. + """ + response = await pyfetch(url, method="POST", headers=headers, body=body) + data = await response.bytes() + try: + resp_headers = dict(response.headers) + except Exception: # pragma: no cover - header shape varies across Pyodide versions + resp_headers = {} + return int(response.status), resp_headers, data diff --git a/packages/web/src/weaviate_client_web/_shim.py b/packages/web/src/weaviate_client_web/_shim.py new file mode 100644 index 000000000..e8c7aa768 --- /dev/null +++ b/packages/web/src/weaviate_client_web/_shim.py @@ -0,0 +1,247 @@ +"""Pure-Python stand-in for the grpc API that weaviate-client imports (Pyodide has no grpcio). + +Installed into sys.modules as grpc, grpc.aio, grpc._utilities, grpc.aio._typing and +grpc.experimental before weaviate is imported. +""" + +import enum +import sys +import types +from typing import Any, Optional + +# grpc.__version__ under the shim; kept equal to weaviate.proto.v1._GRPCIO_FALLBACK_VERSION. +FAKE_GRPC_VERSION = "1.72.1" + +_SHIM_MARKER = "__weaviate_client_web_shim__" + + +class StatusCode(enum.Enum): + """Mirror of ``grpc.StatusCode``; ``value`` is grpcio's ``(int, str)`` tuple.""" + + OK = (0, "ok") + CANCELLED = (1, "cancelled") + UNKNOWN = (2, "unknown") + INVALID_ARGUMENT = (3, "invalid argument") + DEADLINE_EXCEEDED = (4, "deadline exceeded") + NOT_FOUND = (5, "not found") + ALREADY_EXISTS = (6, "already exists") + PERMISSION_DENIED = (7, "permission denied") + RESOURCE_EXHAUSTED = (8, "resource exhausted") + FAILED_PRECONDITION = (9, "failed precondition") + ABORTED = (10, "aborted") + OUT_OF_RANGE = (11, "out of range") + UNIMPLEMENTED = (12, "unimplemented") + INTERNAL = (13, "internal") + UNAVAILABLE = (14, "unavailable") + DATA_LOSS = (15, "data loss") + UNAUTHENTICATED = (16, "unauthenticated") + + +_BY_NUMBER = {member.value[0]: member for member in StatusCode} + + +def status_from_int(code: int) -> StatusCode: + """Map a numeric grpc-status to a :class:`StatusCode` (``UNKNOWN`` if unmapped).""" + return _BY_NUMBER.get(code, StatusCode.UNKNOWN) + + +class RpcError(Exception): + """Stand-in for ``grpc.RpcError`` (imported by ``retry.py``).""" + + +class Call: + """Stand-in for ``grpc.Call`` (imported by ``exceptions.py`` / ``retry.py``). + + Only used for ``isinstance``/type-import purposes; the async-only grpc-web path raises + :class:`AioRpcError`, never a sync ``Call``. + """ + + def code(self) -> StatusCode: # pragma: no cover - never instantiated + raise NotImplementedError + + def details(self) -> str: # pragma: no cover + raise NotImplementedError + + +class AioRpcError(RpcError): + """Stand-in for ``grpc.aio.AioRpcError``. + + Raised by the grpc-web multicallable on a non-OK status; exposes the same + ``code()`` / ``details()`` surface the client uses. + """ + + def __init__( + self, + code: StatusCode, + initial_metadata: Any = None, + trailing_metadata: Any = None, + details: str = "", + debug_error_string: Optional[str] = None, + ) -> None: + self._code = code + self._details = details + self._initial_metadata = initial_metadata + self._trailing_metadata = trailing_metadata + self._debug_error_string = debug_error_string + super().__init__(f"") + + def code(self) -> StatusCode: + return self._code + + def details(self) -> str: + return self._details + + def initial_metadata(self) -> Any: + return self._initial_metadata + + def trailing_metadata(self) -> Any: + return self._trailing_metadata + + def debug_error_string(self) -> Optional[str]: + return self._debug_error_string + + +class StreamStreamCall: + """Stand-in for ``grpc.aio.StreamStreamCall`` (imported as a type by ``connect/v4.py``).""" + + +class ChannelCredentials: + """Stand-in for ``grpc.ChannelCredentials`` (imported by ``config.py``).""" + + +def ssl_channel_credentials(*_args: Any, **_kwargs: Any) -> ChannelCredentials: + return ChannelCredentials() + + +class SyncChannel: + """Stand-in for ``grpc.Channel``; never instantiated (the sync factories raise).""" + + +class AioChannel: + """Stand-in for ``grpc.aio.Channel``; ``GrpcWebChannel`` subclasses it.""" + + +def first_version_is_lower(_version: str, _other: str) -> bool: + """Stand-in for ``grpc._utilities.first_version_is_lower``. + + Returning ``False`` makes the v6300 stub's import-time version gate + (``weaviate_pb2_grpc.py``) pass. + """ + return False + + +_ASYNC_ONLY_MESSAGE = ( + "weaviate-client-web provides an asynchronous-only gRPC transport under " + "WebAssembly/Pyodide. Use an async client (weaviate.use_async_with_local / " + "use_async_with_weaviate_cloud / use_async_with_custom, or WeaviateAsyncClient); " + "the synchronous client is not supported in the browser." +) + + +def _sync_channel_unsupported(*_args: Any, **_kwargs: Any) -> "AioChannel": + raise RuntimeError(_ASYNC_ONLY_MESSAGE) + + +def _path_prefix_from_options(options: Any) -> str: + """Extract the ``("grpc-web.path_prefix", prefix)`` channel option, or "" if absent.""" + for item in options or (): + if isinstance(item, (tuple, list)) and len(item) == 2 and item[0] == "grpc-web.path_prefix": + return item[1] or "" + return "" + + +def _aio_secure_channel( + target: Optional[str] = None, credentials: Any = None, options: Any = None, **_kw: Any +) -> AioChannel: + from ._channel import GrpcWebChannel + + return GrpcWebChannel( + target=target, + secure=True, + options=options, + path_prefix=_path_prefix_from_options(options), + ) + + +def _aio_insecure_channel( + target: Optional[str] = None, options: Any = None, **_kw: Any +) -> AioChannel: + from ._channel import GrpcWebChannel + + return GrpcWebChannel( + target=target, + secure=False, + options=options, + path_prefix=_path_prefix_from_options(options), + ) + + +def _noop(*_args: Any, **_kwargs: Any) -> None: + """Stand-in for server-side registration helpers that *_pb2_grpc imports but never calls.""" + return None + + +def is_installed() -> bool: + return getattr(sys.modules.get("grpc"), _SHIM_MARKER, False) is True + + +def install() -> bool: + """Install the shim as ``grpc`` and its submodules under Emscripten; no-op elsewhere. + + Returns ``True`` if the shim is in place. + """ + if sys.platform != "emscripten": + return False + if is_installed(): + return True + + # __dict__ assignment keeps type checkers quiet on these synthesized modules + utilities = types.ModuleType("grpc._utilities") + utilities.__dict__["first_version_is_lower"] = first_version_is_lower + + experimental = types.ModuleType("grpc.experimental") + experimental.__dict__.update(unary_unary=_noop, stream_stream=_noop) + + aio_typing = types.ModuleType("grpc.aio._typing") + aio_typing.__dict__["ChannelArgumentType"] = Any + + aio = types.ModuleType("grpc.aio") + aio.__dict__.update( + Channel=AioChannel, + AioRpcError=AioRpcError, + StreamStreamCall=StreamStreamCall, + secure_channel=_aio_secure_channel, + insecure_channel=_aio_insecure_channel, + _typing=aio_typing, + ) + + grpc_mod = types.ModuleType("grpc") + grpc_mod.__dict__.update( + { + "__version__": FAKE_GRPC_VERSION, + _SHIM_MARKER: True, + "StatusCode": StatusCode, + "RpcError": RpcError, + "Call": Call, + "Channel": SyncChannel, + "ChannelCredentials": ChannelCredentials, + "ssl_channel_credentials": ssl_channel_credentials, + "secure_channel": _sync_channel_unsupported, + "insecure_channel": _sync_channel_unsupported, + "unary_unary_rpc_method_handler": _noop, + "stream_stream_rpc_method_handler": _noop, + "unary_stream_rpc_method_handler": _noop, + "stream_unary_rpc_method_handler": _noop, + "method_handlers_generic_handler": _noop, + "_utilities": utilities, + "experimental": experimental, + "aio": aio, + } + ) + + sys.modules["grpc"] = grpc_mod + sys.modules["grpc._utilities"] = utilities + sys.modules["grpc.experimental"] = experimental + sys.modules["grpc.aio"] = aio + sys.modules["grpc.aio._typing"] = aio_typing + return True diff --git a/packages/web/src/weaviate_client_web/py.typed b/packages/web/src/weaviate_client_web/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/packages/web/tests/conftest.py b/packages/web/tests/conftest.py new file mode 100644 index 000000000..6546c53ba --- /dev/null +++ b/packages/web/tests/conftest.py @@ -0,0 +1,9 @@ +"""Skip collection off Pyodide: these modules import pyodide. + +Run them with: node --experimental-wasm-jspi ci/pyodide-e2e/units.mjs dist +""" + +import sys + +if sys.platform != "emscripten": + collect_ignore_glob = ["*.py"] diff --git a/packages/web/tests/test_framing.py b/packages/web/tests/test_framing.py new file mode 100644 index 000000000..d2f0e745d --- /dev/null +++ b/packages/web/tests/test_framing.py @@ -0,0 +1,133 @@ +"""grpc-web framing tests. Run by pytest inside Pyodide via ``ci/pyodide-e2e/units.mjs``.""" + +import struct + +import pytest + +from weaviate_client_web._framing import ( + FrameError, + TruncatedFrameError, + UnknownFrameFlagError, + encode_message, + iter_frames, + parse_trailers, + split_response, +) + + +def _frame(payload: bytes, flag: int = 0x00) -> bytes: + return struct.pack(">BI", flag, len(payload)) + payload + + +def test_encode_message_round_trip(): + framed = encode_message(b"hello") + frames = list(iter_frames(framed)) + assert frames == [(0x00, b"hello")] + + +def test_split_response_message_and_trailer(): + body = _frame(b"payload") + _frame(b"grpc-status:0\r\ngrpc-message:\r\n", 0x80) + messages, trailers = split_response(body) + assert messages == [b"payload"] + assert trailers["grpc-status"] == "0" + assert trailers["grpc-message"] == "" + + +def test_split_response_multiple_messages(): + # the splitter returns every message frame; whether more than one is acceptable is + # the channel's decision (a unary RPC rejects it) + body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:0\r\n", 0x80) + messages, trailers = split_response(body) + assert messages == [b"a", b"bb"] + assert trailers["grpc-status"] == "0" + + +def test_split_response_message_after_trailer_raises(): + body = _frame(b"a") + _frame(b"grpc-status:0\r\n", 0x80) + _frame(b"late") + with pytest.raises(FrameError, match="after the trailer"): + split_response(body) + + +def test_split_response_trailers_only(): + body = _frame(b"grpc-status:7\r\ngrpc-message:denied\r\n", 0x80) + messages, trailers = split_response(body) + assert messages == [] + assert trailers == {"grpc-status": "7", "grpc-message": "denied"} + + +def test_parse_trailers_lowercases_keys(): + parsed = parse_trailers(b"Grpc-Status:0\r\nGrpc-Message:ok\r\n") + assert parsed == {"grpc-status": "0", "grpc-message": "ok"} + + +def test_parse_trailers_keeps_status_when_message_is_not_ascii(): + # raw UTF-8 in grpc-message must not lose grpc-status + parsed = parse_trailers("grpc-status:5\r\ngrpc-message:Café not found\r\n".encode("utf-8")) + assert parsed["grpc-status"] == "5" + assert parsed["grpc-message"] == "Café not found" + + +def test_parse_trailers_keeps_status_when_message_is_invalid_utf8(): + # latin-1 (or any non-UTF-8) bytes must degrade to replacement chars, not an error + parsed = parse_trailers(b"grpc-status:9\r\ngrpc-message:tenant caf\xe9 is COLD\r\n") + assert parsed["grpc-status"] == "9" + assert parsed["grpc-message"].startswith("tenant caf") + + +def test_split_response_survives_non_ascii_trailer(): + body = _frame("grpc-status:7\r\ngrpc-message:accès refusé\r\n".encode("utf-8"), 0x80) + messages, trailers = split_response(body) + assert messages == [] + assert trailers["grpc-status"] == "7" + + +def test_parse_trailers_accepts_lf_only_lines(): + parsed = parse_trailers(b"grpc-status:0\ngrpc-message:ok\n") + assert parsed == {"grpc-status": "0", "grpc-message": "ok"} + + +def test_parse_trailers_keeps_status_when_a_key_is_not_ascii(): + # one odd key from a proxy must not throw away the whole block + parsed = parse_trailers("x-café:1\r\ngrpc-status:0\r\n".encode("utf-8")) + assert parsed["grpc-status"] == "0" + assert parsed["x-café"] == "1" + + +def test_truncated_frame_raises(): + framed = encode_message(b"hello")[:-2] + with pytest.raises(TruncatedFrameError): + list(iter_frames(framed)) + with pytest.raises(TruncatedFrameError): + list(iter_frames(b"\x00\x00\x00")) # shorter than one frame header + + +@pytest.mark.parametrize("first_byte", [b"{", b"<", b"\x02", b"\x40", b"\xff"]) +def test_unknown_frame_flag_raises(first_byte): + # a JSON / HTML body, or a flag bit this transport does not know + body = first_byte + b"\x00\x00\x00\x01x" + with pytest.raises(UnknownFrameFlagError, match="unknown grpc-web frame flag"): + list(iter_frames(body)) + + +def test_frame_errors_are_value_errors(): + assert issubclass(TruncatedFrameError, ValueError) + assert issubclass(UnknownFrameFlagError, ValueError) + + +def test_compressed_message_frame_rejected(): + body = _frame(b"x", 0x01) + with pytest.raises(FrameError, match="compressed"): + split_response(body) + + +def test_second_trailer_frame_rejected(): + # a second trailer could overwrite an error grpc-status; the spec allows one + body = _frame(b"a") + _frame(b"grpc-status:7\r\n", 0x80) + _frame(b"grpc-status:0\r\n", 0x80) + with pytest.raises(FrameError, match="second trailer frame"): + split_response(body) + + +def test_compressed_trailer_frame_rejected(): + body = _frame(b"grpc-status:0\r\n", 0x81) + with pytest.raises(FrameError, match="compressed"): + split_response(body) diff --git a/packages/web/tests/test_httpx_fetch.py b/packages/web/tests/test_httpx_fetch.py new file mode 100644 index 000000000..fe6adb2be --- /dev/null +++ b/packages/web/tests/test_httpx_fetch.py @@ -0,0 +1,434 @@ +"""Tests for _httpx_fetch. Fakes patch _httpx_fetch.pyfetch and sys.modules["js"]. + +The install tests at the bottom use the real bootstrap. +""" + +import sys +import types +from typing import Any, Dict, List, Optional + +import httpx +import pytest + +import weaviate_client_web +import weaviate_client_web._httpx_fetch as _httpx_fetch +from weaviate_client_web._httpx_fetch import _MAX_ABORT_SIGNAL_MS, _abort_signal_ms + + +class FakeFetchResponse: + def __init__( + self, status: int = 200, headers: Optional[Any] = None, body: Optional[bytes] = b"" + ): + self.status = status + self.headers: Any = headers or {} + self._body = body + + async def bytes(self) -> bytes: # noqa: A003 - mirrors pyodide's FetchResponse API + # a null JS body (HEAD, 204) resolves to an empty ArrayBuffer, i.e. b"" + return b"" if self._body is None else self._body + + +class FakePyfetch: + def __init__(self, response: Optional[FakeFetchResponse] = None): + self.response = response or FakeFetchResponse() + self.calls: List[Dict[str, Any]] = [] + + async def __call__(self, url: str, **kwargs: Any) -> FakeFetchResponse: + self.calls.append({"url": url, **kwargs}) + return self.response + + +@pytest.fixture +def fake_pyfetch(monkeypatch) -> FakePyfetch: + fetch = FakePyfetch() + monkeypatch.setattr(_httpx_fetch, "pyfetch", fetch) + return fetch + + +class _AbortSignalRecorder: + def __init__(self): + self.timeouts: List[int] = [] + + def timeout(self, ms: int): + self.timeouts.append(ms) + return f"signal-{ms}" + + +@pytest.fixture +def fake_abort_signal(monkeypatch) -> _AbortSignalRecorder: + recorder = _AbortSignalRecorder() + js_mod = types.ModuleType("js") + js_mod.AbortSignal = recorder # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "js", js_mod) + return recorder + + +@pytest.fixture +def missing_js(monkeypatch): + # sys.modules["js"] = None makes ``import js`` raise ImportError — the closest + # in-process stand-in for an environment without the js bridge + monkeypatch.setitem(sys.modules, "js", None) + + +async def _handle(request: httpx.Request) -> httpx.Response: + # self is unused by the handler implementation; a bare transport instance suffices + transport = httpx.AsyncHTTPTransport.__new__(httpx.AsyncHTTPTransport) + response = await _httpx_fetch._fetch_handle_async_request(transport, request) + await response.aread() # httpx.AsyncClient reads non-streamed responses the same way + return response + + +class _FetchTransport(httpx.AsyncBaseTransport): + """Route an ``httpx.AsyncClient`` through the handler without touching global state.""" + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + return await _httpx_fetch._fetch_handle_async_request(self, request) # type: ignore[arg-type] + + +async def _via_client(method: str, url: str, **kwargs: Any) -> httpx.Response: + async with httpx.AsyncClient(transport=_FetchTransport()) as client: + return await client.request(method, url, **kwargs) + + +async def test_basic_get_round_trip(fake_pyfetch): + fake_pyfetch.response = FakeFetchResponse( + status=200, headers={"content-type": "application/json"}, body=b'{"version": "1.30.0"}' + ) + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + + assert response.status_code == 200 + assert response.json() == {"version": "1.30.0"} + assert response.headers["content-type"] == "application/json" + call = fake_pyfetch.calls[0] + assert call["url"] == "http://h:8080/v1/meta" + assert call["method"] == "GET" + + +async def test_response_has_request_attached_for_raise_for_status(fake_pyfetch): + fake_pyfetch.response = FakeFetchResponse(status=404, body=b"") + response = await _handle(httpx.Request("GET", "http://h:8080/v1/schema/Nope")) + with pytest.raises(httpx.HTTPStatusError): + response.raise_for_status() + + +@pytest.mark.parametrize("status", [204, 404]) +async def test_head_response_without_body_yields_empty_content(fake_pyfetch, status): + # data.exists() / tenants.exists() are HEAD requests answered 204/404 with a null + # body; the transport must hand httpx an empty response, not fail on the missing body + fake_pyfetch.response = FakeFetchResponse(status=status, body=None) + response = await _handle(httpx.Request("HEAD", "http://h:8080/v1/objects/A/uuid")) + assert response.status_code == status + assert response.content == b"" + assert "body" not in fake_pyfetch.calls[0] + + +async def test_delete_204_without_body_yields_empty_content(fake_pyfetch): + # data.delete_by_id() / reference_delete() are answered 204 with a null body + fake_pyfetch.response = FakeFetchResponse(status=204, body=None) + response = await _handle(httpx.Request("DELETE", "http://h:8080/v1/objects/A/uuid")) + assert response.status_code == 204 + assert response.content == b"" + + +async def test_body_less_response_through_async_client(fake_pyfetch): + # the full httpx.AsyncClient path (stream wrapping + read) on a body-less response + fake_pyfetch.response = FakeFetchResponse(status=204, body=None) + response = await _via_client("HEAD", "http://h:8080/v1/objects/A/uuid") + assert response.status_code == 204 + assert response.content == b"" + + +async def test_response_through_async_client_exposes_elapsed_and_content(fake_pyfetch): + # the batch-references path reads ``res.elapsed``, which httpx only sets after it + # has read/closed a stream-backed response; a pre-loaded body never gets one + payload = b'[{"result": {"status": "SUCCESS"}}]' + fake_pyfetch.response = FakeFetchResponse(status=200, body=payload) + response = await _via_client("POST", "http://h:8080/v1/batch/references", content=b"[]") + assert response.content == payload + assert response.json() == [{"result": {"status": "SUCCESS"}}] + assert response.elapsed.total_seconds() >= 0 + + +async def test_fetch_managed_request_headers_stripped(fake_pyfetch): + request = httpx.Request( + "POST", + "http://h:8080/v1/objects", + headers={ + "authorization": "Bearer k", + "content-type": "application/json", + "host": "h:8080", + "connection": "keep-alive", + "accept-encoding": "gzip", + "transfer-encoding": "chunked", + }, + content=b"{}", + ) + await _handle(request) + sent = fake_pyfetch.calls[0]["headers"] + assert sent["authorization"] == "Bearer k" + assert sent["content-type"] == "application/json" + for managed in ("host", "connection", "accept-encoding", "content-length", "transfer-encoding"): + assert managed not in sent + + +async def test_get_without_body_omits_body_kwarg(fake_pyfetch): + # fetch rejects GET/HEAD requests that carry a body, so the kwarg must be absent + await _handle(httpx.Request("GET", "http://h:8080/v1/.well-known/ready")) + assert "body" not in fake_pyfetch.calls[0] + + +async def test_post_body_passed(fake_pyfetch): + await _handle(httpx.Request("POST", "http://h:8080/v1/graphql", content=b'{"query": "x"}')) + assert fake_pyfetch.calls[0]["body"] == b'{"query": "x"}' + + +async def test_delete_with_body_passed(fake_pyfetch): + # the REST batch-delete path sends DELETE with a JSON body + await _handle( + httpx.Request("DELETE", "http://h:8080/v1/batch/objects", content=b'{"match": {}}') + ) + assert fake_pyfetch.calls[0]["body"] == b'{"match": {}}' + + +async def test_query_string_preserved_in_url(fake_pyfetch): + await _handle(httpx.Request("GET", "http://h:8080/v1/objects?class=A&limit=10&after=a%20b")) + assert fake_pyfetch.calls[0]["url"] == "http://h:8080/v1/objects?class=A&limit=10&after=a%20b" + + +async def test_content_encoding_stripped_from_response(fake_pyfetch): + # fetch returns decoded bytes; passing content-encoding through would decode twice + fake_pyfetch.response = FakeFetchResponse( + status=200, + headers={"content-encoding": "gzip", "content-length": "23", "x-other": "kept"}, + body=b'{"version": "1.30.0"}', + ) + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert response.json() == {"version": "1.30.0"} + assert "content-encoding" not in response.headers + assert response.headers["x-other"] == "kept" + + +async def test_unreadable_response_headers_tolerated(fake_pyfetch): + class BadHeaders: + def keys(self): + raise TypeError("header shape varies across Pyodide versions") + + fake_pyfetch.response = FakeFetchResponse(status=200, body=b"ok") + fake_pyfetch.response.headers = BadHeaders() + response = await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert response.status_code == 200 + assert response.content == b"ok" + + +def _request_with_timeout(timeouts: Dict[str, Optional[float]]) -> httpx.Request: + request = httpx.Request("GET", "http://h:8080/v1/meta") + request.extensions["timeout"] = timeouts + return request + + +async def test_read_timeout_maps_to_abort_signal_ms(fake_pyfetch, fake_abort_signal): + # mirrors what weaviate's AsyncClient puts in extensions: connect/read/write/pool + await _handle(_request_with_timeout({"connect": 2.0, "read": 30.0, "write": 5.0, "pool": 9.0})) + assert fake_abort_signal.timeouts == [30000] + assert fake_pyfetch.calls[0]["signal"] == "signal-30000" + + +async def test_read_none_means_no_deadline_even_with_pool_and_connect_set( + fake_pyfetch, fake_abort_signal +): + # a non-finite timeout arrives as read=None with pool set; pool must not become the deadline + await _handle(_request_with_timeout({"connect": None, "read": None, "write": None, "pool": 5})) + await _handle(_request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0})) + assert fake_abort_signal.timeouts == [] + assert all("signal" not in c for c in fake_pyfetch.calls) + + +async def test_read_timeout_alone_sets_the_deadline(fake_pyfetch, fake_abort_signal): + await _handle(_request_with_timeout({"connect": None, "read": 7, "write": None, "pool": 5})) + assert fake_abort_signal.timeouts == [7000] + assert fake_pyfetch.calls[0]["signal"] == "signal-7000" + + +async def test_no_timeout_extension_sends_no_signal(fake_pyfetch, fake_abort_signal): + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert fake_abort_signal.timeouts == [] + assert "signal" not in fake_pyfetch.calls[0] + + +async def test_missing_js_module_degrades_to_no_signal(fake_pyfetch, missing_js): + # without the js bridge the AbortSignal import fails; the request must still go out + response = await _handle( + _request_with_timeout({"connect": 2.0, "read": 30.0, "write": None, "pool": None}) + ) + assert response.status_code == 200 + assert "signal" not in fake_pyfetch.calls[0] + + +async def test_zero_timeout_is_an_immediate_deadline(fake_pyfetch, fake_abort_signal): + # read=0 is an immediate deadline, as in httpx; connect is ignored + await _handle(_request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None})) + assert fake_abort_signal.timeouts == [0] + assert fake_pyfetch.calls[0]["signal"] == "signal-0" + + +@pytest.mark.parametrize( + "timeout,expected_ms", + [ + (None, None), + (0, 0), # immediate deadline, matching native httpx read=0 + (-1, None), + (float("inf"), None), + (float("nan"), None), + (0.0001, 1), # rounds up: never an immediate AbortSignal.timeout(0) + (30.0, 30_000), + (1e8, _MAX_ABORT_SIGNAL_MS), + (1e10, _MAX_ABORT_SIGNAL_MS), + (1e308, _MAX_ABORT_SIGNAL_MS), # finite, but *1000 overflows: capped, not an error + ], +) +def test_abort_signal_ms_bounds(timeout, expected_ms): + assert _abort_signal_ms(timeout) == expected_ms + + +async def test_infinite_timeout_sends_no_signal(fake_pyfetch, fake_abort_signal): + # an inf read deadline reaching the transport: no signal, not an OverflowError + await _handle( + _request_with_timeout({"connect": None, "read": float("inf"), "write": None, "pool": 5}) + ) + assert fake_abort_signal.timeouts == [] + assert "signal" not in fake_pyfetch.calls[0] + + +async def test_huge_timeout_is_capped_to_int32_ms(fake_pyfetch, fake_abort_signal): + # setTimeout delays above 2^31-1 ms overflow and fire at once, aborting the request + await _handle( + _request_with_timeout({"connect": None, "read": 1e10, "write": None, "pool": None}) + ) + assert fake_abort_signal.timeouts == [_MAX_ABORT_SIGNAL_MS] + assert fake_pyfetch.calls[0]["signal"] == f"signal-{_MAX_ABORT_SIGNAL_MS}" + + +class RaisingPyfetch: + def __init__(self, exc: BaseException): + self.exc = exc + + async def __call__(self, url: str, **kwargs: Any): + raise self.exc + + +def _install_raising_pyfetch(monkeypatch, exc: BaseException) -> None: + monkeypatch.setattr(_httpx_fetch, "pyfetch", RaisingPyfetch(exc)) + + +async def test_fetch_failure_maps_to_httpx_connect_error(monkeypatch): + # pyodide surfaces JS fetch rejections as OSError; the base client can only classify + # httpx exceptions, so the transport maps OSError to httpx errors + _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) + with pytest.raises(httpx.ConnectError, match="Failed to fetch") as excinfo: + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert isinstance(excinfo.value.__cause__, OSError) + + +async def test_fetch_abort_with_deadline_maps_to_read_timeout(monkeypatch, fake_abort_signal): + # AbortSignal.timeout firing surfaces as an OSError subclass mentioning the abort; + # with a deadline set this must classify as a timeout, not a connection error + _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) + with pytest.raises(httpx.ReadTimeout, match="signal timed out"): + await _handle( + _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) + ) + + +async def test_fetch_failure_with_deadline_but_no_timeout_message_stays_connect_error( + monkeypatch, fake_abort_signal +): + # nearly every weaviate request sets a read deadline; a plain network failure on + # such a request must remain a connection error, not become a timeout + _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) + with pytest.raises(httpx.ConnectError, match="Failed to fetch"): + await _handle( + _request_with_timeout({"connect": None, "read": 30.0, "write": None, "pool": None}) + ) + + +async def test_fetch_abort_without_deadline_stays_connect_error(monkeypatch, missing_js): + # without our deadline (no js bridge -> no signal), an abort is a connection error + _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) + with pytest.raises(httpx.ConnectError): + await _handle( + _request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None}) + ) + + +async def test_empty_oserror_str_keeps_repr_detail(monkeypatch): + _install_raising_pyfetch(monkeypatch, OSError()) + with pytest.raises(httpx.ConnectError) as excinfo: + await _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert "OSError" in str(excinfo.value) + + +async def test_crlf_in_header_value_rejected(fake_pyfetch): + request = httpx.Request( + "GET", "http://h:8080/v1/meta", headers={"x-key": "val\r\nx-injected: evil"} + ) + with pytest.raises(httpx.LocalProtocolError): + await _handle(request) + assert fake_pyfetch.calls == [] + + +# --------------------------------------------------------------------------- +# install semantics, against this interpreter's real installation +# --------------------------------------------------------------------------- + + +def test_bootstrap_installed_fetch_transport(): + # the import at the top ran the real bootstrap, which replaces Pyodide's jsfetch transport + assert weaviate_client_web.is_fetch_transport_installed() + assert ( + getattr(httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False) + is True + ) + + +def test_install_fetch_transport_is_idempotent(): + patched_method = httpx.AsyncHTTPTransport.handle_async_request + weaviate_client_web.install_fetch_transport() + assert httpx.AsyncHTTPTransport.handle_async_request is patched_method + + +def test_sync_transport_left_untouched(): + assert not getattr(httpx.HTTPTransport.handle_request, "__weaviate_fetch_shim__", False) + + +def test_uninstall_restores_original_transport(): + # the module keeps the pre-patch method while installed, so the restore can be + # checked by identity even though the install happened at import time + original = _httpx_fetch._original_handle_async_request + assert original is not None + patched_method = httpx.AsyncHTTPTransport.handle_async_request + try: + weaviate_client_web.uninstall_fetch_transport() + assert not weaviate_client_web.is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is original + assert httpx.AsyncHTTPTransport.handle_async_request is not patched_method + weaviate_client_web.uninstall_fetch_transport() # no-op when not installed + assert httpx.AsyncHTTPTransport.handle_async_request is original + finally: + weaviate_client_web.install_fetch_transport() + assert weaviate_client_web.is_fetch_transport_installed() + assert ( + getattr(httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False) + is True + ) + + +async def test_installed_transport_routes_async_client_through_pyfetch(fake_pyfetch): + # the globally installed transport (no custom transport argument) must reach pyfetch + fake_pyfetch.response = FakeFetchResponse( + status=200, headers={"content-type": "application/json"}, body=b'{"ok": true}' + ) + async with httpx.AsyncClient() as client: + response = await client.get("http://h:8080/v1/meta") + assert response.status_code == 200 + assert response.json() == {"ok": True} + assert fake_pyfetch.calls and fake_pyfetch.calls[0]["url"] == "http://h:8080/v1/meta" diff --git a/packages/web/tests/test_shim_install.py b/packages/web/tests/test_shim_install.py new file mode 100644 index 000000000..710d4f164 --- /dev/null +++ b/packages/web/tests/test_shim_install.py @@ -0,0 +1,75 @@ +"""grpc shim tests; sys.modules["grpc"] is the shim, installed by importing weaviate_client_web.""" + +import struct + +import pytest + +import weaviate_client_web +from weaviate_client_web import GrpcWebChannel, set_sender +from weaviate_client_web._sender import pyfetch_sender +from weaviate_client_web._shim import FAKE_GRPC_VERSION + + +def _frame(payload: bytes, flag: int = 0x00) -> bytes: + return struct.pack(">BI", flag, len(payload)) + payload + + +def test_import_weaviate_under_shim(): + import grpc + + assert weaviate_client_web.is_installed() + assert weaviate_client_web.install() is True # idempotent, reports the shim in place + assert getattr(grpc, "__weaviate_client_web_shim__", False) is True + assert grpc.__version__ == "1.72.1" + assert grpc._utilities.first_version_is_lower("1.0.0", "2.0.0") is False # type: ignore[attr-defined] + from grpc.aio._typing import ChannelArgumentType # noqa: F401 + + import weaviate # noqa: F401 # must not raise even though grpcio is shimmed + from weaviate.proto.v1 import weaviate_pb2_grpc + + ch = GrpcWebChannel("localhost:50051", secure=False) + stub = weaviate_pb2_grpc.WeaviateStub(ch) + assert stub.Search is not None + assert stub.BatchObjects is not None + assert stub.BatchDelete is not None + assert isinstance(ch, grpc.aio.Channel) + + +def test_sync_channel_factory_raises_async_only(): + import grpc + + with pytest.raises(RuntimeError, match="async"): + grpc.insecure_channel("localhost:50051") + + +async def test_real_proto_unary_round_trip_under_shim(): + from weaviate.proto.v1 import tenants_pb2, weaviate_pb2_grpc + + reply = tenants_pb2.TenantsGetReply() + payload = reply.SerializeToString() + body = _frame(payload) + _frame(b"grpc-status:0\r\n", 0x80) + + async def sender(url, headers, body_in, timeout): + assert headers["authorization"] == "Bearer k" + assert url.endswith("/weaviate.v1.Weaviate/TenantsGet") + return 200, {}, body + + set_sender(sender) + try: + ch = GrpcWebChannel("localhost:50051", secure=False) + stub = weaviate_pb2_grpc.WeaviateStub(ch) + res = await stub.TenantsGet( + tenants_pb2.TenantsGetRequest(), + metadata=[("authorization", "Bearer k")], + timeout=5, + ) + assert isinstance(res, tenants_pb2.TenantsGetReply) + finally: + set_sender(pyfetch_sender) + + +def test_fake_grpc_version_matches_base_fallback(): + # grpc.__version__ and the proto-selection fallback describe the same fake grpcio + from weaviate.proto.v1 import _GRPCIO_FALLBACK_VERSION + + assert FAKE_GRPC_VERSION == _GRPCIO_FALLBACK_VERSION diff --git a/packages/web/tests/test_transport.py b/packages/web/tests/test_transport.py new file mode 100644 index 000000000..a9c85bd0c --- /dev/null +++ b/packages/web/tests/test_transport.py @@ -0,0 +1,631 @@ +"""GrpcWebChannel tests through fake senders (no network, no Weaviate).""" + +import asyncio +import struct +import sys +from typing import Dict, List, Optional, Tuple + +import pytest + +from weaviate_client_web import GrpcWebChannel, set_sender +from weaviate_client_web._channel import _body_excerpt, _encode_timeout +from weaviate_client_web._sender import pyfetch_sender +from weaviate_client_web._shim import ( + AioChannel, + AioRpcError, + StatusCode, + _aio_insecure_channel, +) + + +def _frame(payload: bytes, flag: int = 0x00) -> bytes: + return struct.pack(">BI", flag, len(payload)) + payload + + +def _ok_response(payload: bytes) -> bytes: + return _frame(payload) + _frame(b"grpc-status:0\r\n", 0x80) + + +class FakeSender: + def __init__( + self, status: int = 200, headers: Optional[Dict[str, str]] = None, body: bytes = b"" + ): + self.status = status + self.headers = headers or {} + self.body = body + self.calls: List[Tuple[str, Dict[str, str], bytes, Optional[float]]] = [] + + async def __call__(self, url, headers, body, timeout): + self.calls.append((url, headers, body, timeout)) + return self.status, self.headers, self.body + + +def _channel(sender: FakeSender, secure: bool = False) -> GrpcWebChannel: + return GrpcWebChannel("example.com:443", secure=secure, sender=sender) + + +def test_grpcwebchannel_is_grpc_aio_channel(): + assert issubclass(GrpcWebChannel, AioChannel) + assert isinstance(_channel(FakeSender()), AioChannel) + + +async def test_unary_success_round_trip(): + sender = FakeSender(body=_ok_response(b"reply-bytes")) + channel = _channel(sender) + mc = channel.unary_unary( + "/weaviate.v1.Weaviate/Search", + request_serializer=lambda x: x, + response_deserializer=lambda b: b, + _registered_method=True, + ) + + result = await mc(b"request-bytes", metadata=[("authorization", "Bearer k")], timeout=5) + + assert result == b"reply-bytes" + url, headers, body, timeout = sender.calls[0] + assert url == "http://example.com:443/weaviate.v1.Weaviate/Search" + assert body == _frame(b"request-bytes") + assert headers["content-type"] == "application/grpc-web+proto" + assert headers["authorization"] == "Bearer k" + assert headers["grpc-timeout"] == "5000m" + assert timeout == 5 + + +async def test_secure_channel_uses_https(): + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender, secure=True) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + await mc(b"q") + assert sender.calls[0][0].startswith("https://example.com:443/") + + +async def test_health_call_without_metadata(): + sender = FakeSender(body=_ok_response(b"pong")) + channel = _channel(sender) + mc = channel.unary_unary("/grpc.health.v1.Health/Check", lambda x: x, lambda b: b) + # mirrors the health check in connect/v4.py — request + timeout, no metadata + assert await mc(b"ping", timeout=2) == b"pong" + + +async def test_error_trailer_raises_aiorpcerror(): + body = _frame(b"grpc-status:7\r\ngrpc-message:nope\r\n", 0x80) + channel = _channel(FakeSender(body=body)) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q") + assert excinfo.value.code() is StatusCode.PERMISSION_DENIED + assert excinfo.value.code().name == "PERMISSION_DENIED" + assert excinfo.value.details() == "nope" + + +async def test_percent_encoded_grpc_message_decoded(): + body = _frame(b"grpc-status:5\r\ngrpc-message:not%20found\r\n", 0x80) + channel = _channel(FakeSender(body=body)) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q") + assert excinfo.value.details() == "not found" + + +async def test_trailers_only_status_in_http_headers(): + channel = _channel( + FakeSender(status=200, headers={"grpc-status": "16", "grpc-message": "auth"}, body=b"") + ) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q") + assert excinfo.value.code() is StatusCode.UNAUTHENTICATED + + +# --- non-grpc-web responses: bodies captured from real servers and proxies --------- + +# Weaviate's own 404, verbatim from a 1.39.0 server asked for the wrong prefix. +WEAVIATE_404_JSON = ( + b'{"code":404,"message":"path /grpc-web/grpc.health.v1.Health/Check was not found"}' +) +NGINX_502_HTML = ( + b"\r\n502 Bad Gateway\r\n\r\n" + b"

502 Bad Gateway

\r\n
nginx/1.27.3
\r\n" + b"\r\n\r\n" +) +NGINX_404_HTML = ( + b"\r\n404 Not Found\r\n\r\n" + b"

404 Not Found

\r\n
nginx/1.27.3
\r\n" + b"\r\n\r\n" +) +# A single-page app's catch-all route answers 200 with index.html for unknown paths. +SPA_INDEX_HTML = ( + b'\n\n \n My App\n' + b' \n' + b' \n
\n\n' +) + + +async def _details_of(status, body, headers=None, path="/grpc.health.v1.Health/Check"): + """Run one request against a canned HTTP response and return the AioRpcError.""" + channel = _channel(FakeSender(status=status, headers=headers or {}, body=body)) + mc = channel.unary_unary(path, lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q") + return excinfo.value + + +async def test_weaviate_404_json_names_both_candidate_causes(): + # a 404 means the server predates grpc-web or the prefix is wrong; the channel + # cannot tell which + err = await _details_of(404, WEAVIATE_404_JSON, {"content-type": "application/json"}) + details = err.details() + + assert err.code() is StatusCode.UNIMPLEMENTED + assert details.startswith("HTTP 404 ") + assert "/grpc.health.v1.Health/Check" in details # the request path + assert "1.38.3" in details # candidate 1: server too old + assert "path prefix" in details # candidate 2: wrong prefix + assert "/v1/grpc-web" in details # Weaviate's prefix, spelled out + assert "was not found" in details # the server's own explanation + assert "malformed grpc-web response" not in details + + +async def test_nginx_502_maps_to_unavailable_so_the_client_retries(): + # weaviate/retry.py retries only UNAVAILABLE + err = await _details_of(502, NGINX_502_HTML) + assert err.code() is StatusCode.UNAVAILABLE + assert err.details().startswith("HTTP 502 ") + assert "502 Bad Gateway" in err.details() + + +@pytest.mark.parametrize("status", [503, 504]) +async def test_gateway_errors_are_unavailable(status): + err = await _details_of(status, b"upstream down") + assert err.code() is StatusCode.UNAVAILABLE + + +async def test_nginx_404_html_is_reported_as_an_http_404(): + err = await _details_of(404, NGINX_404_HTML) + assert err.code() is StatusCode.UNIMPLEMENTED + assert err.details().startswith("HTTP 404 ") + assert "404 Not Found" in err.details() + assert "malformed grpc-web response" not in err.details() + + +async def test_405_names_the_wrong_prefix(): + # a 405 comes from an existing HTTP route (e.g. /v1/objects): the prefix is wrong + err = await _details_of( + 405, b'{"code":405,"message":"method POST is not allowed, but [GET] are"}' + ) + # UNIMPLEMENTED is what the base client's wrong-path diagnosis keys on for 404/405 + assert err.code() is StatusCode.UNIMPLEMENTED + assert err.details().startswith("HTTP 405 ") + assert "path prefix" in err.details() + assert "/v1/grpc-web" in err.details() + assert "method POST is not allowed" in err.details() + + +async def test_truncated_grpc_web_body_is_reported_as_truncated_not_as_wrong_prefix(): + # a valid frame header with a cut-short payload: the endpoint is grpc-web, so no + # SPA / path-prefix hint + body = _ok_response(b"reply-bytes")[:-6] + err = await _details_of(200, body) + assert err.code() is StatusCode.INTERNAL + assert "truncated" in err.details() + assert "cut short" in err.details() + assert "single-page-app" not in err.details() + assert "path prefix" not in err.details() + + +async def test_spa_html_body_is_not_reported_as_truncated(): + # text bodies decode to an unknown flag byte and a garbage length; they must read as + # "not grpc-web framing", never as a truncated grpc-web body + err = await _details_of(200, SPA_INDEX_HTML) + assert "truncated" not in err.details() + assert "not grpc-web framing" in err.details() + + +async def test_message_frame_after_trailer_is_internal(): + body = _frame(b"a") + _frame(b"grpc-status:0\r\n", 0x80) + _frame(b"late") + err = await _details_of(200, body) + assert err.code() is StatusCode.INTERNAL + assert "malformed grpc-web response" in err.details() + assert "after the trailer" in err.details() + assert "single-page-app" not in err.details() + + +async def test_multiple_message_frames_in_unary_response_is_internal(): + # a unary RPC has exactly one message; silently taking the first would hide a proxy + # or server that streams several + body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:0\r\n", 0x80) + err = await _details_of(200, body) + assert err.code() is StatusCode.INTERNAL + assert "2 message frames" in err.details() + + +async def test_error_status_wins_over_multiple_message_frames(): + body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:5\r\ngrpc-message:gone\r\n", 0x80) + err = await _details_of(200, body) + assert err.code() is StatusCode.NOT_FOUND + assert err.details() == "gone" + + +async def test_spa_fallback_html_200_is_distinguishable_from_a_404(): + # An HTTP 200 serving index.html is the other half of a wrong path prefix: the app's + # catch-all route answers instead of Weaviate. It must not read as malformed framing. + err = await _details_of(200, SPA_INDEX_HTML) + details = err.details() + + assert details.startswith("HTTP 200 ") + assert "" in details + assert "single-page-app" in details # names the actual cause + assert "malformed grpc-web response" not in details + # distinguishable from the 404 case, not the same generic message + assert details != (await _details_of(404, NGINX_404_HTML)).details() + + +async def test_401_json_body_maps_to_unauthenticated(): + err = await _details_of(401, b'{"error":[{"message":"anonymous access not enabled"}]}') + assert err.code() is StatusCode.UNAUTHENTICATED + assert err.details().startswith("HTTP 401 ") + assert "anonymous access not enabled" in err.details() + + +async def test_403_error_body_reaches_details(): + # the response body must reach details() + err = await _details_of(403, b'{"code":403,"message":"forbidden: rbac denied"}') + assert err.code() is StatusCode.PERMISSION_DENIED + assert "forbidden: rbac denied" in err.details() + + +async def test_error_body_excerpt_is_capped(): + err = await _details_of(500, b"E" * 5000) + details = err.details() + assert "EEEE" in details + assert details.endswith("...") + assert len(details) < 600 # the 5000-byte body is excerpted, not pasted in + + +async def test_binary_error_body_does_not_break_the_error(): + # a proxy answering with a binary payload must not raise UnicodeDecodeError while + # the error message is being built + err = await _details_of(502, b"\xff\xfe\x00\x01\x02") + assert err.code() is StatusCode.UNAVAILABLE + assert err.details().startswith("HTTP 502 ") + + +async def test_non_200_with_valid_grpc_web_trailers_still_uses_grpc_status(): + # a grpc-status on a non-200 response wins over the HTTP status + err = await _details_of(500, _frame(b"grpc-status:7\r\ngrpc-message:denied\r\n", 0x80)) + assert err.code() is StatusCode.PERMISSION_DENIED + assert err.details() == "denied" + + +async def test_non_ascii_grpc_message_preserves_the_status(): + # raw UTF-8 grpc-message keeps its status + body = _frame("grpc-status:5\r\ngrpc-message:collection Café not found\r\n".encode(), 0x80) + err = await _details_of(200, body) + assert err.code() is StatusCode.NOT_FOUND + assert "Caf" in err.details() + + +async def test_invalid_utf8_grpc_message_preserves_the_status(): + # latin-1 bytes are not valid UTF-8; the status must still survive + body = _frame(b"grpc-status:9\r\ngrpc-message:tenant caf\xe9 is COLD\r\n", 0x80) + err = await _details_of(200, body) + assert err.code() is StatusCode.FAILED_PRECONDITION + assert "tenant caf" in err.details() + + +async def test_binary_metadata_base64_encoded(): + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + await mc(b"q", metadata=[("trace-bin", b"\x00\x01\x02")]) + assert sender.calls[0][1]["trace-bin"] == "AAEC" + + +def test_stream_stream_raises_clear_error(): + channel = _channel(FakeSender()) + mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) + with pytest.raises(RuntimeError, match="not supported over grpc-web"): + mc(request_iterator=iter([]), timeout=5, metadata=None) + + +async def test_timeout_maps_to_deadline_exceeded(): + async def slow_sender(url, headers, body, timeout): + await asyncio.sleep(0.5) + return 200, {}, _ok_response(b"x") + + channel = GrpcWebChannel("h:1", secure=False, sender=slow_sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q", timeout=0.01) + assert excinfo.value.code() is StatusCode.DEADLINE_EXCEEDED + + +async def test_transport_exception_maps_to_unavailable(): + async def boom(url, headers, body, timeout): + raise ConnectionError("connection refused") + + channel = GrpcWebChannel("h:1", secure=False, sender=boom) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q") + assert excinfo.value.code() is StatusCode.UNAVAILABLE + assert "ConnectionError: connection refused" in str(excinfo.value.details()) + + +async def test_transport_exception_with_empty_str_keeps_type(): + # httpx transport errors commonly stringify to '' — the detail must still name them + async def boom(url, headers, body, timeout): + raise ConnectionError() + + channel = GrpcWebChannel("h:1", secure=False, sender=boom) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q") + assert "ConnectionError" in str(excinfo.value.details()) + + +async def test_empty_ok_response_hints_at_cors_expose_headers(): + # HTTP 200, empty body, no grpc-status anywhere: the shape of a trailers-only error + # whose grpc-status/grpc-message headers were stripped by CORS + channel = _channel(FakeSender(status=200, headers={}, body=b"")) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q") + assert excinfo.value.code() is StatusCode.INTERNAL + assert "Access-Control-Expose-Headers" in str(excinfo.value.details()) + + +async def test_empty_ok_response_with_grpc_status_has_no_cors_hint(): + # grpc-status was visible (status 0, no frames): malformed, not a CORS problem + channel = _channel(FakeSender(status=200, headers={"grpc-status": "0"}, body=b"")) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q") + assert excinfo.value.code() is StatusCode.INTERNAL + assert "Access-Control-Expose-Headers" not in str(excinfo.value.details()) + + +async def test_message_frame_without_grpc_status_is_internal_not_success(): + # a message frame without grpc-status (dropped trailer) is INTERNAL + channel = _channel(FakeSender(status=200, headers={}, body=_frame(b"reply-bytes"))) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q") + assert excinfo.value.code() is StatusCode.INTERNAL + assert "missing grpc-status" in str(excinfo.value.details()) + + +async def test_message_frame_with_grpc_status_header_still_succeeds(): + # trailers-only-in-headers responses (grpc-status as an HTTP header, no trailer + # frame) remain valid per the grpc-web contract + channel = _channel( + FakeSender(status=200, headers={"grpc-status": "0"}, body=_frame(b"reply-bytes")) + ) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + assert await mc(b"q") == b"reply-bytes" + + +def test_stream_stream_error_recommends_insert_many_only(): + channel = _channel(FakeSender()) + mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) + with pytest.raises(RuntimeError) as excinfo: + mc(request_iterator=iter([]), timeout=5, metadata=None) + assert "insert_many" in str(excinfo.value) + for sync_only in ("dynamic", "fixed_size", "rate_limit"): + assert sync_only not in str(excinfo.value) + + +async def test_malformed_frame_maps_to_internal(): + # A 3-byte body cannot contain even a 5-byte frame header -> framing ValueError. + channel = _channel(FakeSender(body=b"\x00\x00\x00")) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q") + assert excinfo.value.code() is StatusCode.INTERNAL + + +async def test_malformed_grpc_status_maps_to_internal(): + body = _frame(b"grpc-status:notanint\r\n", 0x80) + channel = _channel(FakeSender(body=body)) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q") + assert excinfo.value.code() is StatusCode.INTERNAL + + +async def test_grpc_timeout_header_rounds_up(): + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + # 123.4ms must round up to 124ms (never advertise a shorter deadline than requested). + await mc(b"q", timeout=0.1234) + assert sender.calls[0][1]["grpc-timeout"] == "124m" + + +def test_body_excerpt_empty_and_non_printable(): + assert _body_excerpt(b"") == "" + assert _body_excerpt(b"\x00\x01\x02\x7f") == "<4 non-printable bytes>" + assert _body_excerpt(b"ok\x00\x01") == "ok" + + +@pytest.mark.parametrize( + "seconds,expected", + [ + (None, None), + (float("inf"), None), + (float("nan"), None), + (0, "1m"), + (0.1234, "124m"), + (5, "5000m"), + (99_999, "99999000m"), + (100_000, "100000S"), # 1e8 ms would be 9 digits + (1e8, "1666667M"), # 1e8 s would be 9 digits + (1e9, "16666667M"), + (5_999_999_940, "99999999M"), # the largest deadline that still fits in minutes + (1e10, None), # would need hours, which transcoders reject above 8H: no deadline + (1e15, None), + (1e308, None), # finite, but *1000 overflows to infinity: must not raise + ], +) +def test_encode_timeout_stays_within_eight_digits(seconds, expected): + encoded = _encode_timeout(seconds) + assert encoded == expected + if encoded is not None: + assert len(encoded) <= 9 # 8 digits + unit + assert not encoded.endswith("H") + + +async def test_infinite_timeout_sends_no_deadline(): + # Timeout(query=inf): neither a grpc-timeout header nor a client-side wait + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + assert await mc(b"q", timeout=float("inf")) == b"x" + _, headers, _, timeout = sender.calls[0] + assert "grpc-timeout" not in headers + assert timeout is None + + +async def test_huge_timeout_uses_minutes_then_no_deadline(): + # beyond 8 minute digits no grpc-timeout is sent (see _encode_timeout) + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + await mc(b"q", timeout=1e8) + await mc(b"q", timeout=1e9) + await mc(b"q", timeout=1e10) + assert sender.calls[0][1]["grpc-timeout"] == "1666667M" + assert sender.calls[1][1]["grpc-timeout"] == "16666667M" + assert "grpc-timeout" not in sender.calls[2][1] + assert sender.calls[2][3] is None # no client-side wait either + + +@pytest.mark.parametrize("bad", ["val\r\nx-injected: evil", "val\nx", "v\0"]) +async def test_crlf_in_metadata_rejected(bad): + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(ValueError, match="Illegal character"): + await mc(b"q", metadata=[("x-key", bad)]) + with pytest.raises(ValueError, match="Illegal character"): + await mc(b"q", metadata=[("x-key\r\n", "v")]) + assert sender.calls == [] + + +async def _unavailable_details(monkeypatch, path_prefix, platform): + async def boom(url, headers, body, timeout): + raise ConnectionError("Failed to fetch") + + monkeypatch.setattr(sys, "platform", platform) + channel = GrpcWebChannel("h:50051", secure=False, sender=boom, path_prefix=path_prefix) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + await mc(b"q") + assert excinfo.value.code() is StatusCode.UNAVAILABLE + return excinfo.value.details() + + +async def test_unavailable_without_path_prefix_under_emscripten_hints_at_grpc_path_prefix( + monkeypatch, +): + # the connect helpers always set the prefix under Emscripten, so a prefix-less channel + # here means hand-built ConnectionParams; the error must say what to do instead + details = await _unavailable_details(monkeypatch, path_prefix="", platform="emscripten") + assert "grpc_path_prefix='/v1/grpc-web'" in details + assert "1.38.3" in details + assert "connect helpers" in details + + +async def test_unavailable_with_path_prefix_has_no_prefix_hint(monkeypatch): + details = await _unavailable_details( + monkeypatch, path_prefix="/v1/grpc-web", platform="emscripten" + ) + assert "no grpc_path_prefix" not in details + + +async def test_unavailable_without_path_prefix_off_emscripten_has_no_prefix_hint(monkeypatch): + # off Emscripten an empty prefix against a transcoder is the normal configuration + details = await _unavailable_details(monkeypatch, path_prefix="", platform="linux") + assert "no grpc_path_prefix" not in details + + +async def test_close_is_awaitable_noop(): + channel = _channel(FakeSender()) + assert await channel.close() is None + + +async def test_path_prefix_prepended_to_url(): + sender = FakeSender(body=_ok_response(b"r")) + channel = GrpcWebChannel( + "example.com:8090", secure=False, sender=sender, path_prefix="/grpc-web" + ) + mc = channel.unary_unary("/weaviate.v1.Weaviate/Search", lambda x: x, lambda b: b) + await mc(b"q") + assert sender.calls[0][0] == "http://example.com:8090/grpc-web/weaviate.v1.Weaviate/Search" + + +@pytest.mark.parametrize( + "raw,expected_url", + [ + ("grpc-web", "http://h:1/grpc-web/svc/M"), + ("/grpc-web/", "http://h:1/grpc-web/svc/M"), + ("/a/b", "http://h:1/a/b/svc/M"), + ("", "http://h:1/svc/M"), + ], +) +async def test_path_prefix_normalized_in_url(raw, expected_url): + sender = FakeSender(body=_ok_response(b"r")) + channel = GrpcWebChannel("h:1", secure=False, sender=sender, path_prefix=raw) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + await mc(b"q") + assert sender.calls[0][0] == expected_url + + +def test_shim_factory_extracts_path_prefix_option(): + with_prefix = _aio_insecure_channel( + target="h:1", + options=[("grpc.max_send_message_length", 1), ("grpc-web.path_prefix", "/grpc-web")], + ) + assert with_prefix._path_prefix == "/grpc-web" + + without_prefix = _aio_insecure_channel( + target="h:1", options=[("grpc.max_send_message_length", 1)] + ) + assert without_prefix._path_prefix == "" + + +async def test_set_sender_overrides_default(): + sender = FakeSender(body=_ok_response(b"y")) + set_sender(sender) + try: + channel = GrpcWebChannel("h:1", secure=False) # no explicit sender + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + assert await mc(b"q") == b"y" + finally: + # restore the real default so other tests are unaffected + set_sender(pyfetch_sender) + + +async def test_metadata_cannot_replace_protocol_headers(): + # additional_headers reaches RPCs as call metadata; a caller setting Content-Type + # (reasonable for the REST side) must not break the grpc-web framing contract + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + await mc( + b"q", + metadata=[ + ("content-type", "application/json"), + ("accept", "application/json"), + ("x-grpc-web", "0"), + ("x-user-agent", "custom"), + ("x-custom", "kept"), + ], + ) + headers = sender.calls[0][1] + assert headers["content-type"] == "application/grpc-web+proto" + assert headers["accept"] == "application/grpc-web+proto" + assert headers["x-grpc-web"] == "1" + assert headers["x-user-agent"] == "weaviate-client-web" + assert headers["x-custom"] == "kept" diff --git a/proto_test/test_proto.py b/proto_test/test_proto.py index bedbf10c3..e7f1a28b0 100644 --- a/proto_test/test_proto.py +++ b/proto_test/test_proto.py @@ -1,19 +1,111 @@ +import importlib +import pathlib +import re +from importlib.metadata import PackageNotFoundError, version as metadata_version + import pytest -from importlib.metadata import version as metadata_version from packaging import version +# CI installs incompatible grpcio/protobuf pairs; there weaviate.proto.v1 cannot import, so +# the tests below are skipped. This check imports nothing from weaviate. +def _versions_incompatible() -> bool: + """Whether the installed grpcio/protobuf pair makes ``import weaviate.proto.v1`` raise.""" + try: + grpc_ver = version.parse(metadata_version("grpcio")) + pb_ver = version.parse(metadata_version("protobuf")) + except PackageNotFoundError: + return False + return (pb_ver >= version.parse("6.30.0") and grpc_ver < version.parse("1.72.0")) or ( + pb_ver >= version.parse("5.26.1") and grpc_ver < version.parse("1.63.0") + ) + + +_skip_if_incompatible = pytest.mark.skipif( + _versions_incompatible(), + reason="weaviate.proto.v1 cannot be imported with an incompatible grpcio/protobuf " + "pair (CI version-gate matrix); the gate is covered by test_proto_import and the " + "fallback is exercised in every compatible cell", +) + + def test_proto_import(): grpc_ver = version.parse(metadata_version("grpcio")) pb_ver = version.parse(metadata_version("protobuf")) - if (pb_ver >= version.parse("6.30.0") and grpc_ver < version.parse("1.72.0")) or ( pb_ver >= version.parse("5.26.1") and grpc_ver < version.parse("1.63.0") ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception) as e: import weaviate - assert "gRPC incompatibility detected" in str(exc_info.value) + + assert weaviate.version is not None + assert "WeaviateProtobufIncompatibility" in str(e.type) else: import weaviate assert weaviate.version is not None + + +@_skip_if_incompatible +def test_grpcio_metadata_fallback_under_emscripten(monkeypatch): + """Under Emscripten a missing grpcio falls back to _GRPCIO_FALLBACK_VERSION. + + A missing protobuf still raises. + """ + mod = importlib.import_module("weaviate.proto.v1") + + def raises(pkg: str) -> str: + raise PackageNotFoundError(pkg) + + monkeypatch.setattr(mod, "metadata_version", raises) + monkeypatch.setattr("sys.platform", "emscripten") + + assert str(mod.get_version("grpcio")) == "1.72.1" + with pytest.raises(PackageNotFoundError): + mod.get_version("protobuf") + + +@_skip_if_incompatible +def test_grpcio_missing_metadata_raises_off_emscripten(monkeypatch): + """Off Emscripten, missing grpcio metadata surfaces instead of being masked.""" + mod = importlib.import_module("weaviate.proto.v1") + + def raises(pkg: str) -> str: + raise PackageNotFoundError(pkg) + + monkeypatch.setattr(mod, "metadata_version", raises) + monkeypatch.setattr("sys.platform", "linux") + with pytest.raises(PackageNotFoundError): + mod.get_version("grpcio") + + +@_skip_if_incompatible +def test_grpcio_fallback_version_passes_every_vendored_stub_gate(): + """_GRPCIO_FALLBACK_VERSION is at least every vendored stub's GRPC_GENERATED_VERSION.""" + try: + from grpc._utilities import first_version_is_lower + except ImportError: + pytest.skip( + "grpc._utilities.first_version_is_lower is unavailable in this grpcio; " + "newer matrix cells run the comparison" + ) + + fallback = importlib.import_module("weaviate.proto.v1")._GRPCIO_FALLBACK_VERSION + proto_root = pathlib.Path(__file__).resolve().parents[1] / "weaviate" / "proto" / "v1" + stub_files = sorted(proto_root.glob("*/v1/*_pb2_grpc.py")) + assert stub_files, "no vendored *_pb2_grpc.py stubs found" + + gate_pattern = re.compile(r"^GRPC_GENERATED_VERSION = '([^']+)'", re.MULTILINE) + gated = 0 + for stub in stub_files: + match = gate_pattern.search(stub.read_text()) + if match is None: + continue # older codegen (e.g. v4216) emits no version gate + gated += 1 + generated = match.group(1) + assert not first_version_is_lower(fallback, generated), ( + f"{stub.relative_to(proto_root)} requires grpcio>={generated} but " + f"_GRPCIO_FALLBACK_VERSION is {fallback}; bump the fallback (and the " + "grpc-web shim's FAKE_GRPC_VERSION) to match the regenerated stubs" + ) + assert gated > 0, "no stub carried a GRPC_GENERATED_VERSION gate; check the extraction regex" diff --git a/pyrightconfig.json b/pyrightconfig.json index 396d62cfd..61eb3eaa3 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,6 +1,6 @@ { "include": [ - "weaviate", "integration" + "weaviate", "integration", "packages/web/src" ], "exclude": [ diff --git a/setup.cfg b/setup.cfg index 0b5ba855a..26b397613 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,7 @@ install_requires = # When bumping authlib to >=2.0.0, remove the `authlib.jose` deprecation # warning filter implemented in `weaviate/_authlib_compat.py`. pydantic>=2.12.0,<3.0.0 - grpcio>=1.59.5,<1.80.0 + grpcio>=1.59.5,<1.80.0; sys_platform != "emscripten" protobuf>=4.21.6,<7.0.0 packaging>=21.0 python_requires = >=3.10 @@ -48,6 +48,12 @@ python_requires = >=3.10 [options.extras_require] agents = weaviate-agents >=1.0.0, <2.0.0 +grpc-web = + # The Pyodide/WASM grpc-web companion, built from packages/web in this repo. Both + # packages derive their version from the same git tag (setuptools_scm; CI asserts + # the built wheels match). The marker makes the extra a no-op on CPython: the + # companion imports pyodide at module scope and only makes sense under Emscripten. + weaviate-client-web; sys_platform == "emscripten" [options.package_data] # If any package or subpackage contains *.txt, *.rst or *.md files, include them: diff --git a/test/test_connection_params.py b/test/test_connection_params.py new file mode 100644 index 000000000..0be4b87cf --- /dev/null +++ b/test/test_connection_params.py @@ -0,0 +1,188 @@ +import sys + +import pytest +from pydantic import ValidationError + +import weaviate.connect.base as base_mod +from weaviate.connect.base import ConnectionParams +from weaviate.exceptions import WeaviateInvalidInputError + + +def test_same_host_port_raises_without_prefix() -> None: + with pytest.raises(ValidationError, match="must be different"): + ConnectionParams.from_params( + http_host="localhost", + http_port=8090, + http_secure=False, + grpc_host="localhost", + grpc_port=8090, + grpc_secure=False, + ) + + +def test_same_host_port_allowed_with_grpc_web_prefix() -> None: + params = ConnectionParams.from_params( + http_host="localhost", + http_port=8090, + http_secure=False, + grpc_host="localhost", + grpc_port=8090, + grpc_secure=False, + grpc_path_prefix="/grpc-web", + ) + assert params._grpc_web_path_prefix == "/grpc-web" + + +def test_from_url_same_host_port_allowed_with_prefix() -> None: + params = ConnectionParams.from_url( + "http://localhost:8090", grpc_port=8090, grpc_path_prefix="/grpc-web" + ) + assert params._grpc_web_path_prefix == "/grpc-web" + + +@pytest.mark.parametrize( + "raw,expected", + [ + (None, ""), + ("", ""), + ("/", ""), + ("grpc-web", "/grpc-web"), + ("/grpc-web", "/grpc-web"), + ("grpc-web/", "/grpc-web"), + ("/a/b/", "/a/b"), + ], +) +def test_path_prefix_normalization(raw, expected) -> None: + params = ConnectionParams.from_params( + http_host="h", + http_port=8080, + http_secure=False, + grpc_host="g", + grpc_port=50051, + grpc_secure=False, + grpc_path_prefix=raw, + ) + assert params._grpc_web_path_prefix == expected + + +def _grpc_web_params() -> ConnectionParams: + return ConnectionParams.from_params( + http_host="localhost", + http_port=8090, + http_secure=False, + grpc_host="localhost", + grpc_port=8090, + grpc_secure=False, + grpc_path_prefix="/grpc-web", + ) + + +def test_grpc_channel_forwards_path_prefix_option(monkeypatch) -> None: + captured: dict = {} + + def fake_insecure_channel(target, options=None, **kwargs): + captured["target"] = target + captured["options"] = options + return "CHANNEL" + + monkeypatch.setattr(base_mod.grpc.aio, "insecure_channel", fake_insecure_channel) + + channel = _grpc_web_params()._grpc_channel(proxies={}, grpc_msg_size=None, is_async=True) + + assert channel == "CHANNEL" + assert captured["target"] == "localhost:8090" + assert ("grpc-web.path_prefix", "/grpc-web") in captured["options"] + + +def test_grpc_channel_omits_option_without_prefix(monkeypatch) -> None: + captured: dict = {} + + def fake_insecure_channel(target, options=None, **kwargs): + captured["options"] = options + return "CHANNEL" + + monkeypatch.setattr(base_mod.grpc.aio, "insecure_channel", fake_insecure_channel) + + params = ConnectionParams.from_params( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="localhost", + grpc_port=50051, + grpc_secure=False, + ) + params._grpc_channel(proxies={}, grpc_msg_size=None, is_async=True) + + option_keys = [key for key, _ in captured["options"]] + assert "grpc-web.path_prefix" not in option_keys + + +def test_async_client_construction_rejects_prefix_without_shim(monkeypatch) -> None: + # fail at construction with actionable text, not deep inside connect() after the + # OIDC and /v1/meta round trips already succeeded + from weaviate import WeaviateAsyncClient + + monkeypatch.delattr(base_mod.grpc, "__weaviate_client_web_shim__", raising=False) + with pytest.raises(WeaviateInvalidInputError, match="weaviate-client-web"): + WeaviateAsyncClient(_grpc_web_params()) + + +def test_async_client_construction_allows_prefix_with_shim(monkeypatch) -> None: + from weaviate import WeaviateAsyncClient + + monkeypatch.setattr(base_mod.grpc, "__weaviate_client_web_shim__", True, raising=False) + client = WeaviateAsyncClient(_grpc_web_params()) + assert client._connection._connection_params._grpc_web_path_prefix == "/grpc-web" + + +def test_sync_client_construction_rejects_grpc_web_prefix() -> None: + from weaviate import WeaviateClient + + with pytest.raises(WeaviateInvalidInputError, match="async"): + WeaviateClient(_grpc_web_params()) + + +@pytest.mark.parametrize( + "call,expected", + [ + ( + lambda w: w.use_async_with_local(), + { + "http": {"host": "localhost", "port": 8080, "secure": False}, + "grpc": {"host": "localhost", "port": 50051, "secure": False}, + "grpc_path_prefix": None, + }, + ), + ( + lambda w: w.use_async_with_weaviate_cloud("abc.something.weaviate.cloud", None), + { + "http": {"host": "abc.something.weaviate.cloud", "port": 443, "secure": True}, + "grpc": {"host": "grpc-abc.something.weaviate.cloud", "port": 443, "secure": True}, + "grpc_path_prefix": None, + }, + ), + ( + lambda w: w.use_async_with_custom( + http_host="rest.example.com", + http_port=443, + http_secure=True, + grpc_host="grpc.example.com", + grpc_port=443, + grpc_secure=True, + ), + { + "http": {"host": "rest.example.com", "port": 443, "secure": True}, + "grpc": {"host": "grpc.example.com", "port": 443, "secure": True}, + "grpc_path_prefix": None, + }, + ), + ], +) +def test_helper_params_off_emscripten_are_unchanged(call, expected) -> None: + # off Emscripten the helpers use native gRPC: grpc_path_prefix stays None + import weaviate + + assert sys.platform != "emscripten" + params = call(weaviate)._connection._connection_params + assert params.model_dump() == expected + assert params._grpc_web_path_prefix == "" diff --git a/test/test_wasm_compat.py b/test/test_wasm_compat.py new file mode 100644 index 000000000..bcb4b9e8e --- /dev/null +++ b/test/test_wasm_compat.py @@ -0,0 +1,372 @@ +"""CPython tests for the Emscripten guards, grpc-web routing and grpc-web diagnostics.""" + +import asyncio +import pathlib +import subprocess +import sys +import textwrap + +import grpc +import pytest +from grpc.aio import AioRpcError, Metadata + +from weaviate import WeaviateClient +from weaviate.collections.batch.async_ import _BatchBaseAsync +from weaviate.connect.base import ConnectionParams +from weaviate.connect.v4 import _ConnectionBase +from weaviate.embedded import _EmbeddedBase +from weaviate.exceptions import ( + WeaviateBatchStreamError, + WeaviateGRPCUnavailableError, + WeaviateStartUpError, +) +from weaviate.util import _ServerVersion + + +def test_embedded_raises_explicit_error_under_emscripten(monkeypatch) -> None: + monkeypatch.setattr(sys, "platform", "emscripten") + with pytest.raises(WeaviateStartUpError, match="WebAssembly/Pyodide"): + _EmbeddedBase.check_supported_platform() + + +def test_sync_client_construction_raises_async_only_under_emscripten(monkeypatch) -> None: + # fails at construction, not with a ConnectError on the first REST call + monkeypatch.setattr(sys, "platform", "emscripten") + with pytest.raises(WeaviateStartUpError, match="async client"): + WeaviateClient(connection_params=ConnectionParams.from_url("http://localhost:8080", 50051)) + + +def test_batch_stream_fails_fast_when_grpc_web_shim_active(monkeypatch) -> None: + # _start must raise before any background task exists + monkeypatch.setattr(grpc, "__weaviate_client_web_shim__", True, raising=False) + batch = object.__new__(_BatchBaseAsync) # the guard runs before any attribute access + with pytest.raises(WeaviateBatchStreamError, match="insert_many"): + asyncio.run(batch._start()) + + +# --- grpc-web diagnostics ------------------------------------------------------------- + + +def _connection(prefix=None) -> _ConnectionBase: + conn = object.__new__(_ConnectionBase) + conn._client = None + conn._grpc_channel = None + conn._weaviate_version = _ServerVersion.from_string("1.36.0") + conn._connection_params = ConnectionParams.from_url( + "http://localhost:8080", + grpc_port=8080 if prefix else 50051, + grpc_path_prefix=prefix, + ) + return conn + + +def _ping_exception(conn: _ConnectionBase, error: Exception) -> None: + getattr(conn, "_ConnectionBase__handle_ping_exception")(error) # noqa: B009 + + +def test_grpc_web_404_names_the_two_real_causes_and_drops_firewall_advice() -> None: + conn = _connection(prefix="/grpc-web") + error = AioRpcError( + grpc.StatusCode.UNIMPLEMENTED, + Metadata(), + Metadata(), + details="HTTP 404 for /grpc-web/grpc.health.v1.Health/Check: 404 page not found", + ) + with pytest.raises(WeaviateGRPCUnavailableError) as excinfo: + _ping_exception(conn, error) + msg = str(excinfo.value) + + assert "firewall" not in msg + assert "port (localhost:8080) are correct" not in msg + assert "UNIMPLEMENTED" in msg # the real code, not swallowed + assert "HTTP 404 for /grpc-web/grpc.health.v1.Health/Check" in msg # ... and details + assert "/grpc-web" in msg # the prefix that was actually used + assert "1.38.3" in msg # candidate 1: server too old ... + assert "v1.36.0" in msg # ... shown against the observed server version + assert "/v1/grpc-web" in msg # candidate 2: wrong prefix + + +def test_grpc_web_genuine_unimplemented_is_not_diagnosed_as_a_wrong_path() -> None: + # a routed grpc-web endpoint can itself return UNIMPLEMENTED (e.g. the health + # service is missing); only the channel's synthetic HTTP 404/405 means "not routed" + conn = _connection(prefix="/grpc-web") + error = AioRpcError( + grpc.StatusCode.UNIMPLEMENTED, Metadata(), Metadata(), details="Method not implemented" + ) + with pytest.raises(WeaviateGRPCUnavailableError) as excinfo: + _ping_exception(conn, error) + msg = str(excinfo.value) + + assert "too old" not in msg + assert "1.38.3" not in msg + assert "UNIMPLEMENTED" in msg # the real status is still reported + assert "skip_init_checks=True" in msg # the generic grpc-web advice applies + + +def test_grpc_web_non_404_error_still_omits_the_native_port_advice() -> None: + conn = _connection(prefix="/grpc-web") + error = AioRpcError( + grpc.StatusCode.UNAVAILABLE, Metadata(), Metadata(), details="HTTP 502 for /grpc-web/..." + ) + with pytest.raises(WeaviateGRPCUnavailableError) as excinfo: + _ping_exception(conn, error) + msg = str(excinfo.value) + + assert "firewall" not in msg + assert "UNAVAILABLE" in msg + assert "HTTP 502" in msg + assert "skip_init_checks=True" in msg # the still-useful advice is kept + + +def test_native_grpc_message_keeps_its_advice_and_gains_the_real_status() -> None: + conn = _connection() + error = AioRpcError( + grpc.StatusCode.UNAVAILABLE, Metadata(), Metadata(), details="failed to connect" + ) + with pytest.raises(WeaviateGRPCUnavailableError) as excinfo: + _ping_exception(conn, error) + msg = str(excinfo.value) + + # unchanged guidance for native gRPC ... + assert "The gRPC traffic at the specified port is blocked by a firewall." in msg + assert "Please check that the server address and port (localhost:50051) are correct." in msg + # ... plus the call's status and details + assert "UNAVAILABLE" in msg + assert "failed to connect" in msg + + +def test_non_grpc_ping_error_is_still_reported() -> None: + # not every ping failure is an RpcError; those must not lose the generic advice + conn = _connection() + with pytest.raises(WeaviateGRPCUnavailableError) as excinfo: + _ping_exception(conn, ValueError("boom")) + assert "blocked by a firewall" in str(excinfo.value) + + +# --- grpc-web routing under Emscripten ------------------------------------------------ + +GRPC_WEB_PREFIX = "/v1/grpc-web" + + +@pytest.fixture +def emscripten(monkeypatch): + """Fake Emscripten, with the grpc shim marked active. + + Under real Pyodide ``import weaviate`` installs the shim itself; here only the + routing decision is under test, not the environment check that guards it. + """ + import weaviate.connect.base as base_mod + + monkeypatch.setattr(sys, "platform", "emscripten") + monkeypatch.setattr(base_mod.grpc, "__weaviate_client_web_shim__", True, raising=False) + + +def _params(client) -> ConnectionParams: + return client._connection._connection_params + + +def _assert_grpc_rides_rest(client) -> None: + params = _params(client) + assert params.grpc.model_dump() == params.http.model_dump() + assert params._grpc_web_path_prefix == GRPC_WEB_PREFIX + assert params._grpc_target == f"{params.http.host}:{params.http.port}" + + +def test_use_async_with_local_routes_grpc_to_rest_under_emscripten(emscripten) -> None: + import weaviate + + _assert_grpc_rides_rest(weaviate.use_async_with_local(host="localhost", port=8290)) + assert _params(weaviate.use_async_with_local()).model_dump() == { + "http": {"host": "localhost", "port": 8080, "secure": False}, + "grpc": {"host": "localhost", "port": 8080, "secure": False}, + "grpc_path_prefix": GRPC_WEB_PREFIX, + } + + +def test_use_async_with_weaviate_cloud_routes_grpc_to_the_cluster_host(emscripten) -> None: + # Weaviate Cloud serves grpc-web on the cluster's own REST endpoint, not on grpc- + import weaviate + + client = weaviate.use_async_with_weaviate_cloud("abc.something.weaviate.cloud", None) + _assert_grpc_rides_rest(client) + assert _params(client).model_dump() == { + "http": {"host": "abc.something.weaviate.cloud", "port": 443, "secure": True}, + "grpc": {"host": "abc.something.weaviate.cloud", "port": 443, "secure": True}, + "grpc_path_prefix": GRPC_WEB_PREFIX, + } + + +def test_use_async_with_custom_routes_grpc_to_rest_under_emscripten(emscripten) -> None: + import weaviate + + _assert_grpc_rides_rest( + weaviate.use_async_with_custom( + http_host="wv.example.com", + http_port=443, + http_secure=True, + grpc_host="wv.example.com", + grpc_port=443, + grpc_secure=True, + ) + ) + + +def test_matching_grpc_arguments_are_not_warned_about(emscripten, recwarn) -> None: + # gRPC arguments equal to the HTTP ones: nothing is replaced, so no warning + import weaviate + + weaviate.use_async_with_custom( + http_host="localhost", + http_port=8290, + http_secure=False, + grpc_host="localhost", + grpc_port=8290, + grpc_secure=False, + ) + weaviate.use_async_with_local(port=8290) + weaviate.use_async_with_weaviate_cloud("abc.something.weaviate.cloud", None) + assert [str(w.message) for w in recwarn] == [] + + +def test_overridden_grpc_arguments_are_warned_about(emscripten) -> None: + # a replaced caller endpoint must warn + import weaviate + + with pytest.warns(UserWarning, match="Con006") as record: + client = weaviate.use_async_with_custom( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="grpc.example.com", + grpc_port=50051, + grpc_secure=True, + ) + msg = str(record[0].message) + assert "grpc.example.com:50051" in msg # what was discarded ... + assert "localhost:8080" in msg # ... and what is used instead + assert "WebAssembly" in msg # ... and why + _assert_grpc_rides_rest(client) + + +def test_a_secure_only_grpc_mismatch_is_visible_in_the_warning(emscripten) -> None: + # host:port alone would print two identical endpoints; the scheme shows what differed + import weaviate + + with pytest.warns(UserWarning, match="Con006") as record: + client = weaviate.use_async_with_custom( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="localhost", + grpc_port=8080, + grpc_secure=True, + ) + msg = str(record[0].message) + assert "grpcs://localhost:8080" in msg # what was discarded ... + assert "grpc://localhost:8080" in msg # ... and what is used instead + _assert_grpc_rides_rest(client) + + +def test_an_explicit_local_grpc_port_is_warned_about_but_the_default_is_not(emscripten) -> None: + import weaviate + + with pytest.warns(UserWarning, match="Con006"): + client = weaviate.use_async_with_local(port=8080, grpc_port=8081) + _assert_grpc_rides_rest(client) + + +# --- the single-import hook (weaviate/__init__.py) ------------------------------------ +# +# The import hook replaces sys.modules["grpc"], so each scenario runs in a fresh subprocess. +# The success path runs in ci/pyodide-e2e/units.mjs. + +_REPO_ROOT = str(pathlib.Path(__file__).resolve().parents[1]) + +# CPython derives the _sysconfigdata module name from sys.platform on first use, so a +# faked platform breaks any later sysconfig lookup (pydantic imports zoneinfo, which +# calls sysconfig.get_config_var). Prime the cache before faking. +_PRIME_SYSCONFIG = """ +import sysconfig + +sysconfig.get_config_vars() +""" + + +def _run_hook_scenario( + body: str, *, prelude: str = "", path_entry: str = _REPO_ROOT, no_site: bool = False +) -> subprocess.CompletedProcess: + # -I -S: skip site-packages entirely (plain -I still processes the venv's .pth + # files), so nothing pip-installed is importable — only stdlib plus `path_entry`. + interp = [sys.executable, "-I", "-S"] if no_site else [sys.executable] + script = f"import sys\nsys.path.insert(0, {path_entry!r})\n" + prelude + textwrap.dedent(body) + return subprocess.run([*interp, "-c", script], capture_output=True, text=True) + + +def test_bare_import_without_companion_raises_clear_import_error() -> None: + # No site-packages, so neither weaviate_client_web nor grpcio is importable; the repo + # root goes on sys.path so the weaviate package itself is still found. + result = _run_hook_scenario( + """ + sys.platform = "emscripten" + try: + import weaviate + except ImportError as e: + assert "weaviate-client-web" in str(e), str(e) + assert "weaviate-client[grpc-web]" in str(e), str(e) + assert "WebAssembly/Pyodide" in str(e), str(e) + print("OK") + else: + raise AssertionError("expected ImportError without the companion") + """, + no_site=True, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_bare_import_with_grpc_present_falls_through_silently() -> None: + # weaviate_client_web blocked but a real grpc importable (grpcio in the dev env): the + # hook falls through and leaves the normal import path untouched + result = _run_hook_scenario( + prelude=_PRIME_SYSCONFIG, + body=""" + sys.platform = "emscripten" + sys.modules["weaviate_client_web"] = None # makes its import raise ImportError + + import weaviate + import grpc + + assert not getattr(grpc, "__weaviate_client_web_shim__", False) + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_bare_import_with_broken_companion_surfaces_its_own_error(tmp_path) -> None: + # an installed weaviate-client-web that fails to import surfaces its own error, not the + # install hint + fake_pkg = tmp_path / "weaviate_client_web" + fake_pkg.mkdir() + (fake_pkg / "__init__.py").write_text( + "raise ModuleNotFoundError(\"No module named 'anyio'\", name='anyio')\n" + ) + result = _run_hook_scenario( + prelude=_PRIME_SYSCONFIG, + body=""" + sys.platform = "emscripten" + try: + import weaviate + except ImportError as e: + assert e.name == "anyio", (e.name, str(e)) + assert "anyio" in str(e), str(e) + assert "grpc-web" not in str(e), str(e) + print("OK") + else: + raise AssertionError("expected the companion's own ImportError to surface") + """, + path_entry=str(tmp_path), + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout diff --git a/weaviate/__init__.py b/weaviate/__init__.py index f3b38dab5..751663920 100644 --- a/weaviate/__init__.py +++ b/weaviate/__init__.py @@ -1,7 +1,30 @@ """Weaviate Python Client Library used to interact with a Weaviate instance.""" -import os import sys + +# Must run first: under Emscripten weaviate_client_web installs the grpc shim the imports +# below need. +if sys.platform == "emscripten": + try: + import weaviate_client_web # noqa: F401 + except ImportError as exc: + from importlib.util import find_spec + + # only a missing package gets the install hint; if it is installed but fails to + # import (e.g. one of its own dependencies is broken), show that error instead + if not (isinstance(exc, ModuleNotFoundError) and exc.name == "weaviate_client_web"): + raise + if find_spec("grpc") is None: + raise ImportError( + "weaviate requires the weaviate-client-web package under " + "WebAssembly/Pyodide: there is no grpcio wheel for Emscripten, and " + "weaviate-client-web provides the grpc-web (fetch) transport in its " + "place. Install it via the extra (e.g. " + "micropip.install('weaviate-client[grpc-web]')) and import weaviate " + "again." + ) from exc + +import os from importlib.metadata import PackageNotFoundError, version from typing import Any diff --git a/weaviate/collections/batch/async_.py b/weaviate/collections/batch/async_.py index c63ec2106..07083fe9b 100644 --- a/weaviate/collections/batch/async_.py +++ b/weaviate/collections/batch/async_.py @@ -37,6 +37,7 @@ ReferenceToMulti, ) from weaviate.collections.classes.types import WeaviateProperties +from weaviate.connect.base import _grpc_web_shim_active from weaviate.connect.executor import aresult from weaviate.connect.v4 import ConnectionAsync from weaviate.exceptions import ( @@ -133,6 +134,14 @@ def __all_tasks_alive(self) -> bool: return self.__bg_tasks is not None and self.__bg_tasks.all_alive() async def _start(self): + if _grpc_web_shim_active(): + # fail early: over grpc-web the BatchStream RPC would fail inside the background + # tasks, which shows up as silently dropped objects or a flush() that never ends + raise WeaviateBatchStreamError( + "batch.stream() requires bidirectional gRPC streaming, which is not " + "possible over grpc-web/fetch (WebAssembly/Pyodide). Use " + "collection.data.insert_many() instead." + ) self.__number_of_nodes = await self.__cluster.get_number_of_nodes() async def loop_wrapper() -> None: diff --git a/weaviate/connect/base.py b/weaviate/connect/base.py index 99607e3ae..5ac4689d8 100644 --- a/weaviate/connect/base.py +++ b/weaviate/connect/base.py @@ -9,6 +9,7 @@ from pydantic import BaseModel, field_validator, model_validator from weaviate.config import GrpcConfig, Proxies +from weaviate.exceptions import WeaviateInvalidInputError from weaviate.types import NUMBER from weaviate.util import is_weaviate_domain @@ -18,6 +19,15 @@ JSONPayload = Union[Mapping[str, Any], Sequence[Any]] TIMEOUT_TYPE_RETURN = Tuple[NUMBER, NUMBER] MAX_GRPC_MESSAGE_LENGTH = 104858000 # 10mb, needs to be synchronized with GRPC server +# first Weaviate release serving grpc-web on the REST endpoint +GRPC_WEB_MIN_SERVER_VERSION = "1.38.3" +# Weaviate's grpc-web path prefix +GRPC_WEB_SERVER_PATH_PREFIX = "/v1/grpc-web" + + +def _grpc_web_shim_active() -> bool: + """Whether weaviate-client-web's grpc shim is installed (unary grpc-web RPCs, no streaming).""" + return getattr(grpc, "__weaviate_client_web_shim__", False) is True class ProtocolParams(BaseModel): @@ -47,9 +57,18 @@ def is_gcp(self) -> bool: class ConnectionParams(BaseModel): http: ProtocolParams grpc: ProtocolParams + # grpc-web path prefix (e.g. "/v1/grpc-web"); None/"" = native gRPC. When set, gRPC may + # share the REST host:port. + grpc_path_prefix: Optional[str] = None @classmethod - def from_url(cls, url: str, grpc_port: int, grpc_secure: bool = False) -> "ConnectionParams": + def from_url( + cls, + url: str, + grpc_port: int, + grpc_secure: bool = False, + grpc_path_prefix: Optional[str] = None, + ) -> "ConnectionParams": parsed_url = urlparse(url) if parsed_url.scheme not in ["http", "https"]: raise ValueError(f"Unsupported scheme: {parsed_url.scheme}") @@ -69,6 +88,7 @@ def from_url(cls, url: str, grpc_port: int, grpc_secure: bool = False) -> "Conne port=grpc_port, secure=grpc_secure or parsed_url.scheme == "https", ), + grpc_path_prefix=grpc_path_prefix, ) @classmethod @@ -80,6 +100,7 @@ def from_params( grpc_host: str, grpc_port: int, grpc_secure: bool, + grpc_path_prefix: Optional[str] = None, ) -> "ConnectionParams": return cls( http=ProtocolParams( @@ -92,6 +113,7 @@ def from_params( port=grpc_port, secure=grpc_secure, ), + grpc_path_prefix=grpc_path_prefix, ) def is_gcp_on_wcd(self) -> bool: @@ -99,7 +121,8 @@ def is_gcp_on_wcd(self) -> bool: @model_validator(mode="after") def _check_port_collision(self: T) -> T: - if self.http.host == self.grpc.host and self.http.port == self.grpc.port: + same_endpoint = self.http.host == self.grpc.host and self.http.port == self.grpc.port + if same_endpoint and self._grpc_web_path_prefix == "": raise ValueError("http.port and grpc.port must be different if using the same host") return self @@ -111,6 +134,39 @@ def _grpc_address(self) -> Tuple[str, int]: def _grpc_target(self) -> str: return f"{self.grpc.host}:{self.grpc.port}" + @property + def _grpc_web_path_prefix(self) -> str: + """Normalized grpc-web path prefix; "" means native gRPC. + + One leading slash, no trailing slash ("grpc-web/" -> "/grpc-web"); empty or + None -> "". + """ + cleaned = (self.grpc_path_prefix or "").strip("/") + return f"/{cleaned}" if cleaned else "" + + def _check_grpc_web_usable(self, is_async: bool) -> None: + """Raise if a grpc-web prefix is set but unusable (sync client, or no grpc shim). + + grpcio would ignore the prefix option. + """ + if self._grpc_web_path_prefix == "": + return + if not is_async: + raise WeaviateInvalidInputError( + "grpc_path_prefix (grpc-web) is only supported for async clients; " + "use use_async_with_custom(...) / WeaviateAsyncClient" + ) + if not _grpc_web_shim_active(): + raise WeaviateInvalidInputError( + "grpc_path_prefix enables grpc-web, which requires the " + "'weaviate-client-web' package (it installs a grpc shim before " + "'import weaviate'); it is not active in this environment. grpc-web is " + "only available under WebAssembly/Pyodide, where a plain `import " + "weaviate` activates it (install the companion with " + "micropip.install('weaviate-client[grpc-web]')); on CPython use native " + "gRPC instead." + ) + def _grpc_channel( self, proxies: Dict[str, str], @@ -134,6 +190,9 @@ def _grpc_channel( if grpc_config is not None and grpc_config.channel_options is not None: options.extend(grpc_config.channel_options) + if (prefix := self._grpc_web_path_prefix) != "": + options.append(("grpc-web.path_prefix", prefix)) + if is_async: mod = grpc.aio else: diff --git a/weaviate/connect/helpers.py b/weaviate/connect/helpers.py index 29faaa3c2..c35e23b25 100644 --- a/weaviate/connect/helpers.py +++ b/weaviate/connect/helpers.py @@ -1,5 +1,6 @@ """Helper functions for creating new WeaviateClient or WeaviateAsyncClient instances in common scenarios.""" +import sys from typing import Dict, Optional, Tuple, Union from urllib.parse import urlparse @@ -15,12 +16,43 @@ ) from weaviate.client import WeaviateAsyncClient, WeaviateClient from weaviate.config import AdditionalConfig -from weaviate.connect.base import ConnectionParams, ProtocolParams +from weaviate.connect.base import GRPC_WEB_SERVER_PATH_PREFIX, ConnectionParams, ProtocolParams from weaviate.embedded import WEAVIATE_VERSION, EmbeddedOptions from weaviate.util import docstring_deprecated from weaviate.validator import _validate_input, _ValidateArgument from weaviate.warnings import _Warnings +# Default gRPC port of a local Weaviate. use_async_with_local() also uses it to tell +# whether the caller picked a gRPC port of their own. +_LOCAL_GRPC_PORT_DEFAULT = 50051 + + +def _grpc_endpoint_str(params: ProtocolParams) -> str: + """The endpoint as shown in Con006; the scheme makes a secure-only mismatch visible.""" + return f"{'grpcs' if params.secure else 'grpc'}://{params.host}:{params.port}" + + +def _webify( + http: ProtocolParams, grpc: ProtocolParams, *, grpc_chosen_by_caller: bool +) -> ConnectionParams: + """Build ConnectionParams; under Emscripten, route gRPC to the REST endpoint over grpc-web. + + grpc_chosen_by_caller: whether ``grpc`` is caller input rather than a helper default; a + replaced caller endpoint raises warning Con006. + """ + if sys.platform != "emscripten": + # grpc_path_prefix=None is passed explicitly so the constructor call (and pydantic's + # error output for it) looks exactly as it did before grpc-web existed + return ConnectionParams(http=http, grpc=grpc, grpc_path_prefix=None) + + web_grpc = ProtocolParams(host=http.host, port=http.port, secure=http.secure) + if grpc_chosen_by_caller and web_grpc != grpc: + _Warnings.grpc_endpoint_forced_to_grpc_web( + requested=_grpc_endpoint_str(grpc), + effective=_grpc_endpoint_str(web_grpc), + ) + return ConnectionParams(http=http, grpc=web_grpc, grpc_path_prefix=GRPC_WEB_SERVER_PATH_PREFIX) + def __parse_weaviate_cloud_cluster_url(cluster_url: str) -> Tuple[str, str]: _validate_input(_ValidateArgument([str], "cluster_url", cluster_url)) @@ -384,6 +416,9 @@ def use_async_with_weaviate_cloud( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. + Under Pyodide, gRPC runs over grpc-web on the cluster's REST endpoint (443) instead of + the ``grpc-`` host. + Args: cluster_url: The WCD cluster URL or hostname to connect to. Usually in the form: rAnD0mD1g1t5.something.weaviate.cloud auth_credentials: The credentials to use for authentication with your Weaviate instance. This can be an API key, in which case pass a string or use `weaviate.classes.init.Auth.api_key()`, @@ -420,9 +455,11 @@ def use_async_with_weaviate_cloud( """ cluster_url, grpc_host = __parse_weaviate_cloud_cluster_url(cluster_url) return WeaviateAsyncClient( - connection_params=ConnectionParams( + connection_params=_webify( http=ProtocolParams(host=cluster_url, port=443, secure=True), grpc=ProtocolParams(host=grpc_host, port=443, secure=True), + # the grpc- host is the helper's default, not caller input + grpc_chosen_by_caller=False, ), auth_client_secret=__parse_auth_credentials(auth_credentials), additional_headers=headers, @@ -446,10 +483,13 @@ def use_async_with_local( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. + Under Pyodide, gRPC runs over grpc-web on the REST endpoint; a non-default + ``grpc_port`` is ignored with a warning. + Args: host: The host to use for the underlying REST and GraphQL API calls. port: The port to use for the underlying REST and GraphQL API calls. - grpc_port: The port to use for the underlying gRPC API. + grpc_port: The port to use for the underlying gRPC API. Ignored under Pyodide. headers: Additional headers to include in the requests, e.g. API keys for Cloud vectorization. additional_config: This includes many additional, rarely used config options. use wvc.init.AdditionalConfig() to configure. skip_init_checks: Whether to skip the initialization checks when connecting to Weaviate. @@ -486,9 +526,11 @@ def use_async_with_local( >>> # The connection is automatically closed when the context is exited. """ return WeaviateAsyncClient( - connection_params=ConnectionParams( + connection_params=_webify( http=ProtocolParams(host=host, port=port, secure=False), grpc=ProtocolParams(host=host, port=grpc_port, secure=False), + # the default port comes from the helper; anything else the caller chose + grpc_chosen_by_caller=grpc_port != _LOCAL_GRPC_PORT_DEFAULT, ), additional_headers=headers, additional_config=additional_config, @@ -596,13 +638,18 @@ def use_async_with_custom( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. + Under Pyodide, gRPC runs over grpc-web on the REST endpoint: ``grpc_host``, + ``grpc_port`` and ``grpc_secure`` are replaced by the HTTP values, with a warning if + they differ. + Args: http_host: The host to use for the underlying REST and GraphQL API calls. http_port: The port to use for the underlying REST and GraphQL API calls. http_secure: Whether to use https for the underlying REST and GraphQL API calls. - grpc_host: The host to use for the underlying gRPC API. - grpc_port: The port to use for the underlying gRPC API. - grpc_secure: Whether to use a secure channel for the underlying gRPC API. + grpc_host: The host to use for the underlying gRPC API. Ignored under Pyodide. + grpc_port: The port to use for the underlying gRPC API. Ignored under Pyodide. + grpc_secure: Whether to use a secure channel for the underlying gRPC API. Ignored + under Pyodide. headers: Additional headers to include in the requests, e.g. API keys for Cloud vectorization. additional_config: This includes many additional, rarely used config options. use wvc.init.AdditionalConfig() to configure. auth_credentials: The credentials to use for authentication with your Weaviate instance. This can be an API key, in which case pass a string or use `weaviate.classes.init.Auth.api_key()`, @@ -645,13 +692,11 @@ def use_async_with_custom( >>> # The connection is automatically closed when the context is exited. """ return WeaviateAsyncClient( - ConnectionParams.from_params( - http_host=http_host, - http_port=http_port, - http_secure=http_secure, - grpc_host=grpc_host, - grpc_port=grpc_port, - grpc_secure=grpc_secure, + _webify( + http=ProtocolParams(host=http_host, port=http_port, secure=http_secure), + grpc=ProtocolParams(host=grpc_host, port=grpc_port, secure=grpc_secure), + # all three gRPC arguments are required here, so they are caller input + grpc_chosen_by_caller=True, ), auth_client_secret=__parse_auth_credentials(auth_credentials), additional_headers=headers, diff --git a/weaviate/connect/v4.py b/weaviate/connect/v4.py index 60214a8d8..1cba0ec50 100644 --- a/weaviate/connect/v4.py +++ b/weaviate/connect/v4.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import sys import time from copy import copy from dataclasses import dataclass, field @@ -145,6 +146,17 @@ def __init__( self._connection_params = connection_params self._grpc_stub: Optional[weaviate_pb2_grpc.WeaviateStub] = None self._grpc_channel: Union[AsyncChannel, SyncChannel, None] = None + if sys.platform == "emscripten" and isinstance(self, ConnectionSync): + # fail here, at construction, instead of with an unclear ConnectError on the + # first REST call; _client/_grpc_channel are already set, so __del__ does not warn + raise WeaviateStartUpError( + "The synchronous client is not supported under WebAssembly/Pyodide. " + "Use an async client (weaviate.use_async_with_local / " + "use_async_with_weaviate_cloud / use_async_with_custom, or " + "WeaviateAsyncClient) instead." + ) + # a grpc-web prefix this client cannot use fails here, not deep inside connect() + connection_params._check_grpc_web_usable(is_async=not isinstance(self, ConnectionSync)) self.timeout_config = timeout_config self.__connection_config = connection_config self.__trust_env = trust_env @@ -339,15 +351,25 @@ async def execute(): def __handle_ping_response(self, res: health_weaviate_pb2.WeaviateHealthCheckResponse) -> None: if res.status != health_weaviate_pb2.WeaviateHealthCheckResponse.SERVING: raise WeaviateGRPCUnavailableError( - f"v{self.server_version}", self._connection_params._grpc_address + f"v{self.server_version}", + self._connection_params._grpc_address, + grpc_path_prefix=self.__grpc_web_prefix(), ) return None def __handle_ping_exception(self, e: Exception) -> None: + # pass the error on so the message can report its status and details raise WeaviateGRPCUnavailableError( - f"v{self.server_version}", self._connection_params._grpc_address + f"v{self.server_version}", + self._connection_params._grpc_address, + grpc_path_prefix=self.__grpc_web_prefix(), + error=e, ) from e + def __grpc_web_prefix(self) -> Optional[str]: + """The configured grpc-web path prefix, or None for native gRPC.""" + return self._connection_params._grpc_web_path_prefix or None + @property def grpc_stub(self) -> Optional[weaviate_pb2_grpc.WeaviateStub]: if not self.is_connected(): @@ -789,8 +811,11 @@ async def _execute() -> None: async with AsyncClient() as client: res = await client.get(PYPI_PACKAGE_URL, timeout=self.timeout_config.init) return resp(res) - except RequestError: - pass # ignore any errors related to requests, it is a best-effort warning + except (RequestError, OSError): + # ignore any request error, this is a best-effort warning. OSError covers + # fetch failures under Pyodide/WASM, where the page's CSP often blocks + # pypi.org; that must not fail connect(). + pass return _execute() @@ -798,7 +823,7 @@ async def _execute() -> None: with Client() as client: res = client.get(PYPI_PACKAGE_URL, timeout=self.timeout_config.init) return resp(res) - except RequestError: + except (RequestError, OSError): pass # ignore any errors related to requests, it is a best-effort warning def delete( diff --git a/weaviate/embedded.py b/weaviate/embedded.py index a511665cc..731560547 100644 --- a/weaviate/embedded.py +++ b/weaviate/embedded.py @@ -5,6 +5,7 @@ import socket import stat import subprocess +import sys import tarfile import time import urllib.request @@ -175,6 +176,14 @@ def wait_till_listening(self) -> None: @staticmethod def check_supported_platform() -> None: + if sys.platform == "emscripten": + # without this check the port probe below "succeeds" under Emscripten's fake + # sockets and wrongly reports that Weaviate is already running + raise WeaviateStartUpError( + "Embedded Weaviate is not supported under WebAssembly/Pyodide: it spawns a " + "local Weaviate subprocess, and processes are unavailable in the browser. " + "Connect to a remote Weaviate instance instead." + ) if platform.system() in ["Windows"]: raise WeaviateStartUpError( f"""{platform.system()} is not supported with EmbeddedDB. Please upvote this feature request if you want diff --git a/weaviate/exceptions.py b/weaviate/exceptions.py index ce0fe6f7e..f83e05224 100644 --- a/weaviate/exceptions.py +++ b/weaviate/exceptions.py @@ -317,6 +317,18 @@ def __init__(self, data: dict): super().__init__(msg) +def _grpc_status_of( + error: Optional[BaseException], +) -> Tuple[Optional[StatusCode], Optional[str]]: + """Return the (code, details) of a gRPC error, or (None, None) if it carries none.""" + if isinstance(error, (AioRpcError, Call)): + try: + return cast(Optional[StatusCode], error.code()), error.details() + except Exception: # a half-initialized call can raise instead of answering + return None, None + return None, None + + class WeaviateGRPCUnavailableError(WeaviateBaseError): """Is raised when a gRPC-backed query is made with no gRPC connection present.""" @@ -324,7 +336,54 @@ def __init__( self, weaviate_version: str = "", grpc_address: Tuple[str, int] = ("not provided", 0), + grpc_path_prefix: Optional[str] = None, + error: Optional[BaseException] = None, ) -> None: + code, details = _grpc_status_of(error) + observed = "" + if code is not None or details: + code_name = code.name if code is not None else "unknown status" + observed = ( + f"\nThe gRPC call failed with: {code_name}{f' - {details}' if details else ''}\n" + ) + + if grpc_path_prefix: + # local import: weaviate.connect imports this module at import time, so a + # module-level import here would be circular + from weaviate.connect.base import ( + GRPC_WEB_MIN_SERVER_VERSION, + GRPC_WEB_SERVER_PATH_PREFIX, + ) + + # no firewall/wrong-port advice: REST already worked, and grpc-web normally + # shares its endpoint + address = f"{grpc_address[0]}:{grpc_address[1]}" + # weaviate_client_web reports an unrouted path as UNIMPLEMENTED with "HTTP 404/405" + # in details; a routed endpoint's own UNIMPLEMENTED is not a wrong path + if code is StatusCode.UNIMPLEMENTED and any( + marker in (details or "") for marker in ("HTTP 404", "HTTP 405") + ): + reason = f"""The server did not route the grpc-web path '{grpc_path_prefix}' at {address}. Either: +- the server is too old: grpc-web is served from Weaviate {GRPC_WEB_MIN_SERVER_VERSION} onwards, and this server reports {weaviate_version or "an unknown version"}, or +- the grpc-web base path is wrong: Weaviate serves grpc-web at '{GRPC_WEB_SERVER_PATH_PREFIX}'. The connect helpers set it themselves; only hand-built ConnectionParams choose it (grpc_path_prefix). +""" + else: + reason = f"""This error could be due to one of several reasons: +- grpc-web is not enabled or is incorrectly configured on the server at {address}. +- your connection is unstable or has a high latency. In this case you can: + - increase init-timeout in `weaviate.use_async_with_custom(additional_config=wvc.init.AdditionalConfig(timeout=wvc.init.Timeout(init=X)))` + - disable startup checks by connecting using `skip_init_checks=True` +""" + msg = f""" +Weaviate {weaviate_version} makes use of a high-speed gRPC API as well as a REST API. +Unfortunately, the gRPC health check against Weaviate could not be completed. + +This client speaks grpc-web (base path '{grpc_path_prefix}'), which carries gRPC over the REST endpoint {address}; there is no separate gRPC port. + +{reason}{observed}""" + super().__init__(msg) + return + if grpc_address[0] == "not provided": grpc_msg = "Please check the server address and port." else: @@ -340,7 +399,7 @@ def __init__( - your connection is unstable or has a high latency. In this case you can: - increase init-timeout in `weaviate.connect_to_local(additional_config=wvc.init.AdditionalConfig(timeout=wvc.init.Timeout(init=X)))` - disable startup checks by connecting using `skip_init_checks=True` -""" +{observed}""" super().__init__(msg) diff --git a/weaviate/proto/v1/__init__.py b/weaviate/proto/v1/__init__.py index 09171e683..92297b8b9 100644 --- a/weaviate/proto/v1/__init__.py +++ b/weaviate/proto/v1/__init__.py @@ -1,3 +1,4 @@ +import sys import warnings @@ -11,12 +12,21 @@ from packaging import version -from importlib.metadata import version as metadata_version +from importlib.metadata import PackageNotFoundError, version as metadata_version from weaviate.exceptions import WeaviateProtobufIncompatibility -def get_version(pkg: str)-> version.Version: - return version.parse(metadata_version(pkg)) +# grpcio version assumed under Emscripten, where weaviate-client-web provides grpc; must be +# >= 1.72.0, or the protobuf >= 6.30 check below raises. +_GRPCIO_FALLBACK_VERSION = "1.72.1" + +def get_version(pkg: str) -> version.Version: + try: + return version.parse(metadata_version(pkg)) + except PackageNotFoundError: + if pkg == "grpcio" and sys.platform == "emscripten": + return version.parse(_GRPCIO_FALLBACK_VERSION) + raise pb_version, grpc_version = get_version("protobuf"), get_version("grpcio") if pb_version >= version.parse("6.30.0"): diff --git a/weaviate/warnings.py b/weaviate/warnings.py index 1c0a1ae0b..d69027fbb 100644 --- a/weaviate/warnings.py +++ b/weaviate/warnings.py @@ -325,6 +325,19 @@ def grpc_max_msg_size_not_found() -> None: stacklevel=1, ) + @staticmethod + def grpc_endpoint_forced_to_grpc_web(requested: str, effective: str) -> None: + warnings.warn( + message=f"""Con006: The gRPC endpoint you gave ({requested}) was overridden with {effective}. + + Under WebAssembly/Pyodide there is no socket and no grpcio wheel, so native gRPC cannot be used at all; + gRPC runs over grpc-web on the REST listener, which is the endpoint above. Pass gRPC arguments matching + the HTTP ones to silence this warning. A grpc-web transcoder on a separate endpoint is not reachable + through these helpers - build weaviate.connect.ConnectionParams yourself if you need one.""", + category=UserWarning, + stacklevel=1, + ) + @staticmethod def unknown_permission_encountered(permission: Any) -> None: warnings.warn(