From 6543d066943781e1bd6698d020b11660d660d4b4 Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Sun, 25 Oct 2026 09:47:06 -0400 Subject: [PATCH 1/2] Adds structural serving capabilities for external predictors. --- CHANGELOG.md | 15 ++ bead/serving/__init__.py | 36 ++++ bead/serving/conformance.py | 341 +++++++++++++++++++++++++++++++++ bead/serving/protocols.py | 229 ++++++++++++++++++++++ bead/serving/registry.py | 83 ++++++++ docs/api/serving.md | 25 +++ docs/user-guide/api/serving.md | 157 +++++++++++++++ mkdocs.yml | 2 + tests/serving/__init__.py | 1 + tests/serving/test_serving.py | 195 +++++++++++++++++++ 10 files changed, 1084 insertions(+) create mode 100644 bead/serving/__init__.py create mode 100644 bead/serving/conformance.py create mode 100644 bead/serving/protocols.py create mode 100644 bead/serving/registry.py create mode 100644 docs/api/serving.md create mode 100644 docs/user-guide/api/serving.md create mode 100644 tests/serving/__init__.py create mode 100644 tests/serving/test_serving.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bde1722..2ca10e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `bead.serving` publishes independent runtime protocols for model identity, + task compatibility, probabilistic prediction with `ResponseEncoding` column + semantics, native-artifact persistence, and three-way NLI. Externally trained, + read-only predictors implement these capabilities structurally and do not inherit + the trainable mixed-effects hierarchy or the nominal model-adapter ABC. +- `PredictorRegistry` registers artifact-loading factories without an `issubclass` + requirement. Its generic capability type lets each consumer request only the + serving operations it uses. +- `run_predictor_conformance` gives third-party packages a published interoperability + suite over real `Item`, `ItemTemplate`, and `ResponseEncoding` values. It checks + task compatibility, probability shape and normalization, optional participant + identifiers, save-load equality, and three-way NLI probabilities. + ## [0.10.0] - 2026-09-16 ### Changed diff --git a/bead/serving/__init__.py b/bead/serving/__init__.py new file mode 100644 index 0000000..b84a9f0 --- /dev/null +++ b/bead/serving/__init__.py @@ -0,0 +1,36 @@ +"""Structural interfaces for externally trained, read-only predictors.""" + +from bead.serving.conformance import ( + ConformanceError, + ConformanceReport, + NLICase, + PredictionCase, + run_predictor_conformance, +) +from bead.serving.protocols import ( + ModelIdentity, + NLICapability, + NLIProbabilities, + PersistablePredictor, + ProbabilisticPredictor, + ServingPredictor, + TaskCompatibility, +) +from bead.serving.registry import PredictorFactory, PredictorRegistry + +__all__ = [ + "ConformanceError", + "ConformanceReport", + "ModelIdentity", + "NLICapability", + "NLICase", + "NLIProbabilities", + "PersistablePredictor", + "PredictionCase", + "PredictorFactory", + "PredictorRegistry", + "ProbabilisticPredictor", + "ServingPredictor", + "TaskCompatibility", + "run_predictor_conformance", +] diff --git a/bead/serving/conformance.py b/bead/serving/conformance.py new file mode 100644 index 0000000..d6989cc --- /dev/null +++ b/bead/serving/conformance.py @@ -0,0 +1,341 @@ +"""Reusable conformance checks for third-party serving predictors.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from math import isfinite +from pathlib import Path +from tempfile import TemporaryDirectory + +import numpy as np + +from bead.items.item import Item +from bead.items.item_template import ItemTemplate +from bead.items.validation import validate_item +from bead.protocol.encoding import ResponseEncoding +from bead.protocol.items import scale_type_to_task_type +from bead.serving.protocols import ( + ModelIdentity, + NLICapability, + NLIProbabilities, + PersistablePredictor, + ProbabilisticPredictor, + ServingPredictor, + TaskCompatibility, +) + +__all__ = [ + "ConformanceError", + "ConformanceReport", + "NLICase", + "PredictionCase", + "run_predictor_conformance", +] + +type _ServingCapability = ( + ModelIdentity + | TaskCompatibility + | ProbabilisticPredictor + | PersistablePredictor + | NLICapability +) + + +class ConformanceError(AssertionError): + """A serving predictor violated its public capability contract.""" + + +@dataclass(frozen=True, slots=True) +class PredictionCase: + """A real bead item paired with the template that defines its task.""" + + item: Item + item_template: ItemTemplate + + +@dataclass(frozen=True, slots=True) +class NLICase: + """A premise-hypothesis pair used to exercise NLI scoring.""" + + premise: str + hypothesis: str + + +@dataclass(frozen=True, slots=True) +class ConformanceReport: + """Summary returned after every conformance check passes.""" + + model_name: str + model_version: str + n_items: int + n_levels: int + n_nli_cases: int + checks: tuple[str, ...] + + +def _fail(message: str) -> ConformanceError: + return ConformanceError(message) + + +def _validate_identity(predictor: ServingPredictor) -> None: + if not predictor.model_name.strip(): + raise _fail("model_name must be a non-empty string") + if not predictor.model_version.strip(): + raise _fail("model_version must be a non-empty string") + + +def _same_response_encoding(left: ResponseEncoding, right: ResponseEncoding) -> bool: + """Return whether two encodings assign the same response semantics.""" + left_poles = ( + left.semantic_poles.as_tuple() if left.semantic_poles is not None else None + ) + right_poles = ( + right.semantic_poles.as_tuple() if right.semantic_poles is not None else None + ) + return ( + left.name == right.name + and left.n_levels == right.n_levels + and left.scale_type == right.scale_type + and left.labels == right.labels + and left_poles == right_poles + ) + + +def _validate_prediction_matrix( + probabilities: np.ndarray | list[list[float]], + *, + n_items: int, + response_encoding: ResponseEncoding, + source: str, +) -> None: + if not isinstance(probabilities, np.ndarray): + raise _fail(f"{source} must return numpy.ndarray") + + expected_shape = (n_items, response_encoding.n_levels) + if probabilities.shape != expected_shape: + raise _fail( + f"{source} returned shape {probabilities.shape}; expected {expected_shape} " + f"for encoding {response_encoding.name!r}" + ) + + try: + numeric = probabilities.astype(np.float64, copy=False) + except (TypeError, ValueError) as error: + raise _fail(f"{source} returned non-numeric probabilities: {error}") from error + + if not np.isfinite(numeric).all(): + raise _fail(f"{source} returned non-finite probabilities") + if ((numeric < 0.0) | (numeric > 1.0)).any(): + raise _fail(f"{source} returned probabilities outside [0, 1]") + if not np.allclose(numeric.sum(axis=1), 1.0, rtol=1e-6, atol=1e-8): + raise _fail(f"{source} returned rows that do not sum to one") + + +def _validate_nli( + probabilities: NLIProbabilities, + *, + source: str, +) -> None: + expected_keys = {"entailment", "neutral", "contradiction"} + if set(probabilities) != expected_keys: + raise _fail( + f"{source} returned NLI keys {set(probabilities)}; expected {expected_keys}" + ) + values = tuple(probabilities[key] for key in sorted(expected_keys)) + try: + numeric_values = tuple(float(value) for value in values) + except (TypeError, ValueError) as error: + raise _fail(f"{source} returned non-numeric NLI probabilities") from error + if not all(isfinite(value) for value in numeric_values): + raise _fail(f"{source} returned non-finite NLI probabilities") + if not all(0.0 <= value <= 1.0 for value in numeric_values): + raise _fail(f"{source} returned NLI probabilities outside [0, 1]") + if not np.isclose(sum(numeric_values), 1.0, rtol=1e-6, atol=1e-8): + raise _fail(f"{source} returned NLI probabilities that do not sum to one") + + +def _nli_as_array(probabilities: NLIProbabilities) -> np.ndarray: + return np.asarray( + [ + probabilities["entailment"], + probabilities["neutral"], + probabilities["contradiction"], + ], + dtype=np.float64, + ) + + +def run_predictor_conformance( + predictor: _ServingCapability, + *, + prediction_cases: Sequence[PredictionCase], + response_encoding: ResponseEncoding, + nli_cases: Sequence[NLICase], +) -> ConformanceReport: + """Run the published serving-predictor conformance suite. + + The suite verifies structural capability composition, identity, task and + item compatibility, ``ResponseEncoding`` column semantics, probability + shape and normalization, optional participant identifiers, native artifact + save/load equality, and three-way NLI probabilities. + + Parameters + ---------- + predictor : ServingPredictor + Structural implementation of all serving capabilities. It does not + need to inherit from a bead class. + prediction_cases : Sequence[PredictionCase] + Non-empty sequence of real :class:`Item` and :class:`ItemTemplate` + pairs accepted by the predictor. + response_encoding : ResponseEncoding + Encoding whose ordered labels name the columns returned by + ``predict_proba``. + nli_cases : Sequence[NLICase] + Non-empty sequence of premise-hypothesis pairs. + + Returns + ------- + ConformanceReport + Summary of the checks that passed. + + Raises + ------ + ConformanceError + If any capability violates the serving contract. + + Examples + -------- + >>> report = run_predictor_conformance( # doctest: +SKIP + ... predictor, + ... prediction_cases=[PredictionCase(item, item_template)], + ... response_encoding=encoding, + ... nli_cases=[NLICase("A dog runs.", "An animal moves.")], + ... ) + >>> report.checks # doctest: +SKIP + ('identity', 'task_compatibility', 'predict_proba', 'persistence', 'nli') + """ + if not isinstance(predictor, ServingPredictor): + raise _fail("predictor does not implement every ServingPredictor capability") + if not prediction_cases: + raise _fail("prediction_cases must contain at least one case") + if not nli_cases: + raise _fail("nli_cases must contain at least one case") + if not _same_response_encoding(predictor.response_encoding, response_encoding): + raise _fail("predictor.response_encoding does not match response_encoding") + + _validate_identity(predictor) + + expected_task_type = scale_type_to_task_type(response_encoding.scale_type) + items: list[Item] = [] + for index, case in enumerate(prediction_cases): + item_errors = validate_item(case.item, case.item_template) + if item_errors: + raise _fail( + f"prediction_cases[{index}] is not a valid Item/ItemTemplate pair: " + f"{'; '.join(item_errors)}" + ) + if case.item_template.task_type != expected_task_type: + raise _fail( + f"prediction_cases[{index}] uses task type " + f"{case.item_template.task_type!r}, but encoding " + f"{response_encoding.name!r} maps to {expected_task_type!r}" + ) + if case.item_template.task_type not in predictor.supported_task_types: + raise _fail( + f"prediction_cases[{index}] uses unsupported task type " + f"{case.item_template.task_type!r}" + ) + try: + predictor.validate_item_compatibility(case.item, case.item_template) + except (TypeError, ValueError) as error: + raise _fail( + f"prediction_cases[{index}] failed compatibility validation: {error}" + ) from error + items.append(case.item) + + baseline = predictor.predict_proba(items) + _validate_prediction_matrix( + baseline, + n_items=len(items), + response_encoding=response_encoding, + source="predict_proba(items)", + ) + + participant_ids = [f"bead-conformance-{index}" for index in range(len(items))] + participant_predictions = predictor.predict_proba(items, participant_ids) + _validate_prediction_matrix( + participant_predictions, + n_items=len(items), + response_encoding=response_encoding, + source="predict_proba(items, participant_ids)", + ) + + baseline_nli: list[NLIProbabilities] = [] + for index, case in enumerate(nli_cases): + scores = predictor.compute_nli(case.premise, case.hypothesis) + _validate_nli(scores, source=f"compute_nli case {index}") + baseline_nli.append(scores) + + with TemporaryDirectory(prefix="bead-predictor-conformance-") as directory: + artifact_path = Path(directory) / "artifact" + predictor.save(artifact_path) + if not artifact_path.exists(): + raise _fail("save(path) did not create a native artifact") + restored = type(predictor).load(artifact_path) + + if restored.model_name != predictor.model_name: + raise _fail("load(path) changed model_name") + if restored.model_version != predictor.model_version: + raise _fail("load(path) changed model_version") + if set(restored.supported_task_types) != set(predictor.supported_task_types): + raise _fail("load(path) changed supported_task_types") + if not _same_response_encoding(restored.response_encoding, response_encoding): + raise _fail("load(path) changed response_encoding") + + for index, case in enumerate(prediction_cases): + try: + restored.validate_item_compatibility(case.item, case.item_template) + except (TypeError, ValueError) as error: + raise _fail( + f"load(path) changed compatibility for prediction_cases[{index}]: " + f"{error}" + ) from error + + restored_predictions = restored.predict_proba(items) + _validate_prediction_matrix( + restored_predictions, + n_items=len(items), + response_encoding=response_encoding, + source="loaded predict_proba(items)", + ) + if not np.allclose(baseline, restored_predictions, rtol=1e-6, atol=1e-8): + raise _fail("save/load changed predict_proba output") + + for index, (case, expected) in enumerate( + zip(nli_cases, baseline_nli, strict=True) + ): + actual = restored.compute_nli(case.premise, case.hypothesis) + _validate_nli(actual, source=f"loaded compute_nli case {index}") + if not np.allclose( + _nli_as_array(expected), + _nli_as_array(actual), + rtol=1e-6, + atol=1e-8, + ): + raise _fail("save/load changed compute_nli output") + + return ConformanceReport( + model_name=predictor.model_name, + model_version=predictor.model_version, + n_items=len(items), + n_levels=response_encoding.n_levels, + n_nli_cases=len(nli_cases), + checks=( + "identity", + "task_compatibility", + "predict_proba", + "persistence", + "nli", + ), + ) diff --git a/bead/serving/protocols.py b/bead/serving/protocols.py new file mode 100644 index 0000000..86c79f2 --- /dev/null +++ b/bead/serving/protocols.py @@ -0,0 +1,229 @@ +"""Structural capabilities for read-only model serving. + +The protocols in this module describe independent serving capabilities. A +consumer can request only the operations it uses, while a provider can +implement the protocols structurally without inheriting from a bead base +class. In particular, probabilistic predictors do not need to implement +training, mixed effects, language-model scoring, or embeddings. +""" + +from __future__ import annotations + +from collections.abc import Collection, Mapping +from pathlib import Path +from typing import Protocol, Self, runtime_checkable + +import numpy as np + +from bead.items.item import Item +from bead.items.item_template import ItemTemplate, TaskType +from bead.protocol.encoding import ResponseEncoding + +__all__ = [ + "ModelIdentity", + "NLICapability", + "NLIProbabilities", + "PersistablePredictor", + "ProbabilisticPredictor", + "ServingPredictor", + "TaskCompatibility", +] + + +type NLIProbabilities = Mapping[str, float] +"""Three-way natural language inference probabilities. + +The conformance suite requires exactly three keys: ``"entailment"``, +``"neutral"``, and ``"contradiction"``. Values must be finite probabilities +that sum to one. +""" + + +@runtime_checkable +class ModelIdentity(Protocol): + """Identity metadata published by a serving model. + + Attributes + ---------- + model_name + Stable name of the model family or artifact. + model_version + Version of the model or artifact. + """ + + @property + def model_name(self) -> str: + """Return the stable model name. + + Returns + ------- + str + Stable model name. + """ + ... + + @property + def model_version(self) -> str: + """Return the model or artifact version. + + Returns + ------- + str + Model or artifact version. + """ + ... + + +@runtime_checkable +class TaskCompatibility(Protocol): + """Task types and item structures accepted by a predictor. + + Attributes + ---------- + supported_task_types + Collection of task types accepted by the predictor. + """ + + @property + def supported_task_types(self) -> Collection[TaskType]: + """Return the task types accepted by the predictor. + + Returns + ------- + Collection[TaskType] + Accepted task types. + """ + ... + + def validate_item_compatibility( + self, item: Item, item_template: ItemTemplate + ) -> None: + """Validate that an item can be scored under its template. + + Parameters + ---------- + item + Item to validate. + item_template + Template that defines the item's task and structure. + + Raises + ------ + ValueError + If the item cannot be scored by the predictor. + """ + ... + + +@runtime_checkable +class ProbabilisticPredictor(Protocol): + """Read-only probability prediction with declared column semantics. + + Column ``i`` of the returned matrix corresponds to + ``response_encoding.labels[i]``. Implementations return one row per item, + with finite probabilities in ``[0, 1]`` that sum to one. + """ + + @property + def response_encoding(self) -> ResponseEncoding: + """Return the response encoding that names probability columns. + + Returns + ------- + ResponseEncoding + Ordered labels and scale type for the probability matrix. + """ + ... + + def predict_proba( + self, + items: list[Item], + participant_ids: list[str] | None = None, + ) -> np.ndarray: + """Return a probability matrix with shape ``(items, levels)``. + + Parameters + ---------- + items + Items to score. + participant_ids + Optional participant identifier for each item. + + Returns + ------- + numpy.ndarray + Probability matrix with one row per item and one column per + response-encoding label. + """ + ... + + +@runtime_checkable +class PersistablePredictor(Protocol): + """Native-artifact persistence for a serving model.""" + + def save(self, path: str | Path) -> None: + """Save the predictor's native artifact. + + Parameters + ---------- + path + File or directory path for the provider's native artifact. + """ + ... + + @classmethod + def load(cls, path: str | Path) -> Self: + """Load a predictor from a native artifact. + + Parameters + ---------- + path + File or directory path containing the provider's native artifact. + + Returns + ------- + Self + Restored predictor. + """ + ... + + +@runtime_checkable +class NLICapability(Protocol): + """Three-way natural language inference scoring.""" + + def compute_nli(self, premise: str, hypothesis: str) -> NLIProbabilities: + """Return entailment, neutral, and contradiction probabilities. + + Parameters + ---------- + premise + Premise text. + hypothesis + Hypothesis text. + + Returns + ------- + NLIProbabilities + Mapping with exactly the keys ``"entailment"``, ``"neutral"``, + and ``"contradiction"``. + """ + ... + + +@runtime_checkable +class ServingPredictor( + ModelIdentity, + TaskCompatibility, + ProbabilisticPredictor, + PersistablePredictor, + NLICapability, + Protocol, +): + """Composition of all read-only serving capabilities. + + This convenience protocol is used by the full conformance suite. Consumers + that need fewer operations should type against the independent protocols + above. + """ diff --git a/bead/serving/registry.py b/bead/serving/registry.py new file mode 100644 index 0000000..dabc76e --- /dev/null +++ b/bead/serving/registry.py @@ -0,0 +1,83 @@ +"""Factory registry for structurally typed serving predictors.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +__all__ = ["PredictorFactory", "PredictorRegistry"] + +type PredictorFactory[T] = Callable[[str | Path], T] +"""A callable that loads a predictor from its native artifact.""" + + +class PredictorRegistry[T]: + """Map provider names to native-artifact loading factories. + + The registry is generic in the capability required by its consumer. It + accepts any callable with the factory signature and performs no inheritance + or ``issubclass`` check. + + Examples + -------- + >>> from bead.serving import PredictorRegistry, ServingPredictor + >>> registry = PredictorRegistry[ServingPredictor]() + >>> registry.register("external", ExternalPredictor.load) # doctest: +SKIP + >>> predictor = registry.load("external", "model.bin") # doctest: +SKIP + """ + + def __init__(self) -> None: + self._factories: dict[str, PredictorFactory[T]] = {} + + def register(self, name: str, factory: PredictorFactory[T]) -> None: + """Register a loading factory under ``name``. + + Parameters + ---------- + name : str + Provider name used by :meth:`load`. + factory : PredictorFactory[T] + Callable that accepts an artifact path and returns a predictor. + + Raises + ------ + ValueError + If ``name`` is empty. + """ + normalized_name = name.strip() + if not normalized_name: + raise ValueError("Predictor factory name cannot be empty") + self._factories[normalized_name] = factory + + def load(self, name: str, path: str | Path) -> T: + """Load a predictor through the factory registered under ``name``. + + Parameters + ---------- + name : str + Registered provider name. + path : str | Path + Native artifact path passed to the provider factory. + + Returns + ------- + T + Predictor returned by the registered factory. + + Raises + ------ + KeyError + If no factory is registered under ``name``. + """ + try: + factory = self._factories[name] + except KeyError: + available = ", ".join(sorted(self._factories)) or "none" + raise KeyError( + f"Unknown predictor factory {name!r}; available factories: {available}" + ) from None + return factory(path) + + def list_factories(self) -> tuple[str, ...]: + """Return registered factory names in sorted order.""" + return tuple(sorted(self._factories)) diff --git a/docs/api/serving.md b/docs/api/serving.md new file mode 100644 index 0000000..0b5bfce --- /dev/null +++ b/docs/api/serving.md @@ -0,0 +1,25 @@ +# bead.serving + +Structural capabilities, factory registration, and conformance checks for +externally trained, read-only predictors. + +## Capability protocols + +::: bead.serving.protocols + options: + show_root_heading: true + show_source: false + +## Predictor registry + +::: bead.serving.registry + options: + show_root_heading: true + show_source: false + +## Conformance suite + +::: bead.serving.conformance + options: + show_root_heading: true + show_source: false diff --git a/docs/user-guide/api/serving.md b/docs/user-guide/api/serving.md new file mode 100644 index 0000000..0f83914 --- /dev/null +++ b/docs/user-guide/api/serving.md @@ -0,0 +1,157 @@ +# Read-only predictors + +`bead.serving` defines the serving contract for externally trained predictors. +Providers implement ordinary Python classes. They do not inherit +`ActiveLearningModel`, implement training or random effects, or supply unrelated +language-model scoring and embedding operations. + +## Compose capabilities + +The public protocols are independent: + +- `ModelIdentity` provides `model_name` and `model_version`. +- `TaskCompatibility` provides `supported_task_types` and + `validate_item_compatibility(item, item_template)`. +- `ProbabilisticPredictor` provides `response_encoding` and + `predict_proba(items, participant_ids=None)`. +- `PersistablePredictor` provides `save(path)` and class method `load(path)`. +- `NLICapability` provides `compute_nli(premise, hypothesis)`. + +Type a consumer against the smallest protocol it uses. The `ServingPredictor` +protocol composes all five capabilities for consumers that require the complete +read-only surface and for the conformance suite. + +The `response_encoding` property defines probability-column semantics. A +predictor returning a matrix `probabilities` must make +`probabilities[:, i]` the probability of +`response_encoding.labels[i]`. The matrix has shape +`(len(items), response_encoding.n_levels)`, contains finite values in the closed +interval `[0, 1]`, and has rows that sum to one. + +## Register a loading factory + +The factory registry accepts a callable, not a subclass. For a provider class +named `LoFIPredictor`, register its native-artifact class method directly: + +```python +from pathlib import Path +from typing import Self + +from bead.serving import PredictorRegistry + + +class LoFIPredictor: + """Example predictor restored from its native artifact.""" + + @classmethod + def load(cls, path: str | Path) -> Self: + return cls() + + +registry = PredictorRegistry[LoFIPredictor]() +registry.register("lofi", LoFIPredictor.load) + +predictor = registry.load("lofi", "artifacts/judgment-model") + +assert isinstance(predictor, LoFIPredictor) +``` + +`LoFIPredictor` only needs to implement the structural protocol. It does not +need to import or subclass a bead implementation class. + +## Run the conformance suite + +Third-party packages should run the published suite in their own tests with +real bead values: + +```python +from pathlib import Path +from typing import Self + +import numpy as np + +from bead.items import Item, ItemTemplate, PresentationSpec, TaskSpec +from bead.protocol import ResponseEncoding, ScaleType +from bead.serving import NLICase, PredictionCase, run_predictor_conformance + + +class ExamplePredictor: + """Minimal complete serving predictor.""" + + model_name = "example" + model_version = "1.0.0" + supported_task_types = ("binary",) + response_encoding = ResponseEncoding( + name="inference", + n_levels=2, + scale_type=ScaleType.BINARY, + labels=("no", "yes"), + ) + + def validate_item_compatibility( + self, item: Item, item_template: ItemTemplate + ) -> None: + if item.item_template_id != item_template.id: + raise ValueError("item and template do not match") + + def predict_proba( + self, + items: list[Item], + participant_ids: list[str] | None = None, + ) -> np.ndarray: + return np.tile([0.25, 0.75], (len(items), 1)) + + def save(self, path: str | Path) -> None: + Path(path).mkdir(parents=True) + + @classmethod + def load(cls, path: str | Path) -> Self: + if not Path(path).is_dir(): + raise FileNotFoundError(path) + return cls() + + def compute_nli(self, premise: str, hypothesis: str) -> dict[str, float]: + return {"entailment": 0.7, "neutral": 0.2, "contradiction": 0.1} + + +item_template = ItemTemplate( + name="binary-inference", + judgment_type="inference", + task_type="binary", + task_spec=TaskSpec( + prompt="Does the hypothesis follow?", + options=("no", "yes"), + ), + presentation_spec=PresentationSpec(), +) +item = Item(item_template_id=item_template.id, rendered_elements={}) +predictor = ExamplePredictor() + +report = run_predictor_conformance( + predictor, + prediction_cases=[ + PredictionCase(item=item, item_template=item_template), + ], + response_encoding=predictor.response_encoding, + nli_cases=[ + NLICase( + premise="A dog is running.", + hypothesis="An animal is moving.", + ), + ], +) + +assert report.checks == ( + "identity", + "task_compatibility", + "predict_proba", + "persistence", + "nli", +) +``` + +The suite verifies the item-template link with bead's standard item validator, +checks that the encoding maps to the template task type, calls compatibility +validation, tests prediction with and without participant identifiers, validates +probability shape and normalization, and requires prediction and NLI equality +after a native save-load round trip. diff --git a/mkdocs.yml b/mkdocs.yml index c30d6a4..2de4931 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -79,6 +79,7 @@ nav: - Layers Interoperability: user-guide/api/layers-interop.md - Deployment: user-guide/api/deployment.md - Training: user-guide/api/training.md + - Read-only Predictors: user-guide/api/serving.md - Workflows: user-guide/api/workflows.md - CLI Reference: cli/reference.md - API Reference: @@ -92,6 +93,7 @@ nav: - bead.corpus: api/corpus.md - bead.interop: api/interop.md - bead.active_learning: api/active_learning.md + - bead.serving: api/serving.md - bead.simulation: api/simulation.md - bead.evaluation: api/evaluation.md - bead.labels: api/labels.md diff --git a/tests/serving/__init__.py b/tests/serving/__init__.py new file mode 100644 index 0000000..b345bd4 --- /dev/null +++ b/tests/serving/__init__.py @@ -0,0 +1 @@ +"""Tests for structural serving capabilities.""" diff --git a/tests/serving/test_serving.py b/tests/serving/test_serving.py new file mode 100644 index 0000000..b84c456 --- /dev/null +++ b/tests/serving/test_serving.py @@ -0,0 +1,195 @@ +"""Tests for read-only predictor protocols, registration, and conformance.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest + +from bead.items import Item, ItemTemplate, PresentationSpec, TaskSpec +from bead.protocol import ResponseEncoding, ScaleType +from bead.serving import ( + ConformanceError, + ModelIdentity, + NLICapability, + NLICase, + PersistablePredictor, + PredictionCase, + PredictorRegistry, + ProbabilisticPredictor, + ServingPredictor, + TaskCompatibility, + run_predictor_conformance, +) +from bead.serving.protocols import NLIProbabilities + + +class ExternalPredictor: + """Structural predictor with no bead implementation base class.""" + + def __init__(self, *, invalid_probabilities: bool = False) -> None: + self.model_name = "external-test-model" + self.model_version = "1.2.3" + self.response_encoding = ResponseEncoding( + name="inference", + n_levels=2, + scale_type=ScaleType.BINARY, + labels=("no", "yes"), + ) + self.supported_task_types = ("binary",) + self.invalid_probabilities = invalid_probabilities + + def validate_item_compatibility( + self, item: Item, item_template: ItemTemplate + ) -> None: + if item.item_template_id != item_template.id: + raise ValueError("item and template do not match") + if item_template.task_type not in self.supported_task_types: + raise ValueError(f"unsupported task type: {item_template.task_type}") + + def predict_proba( + self, + items: list[Item], + participant_ids: list[str] | None = None, + ) -> np.ndarray: + if participant_ids is not None and len(participant_ids) != len(items): + raise ValueError("participant_ids length does not match items") + row = [0.8, 0.8] if self.invalid_probabilities else [0.25, 0.75] + return np.asarray([row for _item in items], dtype=np.float64) + + def save(self, path: str | Path) -> None: + artifact_path = Path(path) + artifact_path.mkdir(parents=True, exist_ok=True) + state = { + "model_name": self.model_name, + "model_version": self.model_version, + "invalid_probabilities": self.invalid_probabilities, + } + (artifact_path / "predictor.json").write_text( + json.dumps(state), encoding="utf-8" + ) + + @classmethod + def load(cls, path: str | Path) -> ExternalPredictor: + state = json.loads((Path(path) / "predictor.json").read_text(encoding="utf-8")) + predictor = cls(invalid_probabilities=state["invalid_probabilities"]) + predictor.model_name = state["model_name"] + predictor.model_version = state["model_version"] + return predictor + + def compute_nli(self, premise: str, hypothesis: str) -> NLIProbabilities: + return { + "entailment": 0.7, + "neutral": 0.2, + "contradiction": 0.1, + } + + +@pytest.fixture +def binary_case() -> PredictionCase: + template = ItemTemplate( + name="binary-inference", + judgment_type="inference", + task_type="binary", + task_spec=TaskSpec(prompt="Does the hypothesis follow?", options=("no", "yes")), + presentation_spec=PresentationSpec(), + ) + item = Item( + item_template_id=template.id, + rendered_elements={}, + ) + return PredictionCase(item=item, item_template=template) + + +def test_external_predictor_satisfies_independent_protocols() -> None: + predictor = ExternalPredictor() + + assert isinstance(predictor, ModelIdentity) + assert isinstance(predictor, TaskCompatibility) + assert isinstance(predictor, ProbabilisticPredictor) + assert isinstance(predictor, PersistablePredictor) + assert isinstance(predictor, NLICapability) + assert isinstance(predictor, ServingPredictor) + + +def test_registry_accepts_structural_loading_factory(tmp_path: Path) -> None: + original = ExternalPredictor() + artifact_path = tmp_path / "native-artifact" + original.save(artifact_path) + + registry = PredictorRegistry[ServingPredictor]() + registry.register("external", ExternalPredictor.load) + restored = registry.load("external", artifact_path) + + assert registry.list_factories() == ("external",) + assert isinstance(restored, ExternalPredictor) + assert restored.model_name == original.model_name + assert restored.model_version == original.model_version + + +def test_registry_reports_unknown_factory(tmp_path: Path) -> None: + registry = PredictorRegistry[ServingPredictor]() + + with pytest.raises(KeyError, match="Unknown predictor factory"): + registry.load("missing", tmp_path / "artifact") + + +def test_full_conformance_suite(binary_case: PredictionCase) -> None: + predictor = ExternalPredictor() + + report = run_predictor_conformance( + predictor, + prediction_cases=[binary_case], + response_encoding=predictor.response_encoding, + nli_cases=[NLICase("A dog runs.", "An animal moves.")], + ) + + assert report.model_name == "external-test-model" + assert report.model_version == "1.2.3" + assert report.n_items == 1 + assert report.n_levels == 2 + assert report.n_nli_cases == 1 + assert report.checks == ( + "identity", + "task_compatibility", + "predict_proba", + "persistence", + "nli", + ) + + +def test_conformance_rejects_unnormalized_probabilities( + binary_case: PredictionCase, +) -> None: + predictor = ExternalPredictor(invalid_probabilities=True) + + with pytest.raises(ConformanceError, match="do not sum to one"): + run_predictor_conformance( + predictor, + prediction_cases=[binary_case], + response_encoding=predictor.response_encoding, + nli_cases=[NLICase("A dog runs.", "An animal moves.")], + ) + + +def test_conformance_rejects_encoding_task_mismatch( + binary_case: PredictionCase, +) -> None: + predictor = ExternalPredictor() + ordinal_encoding = ResponseEncoding( + name="rating", + n_levels=3, + scale_type=ScaleType.ORDINAL, + labels=("low", "middle", "high"), + ) + predictor.response_encoding = ordinal_encoding + + with pytest.raises(ConformanceError, match="maps to 'ordinal_scale'"): + run_predictor_conformance( + predictor, + prediction_cases=[binary_case], + response_encoding=ordinal_encoding, + nli_cases=[NLICase("A dog runs.", "An animal moves.")], + ) From 7e894b072e606d33e910a169069a27625772ca2b Mon Sep 17 00:00:00 2001 From: Aaron Steven White Date: Sun, 25 Oct 2026 10:30:20 -0400 Subject: [PATCH 2/2] Releases 0.11.0 with structural serving capabilities. --- CHANGELOG.md | 5 ++++- pyproject.toml | 2 +- uv.lock | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ca10e6..7bd7889 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.11.0] - 2026-10-25 + ### Added - `bead.serving` publishes independent runtime protocols for model identity, @@ -732,7 +734,8 @@ guards as type-checkers. - CI/CD: GitHub Actions for testing, docs, PyPI publishing - Read the Docs integration -[Unreleased]: https://github.com/FACTSlab/bead/compare/v0.10.0...HEAD +[Unreleased]: https://github.com/FACTSlab/bead/compare/v0.11.0...HEAD +[0.11.0]: https://github.com/FACTSlab/bead/compare/v0.10.0...v0.11.0 [0.10.0]: https://github.com/FACTSlab/bead/compare/v0.9.0...v0.10.0 [0.9.0]: https://github.com/FACTSlab/bead/compare/v0.8.0...v0.9.0 [0.8.0]: https://github.com/FACTSlab/bead/compare/v0.7.0...v0.8.0 diff --git a/pyproject.toml b/pyproject.toml index 57c2c74..08fd250 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "bead" -version = "0.10.0" +version = "0.11.0" description = "Lexicon and Template Collection Construction Pipeline for Acceptability and Inference Judgment Data" authors = [{name = "Aaron Steven White", email = "aaron.white@rochester.edu"}] readme = "README.md" diff --git a/uv.lock b/uv.lock index 5458b22..263b268 100644 --- a/uv.lock +++ b/uv.lock @@ -170,7 +170,7 @@ wheels = [ [[package]] name = "bead" -version = "0.10.0" +version = "0.11.0" source = { editable = "." } dependencies = [ { name = "accelerate" },