From 424790b5e58774798f45821d220856f7212e84fb Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 20 Oct 2026 18:09:04 -0400 Subject: [PATCH 1/2] Foundation: enforce the Images API's 280 px minimum, hash image bytes in replay keys, bound tiles (#306) Co-Authored-By: Claude Sonnet 5.5 --- docs/backends.md | 8 +- docs/vision.md | 22 +++-- src/jev_cookbook/backends.py | 15 ++-- src/jev_cookbook/vision.py | 75 ++++++++++++++--- tests/test_multimodal_backends.py | 80 +++++++++++++++++- tests/test_vision.py | 131 ++++++++++++++++++++++++++++-- 6 files changed, 299 insertions(+), 32 deletions(-) diff --git a/docs/backends.md b/docs/backends.md index 9ed7fa3..f9a1772 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -400,8 +400,12 @@ Keep image files under `assets/` beside `fixtures/`, and put their relative path The text-only replay envelope is unchanged: omitted, `None` and empty assets produce exactly the existing keys. A request with images adds an `assets` object sorted by reference, holding -each asset's `type`, canonical `format` (`png` or `jpeg`) and SHA-256 of its base64 ASCII text. -That means changed image +each asset's `type`, canonical `format` (`png` or `jpeg`) and `sha256`, the lowercase hex +SHA-256 of the **decoded file bytes** (the same digest `image_reference` embeds), not of the +base64 text. Base64 that decodes to the same bytes but is spelled differently (for example +non-canonical trailing bits, `QQ==` and `QR==`) therefore keys once. This is the envelope +every image fixture binds to; no image fixture existed before it was fixed, so no recorded +key changed (text-only keys are untouched and pinned by a test). Changed image bytes or references change the key, while reordering the assets map does not. No image data is stored in a response fixture. The scripted random seed includes this key; scripts retain their existing `(state, questions, rng)` interface. diff --git a/docs/vision.md b/docs/vision.md index 27737d8..2c3ce3d 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -40,8 +40,12 @@ contained real path for code that opens files itself. decoded, still-encoded PNG/JPEG file bytes. Neither function changes those bytes. The accepted local files are PNG or JPEG stills, at most 16 MiB, with each -dimension 256–1568 and total area 78,400–2,458,624 pixels. A 384 × 320 fixture fits; -a 256 × 256 fixture does not. The loader rejects orientation metadata requiring a +dimension 280–1568 and total area 78,400–2,458,624 pixels (all ranges include both ends; +the same numbers are `vision.IMAGE_LIMITS`). A 384 × 320 or a 280 × 280 fixture fits; a +256 × 256 or a 279 × 400 fixture does not, and neither does 400 × 279: the API rejects an +image with either side under 280 pixels, never resizing it, and rejects it only after the request +is sent, so the loader refuses it first. Scale up or pad a smaller image before saving it. +The loader rejects orientation metadata requiring a rotation: prepare upright pixels explicitly with Pillow's `ImageOps.exif_transpose` before saving. Limits follow the experimental [Images documentation](https://docs.typesafe.ai/images). @@ -115,7 +119,9 @@ backgrounds and ink text from the cookbook palette. `box_iou(first, second)` computes intersection divided by union, using continuous normalized geometry before pixel rounding. It returns zero for disjoint or merely -touching rectangles and one for identical rectangles. +touching rectangles and one for identical rectangles. A valid but vanishingly small box +(for example 1e-200 wide) has an area that underflows to zero; when both do, the union is +zero and `box_iou` (and `localization_metrics`) raise `ValueError` rather than invent a score. ## Presence, confidence and evaluation @@ -123,7 +129,8 @@ A Box returns one rectangle even when its target is absent. Determine presence with a companion question before treating a rectangle as a detection. [Box documentation](https://docs.typesafe.ai/primitives/box) -`box_confidence(answer)` returns the supplied confidence or `None` if missing/null; +`box_confidence(answer)` returns the supplied confidence, including a reported `0.0` +or `1.0`, or `None` if missing/null (absence is never turned into a number); invalid confidence values raise. A missing value cannot pass a confidence gate: ```python @@ -183,7 +190,12 @@ with Image.open("assets/screenshot.png") as image: `.bounds` in the original image, in row-major order. `overlap` is an integer number of pixels smaller than both tile dimensions. The last row/column anchors to the bottom/right edge, which can increase overlap; there are no uncovered gaps or -partial sliver tiles. A source smaller than a requested tile keeps its full extent. +partial sliver tiles. A source smaller than a requested tile keeps its full extent, so a +tile (or a source) under 280 pixels on a side must be scaled up or padded before +`image_asset` accepts it. The tile count is computed before any tile is cropped and +`max_tiles` (default 256) bounds it: an overlap near the tile size makes the count grow +roughly with its square, and a configuration above the cap raises `ValueError`. +Pass a larger `max_tiles=` only deliberately. `tile_to_image` applies an affine transform using the actual pixel bounds, so a rectangle found inside a tile maps back to the whole image. It also handles an diff --git a/src/jev_cookbook/backends.py b/src/jev_cookbook/backends.py index b17dad7..5e433c8 100644 --- a/src/jev_cookbook/backends.py +++ b/src/jev_cookbook/backends.py @@ -139,10 +139,15 @@ def _check_assets( raise ValueError( f"asset {ref!r} is not referenced in a state/instructions/criteria value" ) - validate_image_asset(asset) + data = validate_image_asset(asset) # Canonical format ("jpg" -> "jpeg", lower case) so the replay key and every consumer - # see one spelling per set of bytes. - checked[ref] = {**asset, "format": canonical_image_format(asset["format"])} + # see one spelling per set of bytes. The digest is of the decoded file bytes, so + # non-canonical base64 of the same bytes ("QQ==" and "QR==") keys identically. + checked[ref] = { + **asset, + "format": canonical_image_format(asset["format"]), + "sha256": hashlib.sha256(data).hexdigest(), + } if reference_bytes > 65536: raise ValueError("all asset references together must be at most 65536 UTF-8 bytes") return checked @@ -158,7 +163,7 @@ def replay_key( Choice options) and every array in the order written; no whitespace, ASCII escapes, shortest-repr floats. Non-empty ``assets`` adds a sorted map of reference to asset type, canonical format (``png`` or ``jpeg``; ``jpg`` and case variants are the same key) and - SHA-256 of its base64 text. Text-only requests retain their original keys. + SHA-256 of its decoded file bytes (the digest inside ``vision.image_reference``). Text-only requests retain their original keys. """ plain_state, checked = _check_request(state, questions) questions_json = {} @@ -181,7 +186,7 @@ def replay_key( ref: { "type": asset["type"], "format": asset["format"], - "sha256": hashlib.sha256(asset["data_base64"].encode("ascii")).hexdigest(), + "sha256": asset["sha256"], } for ref, asset in sorted(checked_assets.items()) } diff --git a/src/jev_cookbook/vision.py b/src/jev_cookbook/vision.py index 5bb4ff0..13a6d48 100644 --- a/src/jev_cookbook/vision.py +++ b/src/jev_cookbook/vision.py @@ -24,6 +24,8 @@ from .style import INK, PAPER, PINK __all__ = [ + "DEFAULT_MAX_TILES", + "IMAGE_LIMITS", "AssetPathError", "ImageTile", "LocalizationMetrics", @@ -46,7 +48,23 @@ "validate_image_asset", ] -_MAX_IMAGE_BYTES = 16 * 1024 * 1024 +# The Images API limits this module enforces locally. Source of truth: the "Limits" table at +# https://docs.typesafe.ai/images (fetched 2026-10-20): "Width and height: 280 to 1568 pixels +# each", "Total pixels (width x height): 78,400 to 2,458,624" and "16 MiB per image file". All +# ranges include both ends. tests/test_vision.py pins these numbers against docs/vision.md. +IMAGE_LIMITS = { + "min_side": 280, + "max_side": 1568, + "min_area": 78_400, + "max_area": 2_458_624, + "max_bytes": 16 * 1024 * 1024, +} +_MIN_SIDE = IMAGE_LIMITS["min_side"] +_MAX_SIDE = IMAGE_LIMITS["max_side"] +_MIN_AREA = IMAGE_LIMITS["min_area"] +_MAX_AREA = IMAGE_LIMITS["max_area"] +_MAX_IMAGE_BYTES = IMAGE_LIMITS["max_bytes"] +DEFAULT_MAX_TILES = 256 _FORMATS = {"PNG": "png", "JPEG": "jpeg"} _EDGES = ("top", "bottom", "left", "right") PixelBounds = tuple[int, int, int, int] @@ -63,10 +81,12 @@ def _image_format(data: bytes) -> str: if getattr(image, "is_animated", False): raise ValueError("image must be a still image, not an animation") width, height = image.size - if not (256 <= width <= 1568 and 256 <= height <= 1568): - raise ValueError("image width and height must each be 256 to 1568 pixels") - if not 78_400 <= width * height <= 2_458_624: - raise ValueError("image area must be 78,400 to 2,458,624 pixels") + if not (_MIN_SIDE <= width <= _MAX_SIDE and _MIN_SIDE <= height <= _MAX_SIDE): + raise ValueError( + f"image width and height must each be {_MIN_SIDE} to {_MAX_SIDE} pixels" + ) + if not _MIN_AREA <= width * height <= _MAX_AREA: + raise ValueError(f"image area must be {_MIN_AREA:,} to {_MAX_AREA:,} pixels") image.verify() # verify() checks file integrity; load() also catches truncated pixel data. with Image.open(io.BytesIO(data)) as image: @@ -84,8 +104,9 @@ def _image_format(data: bytes) -> str: def validate_image_asset(asset: Mapping[str, Any]) -> bytes: """Validate a raw image asset and return its original encoded bytes. - Checks actual PNG/JPEG content, stillness, orientation, dimensions and the 16 MiB - file limit. This does not certify aggregate request limits or API acceptance. + Checks actual PNG/JPEG content, stillness, orientation, dimensions (each side 280 to + 1568 pixels, area 78,400 to 2,458,624; a side under 280 is rejected by the API) and the + 16 MiB file limit. This does not certify aggregate request limits or API acceptance. """ if not isinstance(asset, Mapping) or set(asset) != {"type", "format", "data_base64"}: raise ValueError("image asset needs exactly type, format and data_base64") @@ -275,14 +296,22 @@ def normalized_to_pixels(box: Any, width: int, height: int) -> PixelBounds: def box_iou(first: Any, second: Any) -> float: - """Intersection area divided by union area of two valid normalized rectangles.""" + """Intersection area divided by union area of two valid normalized rectangles. + + Raises ``ValueError`` when both areas are so small that the union underflows to zero. + """ a, b = normalized_box(first), normalized_box(second) intersection = max(0.0, min(a["right"], b["right"]) - max(a["left"], b["left"])) * max( 0.0, min(a["bottom"], b["bottom"]) - max(a["top"], b["top"]) ) area_a = (a["right"] - a["left"]) * (a["bottom"] - a["top"]) area_b = (b["right"] - b["left"]) * (b["bottom"] - b["top"]) - return intersection / (area_a + area_b - intersection) + union = area_a + area_b - intersection + if union <= 0.0: + # A valid box can be so small that its area underflows to 0.0; the ratio is then + # undefined, and 0.0 would wrongly call two identical boxes disjoint. + raise ValueError("box areas are too small to compare: the union underflows to zero") + return intersection / union def expand_box(box: Any, fraction: float = 0.0) -> dict[str, float]: @@ -342,7 +371,11 @@ class ImageTile: def generate_tiles( - image: Image.Image, *, tile_size: tuple[int, int] = (512, 512), overlap: int = 0 + image: Image.Image, + *, + tile_size: tuple[int, int] = (512, 512), + overlap: int = 0, + max_tiles: int = DEFAULT_MAX_TILES, ) -> list[ImageTile]: """Cover an image in row-major order with tiles of at most ``(width, height)``. @@ -350,7 +383,12 @@ def generate_tiles( right/bottom boundary, so they may overlap more. Images smaller than a tile yield their full extent. Tile sizes are geometry choices; validate saved tiles with :func:`image_asset` before sending them to the API. + + The number of tiles grows quadratically as ``overlap`` approaches the tile size, so the + tile count is computed first and a configuration above ``max_tiles`` (default 256) + raises ``ValueError`` before any tile is cropped. Raise ``max_tiles`` deliberately. """ + max_tiles = _positive_int(max_tiles, "max_tiles") try: tile_width, tile_height = tile_size except (TypeError, ValueError) as exc: @@ -375,9 +413,16 @@ def starts(length: int, window: int) -> list[int]: positions.append(length - window) return positions + rows, columns = starts(height, tile_height), starts(width, tile_width) + if len(rows) * len(columns) > max_tiles: + raise ValueError( + f"tile_size={tile_width, tile_height} with overlap={overlap} would make " + f"{len(rows) * len(columns):,} tiles, above max_tiles={max_tiles:,}; " + "use a larger tile, less overlap or a larger max_tiles" + ) tiles = [] - for top in starts(height, tile_height): - for left in starts(width, tile_width): + for top in rows: + for left in columns: bounds = (left, top, min(left + tile_width, width), min(top + tile_height, height)) tiles.append(ImageTile(bounds, image.crop(bounds))) return tiles @@ -400,7 +445,11 @@ def draw_boxes(image: Image.Image, boxes: Mapping[str, Any]) -> Image.Image: def box_confidence(answer: Any) -> float | None: - """Read confidence conservatively: absent or null stays ``None``, never zero or one.""" + """Read confidence conservatively: absent or null stays ``None``, never a number. + + A reported ``0.0`` or ``1.0`` is returned as reported; only absence is never coerced + into a confidence. A non-numeric or out-of-range value raises ``ValueError``. + """ value = ( answer.get("confidence") if isinstance(answer, Mapping) diff --git a/tests/test_multimodal_backends.py b/tests/test_multimodal_backends.py index 15b353b..635cc3d 100644 --- a/tests/test_multimodal_backends.py +++ b/tests/test_multimodal_backends.py @@ -3,8 +3,10 @@ from __future__ import annotations import base64 +import hashlib import io import json +import string from types import SimpleNamespace import pytest @@ -237,7 +239,7 @@ def test_invalid_assets_are_rejected(change): def test_image_dimensions_and_reference_limits(): with pytest.raises(ValueError): - replay_key(STATE, QUESTIONS, assets=image_assets(size=(256, 256))) + replay_key(STATE, QUESTIONS, assets=image_assets(size=(279, 400))) for ref in ["", " ", "x" * 1025, "🖼" * 257]: with pytest.raises(ValueError): replay_key(ref, {"n": Noul()}, assets={ref: image_assets()[REF]}) @@ -380,3 +382,79 @@ def test_image_metadata_fixtures_validate_box_answers(tmp_path): responses[examples[0].replay_keys[0]]["answers"]["target"]["box"]["left"] = 1.0 (folder / "responses.json").write_text(json.dumps(responses)) assert any("left < right" in str(problem) for problem in validate_recipe(tmp_path)) + + +def _noncanonical_twin(asset): + """Same decoded bytes, different base64 text (spare low bits set in the final group).""" + encoded = asset["data_base64"] + data = base64.b64decode(encoded) + assert len(data) % 3 != 0, "need a padded encoding to vary its trailing bits" + alphabet = string.ascii_uppercase + string.ascii_lowercase + string.digits + "+/" + body = encoded.rstrip("=") + pad = encoded[len(body) :] + spare = 0b1111 if len(pad) == 2 else 0b11 + twin = body[:-1] + alphabet[alphabet.index(body[-1]) | spare] + pad + assert twin != encoded and base64.b64decode(twin, validate=True) == data + return {**asset, "data_base64": twin} + + +def _padded_png(size=(280, 280)): + """A PNG whose length is not a multiple of 3, so its base64 ends in padding.""" + for extra in range(8): + data = io.BytesIO() + Image.new("RGB", size, "pink").save(data, format="PNG", comment=b"x" * extra) + if len(data.getvalue()) % 3 != 0: + return { + "type": "image", + "format": "png", + "data_base64": base64.b64encode(data.getvalue()).decode("ascii"), + } + raise AssertionError("could not build a padded PNG") + + +def test_non_canonical_base64_of_identical_bytes_yields_one_replay_key(): + assert base64.b64decode("QQ==", validate=True) == base64.b64decode("QR==", validate=True) + original = _padded_png() + twin = _noncanonical_twin(original) + assert twin["data_base64"] != original["data_base64"] + assert replay_key(STATE, QUESTIONS, assets={REF: twin}) == replay_key( + STATE, QUESTIONS, assets={REF: original} + ) + + +def test_a_recording_replays_for_non_canonical_base64_of_the_same_bytes(): + original = _padded_png() + key = replay_key(STATE, QUESTIONS, assets={REF: original}) + replay = ReplayBackend({key: box_result().to_dict()}) + twin = _noncanonical_twin(original) + assert replay.decide(STATE, QUESTIONS, assets={REF: twin}) == box_result() + + +def test_image_key_hashes_decoded_bytes_in_the_documented_envelope(): + # The documented envelope: {"v": 1, state, questions, "assets": {ref: {type, format, + # sha256 of the decoded file bytes}}}, canonical JSON, SHA-256. Rebuilt here independently. + assets = image_assets() + data = base64.b64decode(assets[REF]["data_base64"]) + payload = { + "v": 1, + "state": STATE, + "questions": {"target": QUESTIONS["target"].to_dict()}, + "assets": { + REF: {"type": "image", "format": "png", "sha256": hashlib.sha256(data).hexdigest()} + }, + } + expected = hashlib.sha256( + json.dumps(payload, separators=(",", ":"), ensure_ascii=True).encode() + ).hexdigest() + assert replay_key(STATE, QUESTIONS, assets=assets) == expected + # image_reference's digest is the same digest the key embeds. + from jev_cookbook.vision import image_reference + + assert image_reference(assets[REF]) == "image:sha256:" + hashlib.sha256(data).hexdigest() + + +@pytest.mark.parametrize("size", [(279, 400), (400, 279)]) +def test_replay_key_refuses_a_side_under_280_on_either_axis(size): + with pytest.raises(ValueError, match="280 to 1568"): + replay_key(STATE, QUESTIONS, assets=image_assets(size=size)) + replay_key(STATE, QUESTIONS, assets=image_assets(size=(280, 280))) diff --git a/tests/test_vision.py b/tests/test_vision.py index b812019..e2a31f6 100644 --- a/tests/test_vision.py +++ b/tests/test_vision.py @@ -4,6 +4,7 @@ import base64 import hashlib +import io import json import math import random @@ -22,7 +23,7 @@ BOX = {"top": 0.25, "bottom": 0.75, "left": 0.125, "right": 0.625} -def save_image(tmp_path, *, size=(320, 256), fmt="PNG", name="image", color="white", **kwargs): +def save_image(tmp_path, *, size=(320, 280), fmt="PNG", name="image", color="white", **kwargs): path = tmp_path / name Image.new("RGB", size, color).save(path, format=fmt, **kwargs) return path @@ -54,17 +55,71 @@ def test_image_asset_uses_actual_format_and_preserves_encoded_bytes(tmp_path, fm assert vision.validate_image_asset(result) == path.read_bytes() -@pytest.mark.parametrize("size", [(280, 280), (256, 307), (1568, 1568)]) +@pytest.mark.parametrize("size", [(280, 280), (280, 1568), (1568, 280), (1568, 1568), (320, 280)]) def test_image_asset_accepts_boundary_dimensions(tmp_path, size): assert vision.image_asset(save_image(tmp_path, size=size))["type"] == "image" -@pytest.mark.parametrize("size", [(255, 400), (400, 255), (256, 306), (1569, 256), (256, 1569)]) +@pytest.mark.parametrize( + "size", + [ + (279, 400), # width one under the 280 minimum + (400, 279), # height one under the 280 minimum + (279, 279), + (256, 307), # accepted before the minimum was corrected from 256 to 280 + (255, 400), + (400, 255), + (1569, 280), + (280, 1569), + ], +) def test_image_asset_rejects_invalid_dimensions(tmp_path, size): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="width and height must each be 280 to 1568"): vision.image_asset(save_image(tmp_path, size=size)) +@pytest.mark.parametrize("size", [(279, 400), (400, 279)]) +def test_validate_image_asset_rejects_a_279_pixel_side_on_either_axis(tmp_path, size): + buffer = io.BytesIO() + Image.new("RGB", size, "white").save(buffer, format="PNG") + encoded = base64.b64encode(buffer.getvalue()).decode("ascii") + with pytest.raises(ValueError, match="280 to 1568"): + vision.validate_image_asset({"type": "image", "format": "png", "data_base64": encoded}) + # 280 on the same axis is accepted. + good = (280, size[1] + 1) if size[0] == 279 else (size[0] + 1, 280) + buffer = io.BytesIO() + Image.new("RGB", good, "white").save(buffer, format="PNG") + encoded = base64.b64encode(buffer.getvalue()).decode("ascii") + asset = {"type": "image", "format": "png", "data_base64": encoded} + assert vision.validate_image_asset(asset) == buffer.getvalue() + + +# The upstream numbers, copied by hand from the "Limits" table at +# https://docs.typesafe.ai/images (fetched 2026-10-20). If the page changes, update this +# table, vision.IMAGE_LIMITS and docs/vision.md together. +UPSTREAM_LIMITS = { + "min_side": 280, + "max_side": 1568, + "min_area": 78_400, + "max_area": 2_458_624, + "max_bytes": 16 * 1024 * 1024, +} + + +def test_image_limits_match_the_upstream_table_and_the_docs(): + assert vision.IMAGE_LIMITS == UPSTREAM_LIMITS + assert UPSTREAM_LIMITS["min_area"] == UPSTREAM_LIMITS["min_side"] ** 2 + assert UPSTREAM_LIMITS["max_area"] == UPSTREAM_LIMITS["max_side"] ** 2 + docs = (Path(__file__).resolve().parents[1] / "docs" / "vision.md").read_text("utf-8") + flat = " ".join(docs.split()) + sides = f"{UPSTREAM_LIMITS['min_side']}–{UPSTREAM_LIMITS['max_side']}" + areas = f"{UPSTREAM_LIMITS['min_area']:,}–{UPSTREAM_LIMITS['max_area']:,}" + assert f"dimension {sides} and total area {areas} pixels" in flat + assert "at most 16 MiB" in flat + assert "256–1568" not in flat + assert "280 to 1568" in " ".join((vision.validate_image_asset.__doc__ or "").split()) + + def test_image_asset_missing_file_and_url(tmp_path): with pytest.raises(FileNotFoundError): vision.image_asset(tmp_path / "missing.png") @@ -90,10 +145,10 @@ def test_image_asset_rejects_corruption_and_oversized_file(tmp_path, image_path) def test_image_asset_rejects_animated_png(tmp_path): path = tmp_path / "animated.png" - Image.new("RGB", (320, 256), "white").save( + Image.new("RGB", (320, 280), "white").save( path, save_all=True, - append_images=[Image.new("RGB", (320, 256), "black")], + append_images=[Image.new("RGB", (320, 280), "black")], duration=100, loop=0, ) @@ -750,3 +805,67 @@ def test_run_record_rejects_invalid_explicit_measurements(value, field): def test_run_record_rejects_unknown_mode(): with pytest.raises(ValueError): vision.run_record(SimpleNamespace(), mode="unknown") + + +TINY = {"top": 0.0, "bottom": 1e-200, "left": 0.0, "right": 1e-200} + + +@pytest.mark.parametrize("other", [TINY, {**TINY, "right": 2e-200}]) +def test_box_iou_raises_a_value_error_when_both_areas_underflow(other): + # Valid boxes (positive width and height) whose areas are 0.0 in floating point. + assert vision.normalized_box(TINY) == TINY + assert (TINY["right"] - TINY["left"]) * (TINY["bottom"] - TINY["top"]) == 0.0 + with pytest.raises(ValueError, match="union underflows"): + vision.box_iou(TINY, other) + + +def test_localization_metrics_propagates_the_underflow_as_a_value_error(): + with pytest.raises(ValueError, match="union underflows"): + vision.localization_metrics([TINY], [TINY]) + + +def test_box_iou_with_one_underflowing_box_is_still_defined(): + assert vision.box_iou(TINY, BOX) == 0.0 + assert vision.box_iou(BOX, TINY) == 0.0 + + +def test_generate_tiles_refuses_a_configuration_above_the_tile_cap_before_cropping(monkeypatch): + image = Image.new("RGB", (1568, 1568)) + # The reviewed case that exhausted memory: overlap 511 on a 512 tile is ~1.1 million tiles. + monkeypatch.setattr( + Image.Image, "crop", lambda *a, **k: pytest.fail("cropped before checking the tile cap") + ) + with pytest.raises(ValueError, match="would make 1,[0-9,]+ tiles, above max_tiles=256"): + vision.generate_tiles(image, tile_size=(512, 512), overlap=511) + with pytest.raises(ValueError, match="above max_tiles=4"): + vision.generate_tiles(image, tile_size=(512, 512), max_tiles=4) + + +def test_generate_tiles_cap_is_inclusive_and_can_be_raised(): + image = Image.new("RGB", (12, 8)) + count = len(vision.generate_tiles(image, tile_size=(4, 4))) + assert count == 6 and vision.DEFAULT_MAX_TILES >= count + assert len(vision.generate_tiles(image, tile_size=(4, 4), max_tiles=6)) == 6 + with pytest.raises(ValueError, match="above max_tiles=5"): + vision.generate_tiles(image, tile_size=(4, 4), max_tiles=5) + # A deliberate larger cap permits a heavily overlapping grid that the default refuses. + crowded = Image.new("RGB", (60, 60)) + with pytest.raises(ValueError, match="max_tiles=256"): + vision.generate_tiles(crowded, tile_size=(4, 4), overlap=3) + tiles = vision.generate_tiles(crowded, tile_size=(4, 4), overlap=3, max_tiles=10_000) + assert len(tiles) == 57 * 57 + + +@pytest.mark.parametrize("max_tiles", [0, -1, True, 2.5, "9"]) +def test_generate_tiles_rejects_invalid_max_tiles(max_tiles): + with pytest.raises(ValueError, match="max_tiles"): + vision.generate_tiles(Image.new("RGB", (10, 7)), tile_size=(4, 3), max_tiles=max_tiles) + + +def test_box_confidence_docstring_matches_its_behaviour(): + doc = vision.box_confidence.__doc__ or "" + assert "never zero or one" not in doc + assert "0.0" in doc and "1.0" in doc + assert vision.box_confidence({"confidence": 0.0}) == 0.0 + assert vision.box_confidence({"confidence": 1.0}) == 1.0 + assert vision.box_confidence({}) is None From fd39eb8dee587e83c2945fa6962316c1818f5189 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 20 Oct 2026 21:14:56 -0400 Subject: [PATCH 2/2] Foundation: state the key-version rule, fix tile-growth wording, pin an image replay key (#306) Co-Authored-By: Claude Sonnet 5.5 --- docs/backends.md | 8 +++++--- docs/vision.md | 9 ++++++--- src/jev_cookbook/backends.py | 3 ++- src/jev_cookbook/vision.py | 12 ++++++++---- tests/test_multimodal_backends.py | 17 +++++++++++++++++ 5 files changed, 38 insertions(+), 11 deletions(-) diff --git a/docs/backends.md b/docs/backends.md index f9a1772..da582c6 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -292,7 +292,10 @@ styles (`\r\n`, `\r`, `\n`) differ, and Unicode is not re-normalized. It never u What changes the key: the state (including key order), question names, each question's type, `instructions`, and `criteria` (Score level order and Choice option order matter; Noul criteria with all-`None` values equal no criteria). What does not: question order. The key version `v` is -bumped if this rule ever changes. +bumped if this rule changes after a fixture has been recorded under it. The image `assets` +member (below) was redefined once, before any image fixture existed (its digest moved from the +base64 text to the decoded file bytes), so `v` stayed 1 and no recorded key moved. Any later +change that would alter an existing key, text-only or image, bumps `v`. The state, a question's `instructions`, and each `criteria` value (each Score level, Choice option description, Noul outcome) each nest at most **64 levels** of list/object, counting the @@ -404,8 +407,7 @@ each asset's `type`, canonical `format` (`png` or `jpeg`) and `sha256`, the lowe SHA-256 of the **decoded file bytes** (the same digest `image_reference` embeds), not of the base64 text. Base64 that decodes to the same bytes but is spelled differently (for example non-canonical trailing bits, `QQ==` and `QR==`) therefore keys once. This is the envelope -every image fixture binds to; no image fixture existed before it was fixed, so no recorded -key changed (text-only keys are untouched and pinned by a test). Changed image +every image fixture binds to; text-only keys are untouched and pinned by a test. Changed image bytes or references change the key, while reordering the assets map does not. No image data is stored in a response fixture. The scripted random seed includes this key; scripts retain their existing `(state, questions, rng)` interface. diff --git a/docs/vision.md b/docs/vision.md index 2c3ce3d..f9ebcad 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -122,6 +122,8 @@ normalized geometry before pixel rounding. It returns zero for disjoint or merel touching rectangles and one for identical rectangles. A valid but vanishingly small box (for example 1e-200 wide) has an area that underflows to zero; when both do, the union is zero and `box_iou` (and `localization_metrics`) raise `ValueError` rather than invent a score. +Because the union is at least as large as each area, this happens only when both boxes +underflow, so in `localization_metrics` it means a degenerate gold label: fix the fixture. ## Presence, confidence and evaluation @@ -193,9 +195,10 @@ bottom/right edge, which can increase overlap; there are no uncovered gaps or partial sliver tiles. A source smaller than a requested tile keeps its full extent, so a tile (or a source) under 280 pixels on a side must be scaled up or padded before `image_asset` accepts it. The tile count is computed before any tile is cropped and -`max_tiles` (default 256) bounds it: an overlap near the tile size makes the count grow -roughly with its square, and a configuration above the cap raises `ValueError`. -Pass a larger `max_tiles=` only deliberately. +`max_tiles` (default 256) bounds it: the count grows with the inverse square of the +stride (tile size minus overlap), so an overlap near the tile size makes it explode, and a +configuration above the cap raises `ValueError`. Each tile is usually one API call, so +`max_tiles` also caps cost, not only memory. Pass a larger `max_tiles=` only deliberately. `tile_to_image` applies an affine transform using the actual pixel bounds, so a rectangle found inside a tile maps back to the whole image. It also handles an diff --git a/src/jev_cookbook/backends.py b/src/jev_cookbook/backends.py index 5e433c8..80a5c90 100644 --- a/src/jev_cookbook/backends.py +++ b/src/jev_cookbook/backends.py @@ -163,7 +163,8 @@ def replay_key( Choice options) and every array in the order written; no whitespace, ASCII escapes, shortest-repr floats. Non-empty ``assets`` adds a sorted map of reference to asset type, canonical format (``png`` or ``jpeg``; ``jpg`` and case variants are the same key) and - SHA-256 of its decoded file bytes (the digest inside ``vision.image_reference``). Text-only requests retain their original keys. + SHA-256 of its decoded file bytes (the digest inside ``vision.image_reference``). + Text-only requests retain their original keys. """ plain_state, checked = _check_request(state, questions) questions_json = {} diff --git a/src/jev_cookbook/vision.py b/src/jev_cookbook/vision.py index 13a6d48..6d5148c 100644 --- a/src/jev_cookbook/vision.py +++ b/src/jev_cookbook/vision.py @@ -384,9 +384,11 @@ def generate_tiles( their full extent. Tile sizes are geometry choices; validate saved tiles with :func:`image_asset` before sending them to the API. - The number of tiles grows quadratically as ``overlap`` approaches the tile size, so the - tile count is computed first and a configuration above ``max_tiles`` (default 256) - raises ``ValueError`` before any tile is cropped. Raise ``max_tiles`` deliberately. + The number of tiles grows with the inverse square of the stride (tile size minus + ``overlap``), so the tile count is computed first and a configuration above + ``max_tiles`` (default 256) raises ``ValueError`` before any tile is cropped. Each tile + is usually one API call, so the cap bounds cost as well as memory. Raise ``max_tiles`` + deliberately. """ max_tiles = _positive_int(max_tiles, "max_tiles") try: @@ -480,7 +482,9 @@ def localization_metrics(gold: Iterable[Any], predicted: Iterable[Any]) -> Local Predictions must already have passed the recipe's presence/confidence policy. A review outcome is not an absent prediction: report reviews separately through ``evaluation.evaluate_outcomes`` and use this function on the resolved subset. - Invalid rectangles raise, including predictions on absent gold examples. + Invalid rectangles raise, including predictions on absent gold examples. A + ``ValueError`` from :func:`box_iou` (both areas underflow to zero) means a degenerate + gold label, since the union is at least as large as the gold box's own area. """ actual, guesses = list(gold), list(predicted) if not actual or len(actual) != len(guesses): diff --git a/tests/test_multimodal_backends.py b/tests/test_multimodal_backends.py index 635cc3d..6cba1d4 100644 --- a/tests/test_multimodal_backends.py +++ b/tests/test_multimodal_backends.py @@ -458,3 +458,20 @@ def test_replay_key_refuses_a_side_under_280_on_either_axis(size): with pytest.raises(ValueError, match="280 to 1568"): replay_key(STATE, QUESTIONS, assets=image_assets(size=size)) replay_key(STATE, QUESTIONS, assets=image_assets(size=(280, 280))) + + +def test_image_replay_key_is_pinned_to_the_published_envelope(monkeypatch): + # Fixed stand-in bytes (validation is bypassed so the literal does not depend on the + # Pillow/zlib version that would encode a real PNG): this pins the envelope, the + # field names and the decoded-bytes digest, so drift on both sides at once is caught. + from jev_cookbook import vision + + monkeypatch.setattr(vision, "validate_image_asset", lambda asset: b"fixed image bytes") + asset = { + "type": "image", + "format": "png", + "data_base64": base64.b64encode(b"fixed image bytes").decode("ascii"), + } + assert replay_key(STATE, QUESTIONS, assets={REF: asset}) == ( + "00c4d3b2fcb92ce23200e54bed0dcc9c91a7bc134b7862838a1facaddf9acba8" + )