From 34249bf661d919291532ed84ebf1e9cf8057c808 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 20 Oct 2026 21:48:44 -0400 Subject: [PATCH 1/2] Foundation: Box evaluation report, gold Box label format and validator check (#311) Co-Authored-By: Claude Sonnet 5.5 --- docs/evaluation.md | 26 ++ docs/fixtures.md | 31 ++- docs/vision.md | 100 +++++++- src/jev_cookbook/fixtures/__init__.py | 31 +++ src/jev_cookbook/fixtures/schema.json | 2 +- src/jev_cookbook/vision.py | 233 +++++++++++++++++- tests/test_fixtures.py | 64 +++++ tests/test_vision.py | 342 ++++++++++++++++++++++++++ 8 files changed, 819 insertions(+), 10 deletions(-) diff --git a/docs/evaluation.md b/docs/evaluation.md index 47829e39..443006df 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -736,6 +736,32 @@ numbers, for the reason "Outcomes versus the confidence-only view" gives general `confidence` from `pool_label_decisions` fed to `evaluate_selective` agree with `accepted` and `correct` from `pool_label_outcomes` fed to `evaluate_outcomes`, on every field but `threshold`. +### Box localization + +Box answers are scored in `jev_cookbook.vision`, not here, because they need rectangle +geometry. `vision.localization_report(gold, outcomes, iou_threshold=0.5)` takes the gold Box +labels (`{"present": true, "box": {...}}` or `{"present": false}`, see +[fixtures](fixtures.md#box-labels)) and one `("located", box)`, `("absent", None)` or +`("review", None)` outcome per example. Like every evaluation in this toolkit it states its +denominators and lets no case disappear: reviews and absent targets are counted, never +dropped from the report. + +| Number | Denominator | +| --- | --- | +| mean IoU over located present targets | present targets with a returned box | +| mean IoU over all present targets (missed and reviewed count 0) | all present targets | +| hit rate at the IoU threshold | all present targets | +| false positive rate (a box for an absent target) | all absent targets | +| correct absent rate | all absent targets | + +A present target sent to review therefore counts as 0 in the second mean and as a miss in +the hit rate, and is simply not one of the first mean's located targets: the printed report +shows both denominators. Only +located boxes on present targets contribute an IoU; a box returned for an absent target is a +false positive, because a rectangle never proves the target exists. The worked example, and +the test that runs it, are in [vision](vision.md#the-localization-report). +Use `evaluate_outcomes` above for the review rate of the presence question alone. + ### Calibration `reliability_table(probability, outcome, n_bins=10)` bins stated probability in equal widths diff --git a/docs/fixtures.md b/docs/fixtures.md index b0f5b895..67ab790f 100644 --- a/docs/fixtures.md +++ b/docs/fixtures.md @@ -102,6 +102,35 @@ here too. labels). Every `train`, `validation` and `test` example needs exactly one label; a label whose `id` is not in the inputs is an error. +### Box labels + +A recipe whose question asks for a `Box` labels each example with a gold Box label, so ten +recipes do not invent ten formats. An object with a boolean `present`: + +```json +{"id": "t07", "label": {"present": true, "box": {"top": 0.25, "bottom": 0.75, "left": 0.125, "right": 0.625}}} +{"id": "t08", "label": {"present": false}} +``` + +- `present: true` has exactly `present` and `box`. `box` has exactly `top`, `bottom`, `left` + and `right`: finite numbers in [0, 1] (normalized to the image, as in + [`docs/vision.md`](vision.md)), with `left < right` and `top < bottom`, and an area that + does not underflow to zero. +- `present: false` has exactly `present`. It means the target is not in the image. It carries + no `box`, not even a `null` one; `null` is never a label. +- `present` is a JSON boolean. `"yes"`, `1` and `null` are errors. + +`vision.gold_box(label)` decodes one (the rectangle, or `None` for absent) and raises +`ValueError` on anything else; `vision.localization_report` takes these labels directly. + +The validator treats a label that is an object with a `present` key as a Box label: that is +how a recipe declares the format, so `present` is reserved at the top level of a label object. +Every such label must pass `gold_box`, and once any label in the file is a Box label, a label +that is not (a bare box, a string) is an error. A recipe whose label holds a Box label among +other values nests it under its own key (for example `{"tile": "b3", "target": {"present": +false}}`) and calls `gold_box(label["target"])` in its own test; the validator does not look +inside other labels. + ## Nesting limit No JSON value in a fixture file nests deeper than **64 levels**, counting the line (or, in a @@ -366,7 +395,7 @@ content. Recipe tests must also check image content and lineage across splits, s paths can contain identical pixels and the fixture validator does not open image files. The input row schema is unchanged; do not add a top-level `assets` field. Labels can contain -normalized gold boxes or an explicit absent-target label. A response's `BoxAnswer.to_dict()` +the gold Box label above (a normalized box, or the explicit `{"present": false}`). A response's `BoxAnswer.to_dict()` stores `type`, the four-edge `box`, nullable `confidence`, and the ordinary provenance. `DecisionResult` and the fixture validator reject malformed geometry and confidence. Missing confidence means unknown, never certainty. A Box is still returned for an absent diff --git a/docs/vision.md b/docs/vision.md index f9ebcad0..b4ed11ee 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -123,7 +123,8 @@ touching rectangles and one for identical rectangles. A valid but vanishingly sm (for example 1e-200 wide) has an area that underflows to zero; when both do, the union is zero and `box_iou` (and `localization_metrics`) raise `ValueError` rather than invent a score. Because the union is at least as large as each area, this happens only when both boxes -underflow, so in `localization_metrics` it means a degenerate gold label: fix the fixture. +underflow, so against a gold box it means a degenerate gold label: `gold_box` rejects +such a label up front, so fix the fixture. ## Presence, confidence and evaluation @@ -153,12 +154,99 @@ def localization_outcome(answer, presence, threshold): Choose the presence and confidence thresholds on validation examples and freeze them before evaluating the test split. Keep reviews in the recipe's review queue -and account for them with `evaluation.evaluate_outcomes`; a review is not a claim -that the target is absent. +and account for them in `localization_report` (and, for the overall review rate, +`evaluation.evaluate_outcomes`); a review is not a claim that the target is absent. + +### Gold labels + +A Box recipe labels every example with a gold Box label (specified in +[Box labels](fixtures.md#box-labels)): `{"present": true, "box": {"top": ..., "bottom": ..., +"left": ..., "right": ...}}` when the target is in the image, `{"present": false}` when it is +not. `gold_box(label)` decodes one: it returns the validated normalized rectangle for a +present target and `None` for an absent one, and raises `ValueError` for anything malformed, +including a `box` on an absent label, a non-boolean `present`, edges outside [0, 1], zero +width or height, and a box so small its area underflows to zero (which `box_iou` cannot score). +The fixture validator applies the same check to every label file that uses the format. + +### The localization report + +`localization_report(gold, outcomes, *, iou_threshold=0.5)` is the entry every Box recipe +reports through. `gold` is one gold Box label per example; `outcomes` is one pair per example +from the localization policy above: `("located", box)`, `("absent", None)` or +`("review", None)`. Nothing is filtered, so no example can leave a denominator: each lands in +exactly one cell of a present table or an absent table. + +| Gold | `located` (a box was returned) | `absent` | `review` | +| --- | --- | --- | --- | +| present | IoU is scored | missed (`present_missed`) | `present_review` | +| absent | false positive (`false_positives`) | correct absent (`correct_absents`) | `absent_review` | + +A returned rectangle never proves a target exists, so a `located` box on absent gold is a +false positive, never an IoU. A review is never an absent prediction. The report states: + +- `n_total`, `n_present`, `n_absent`; +- `mean_iou_located`: mean IoU over located present targets, denominator `present_located`; +- `mean_iou_present`: mean IoU over all present targets with missed and reviewed targets + counted as 0, denominator `n_present`; +- `hit_rate`: located present targets with IoU at or above `iou_threshold` (default 0.5), + over `n_present`; +- `false_positives` and `false_positive_rate` (over `n_absent`), `correct_absents` and + `correct_absent_rate` (over `n_absent`), `absent_review`. + +A mean or rate whose denominator is zero is NaN (and prints `n/a`), never 0. `str(report)` +names every denominator, and `report.denominators` returns them by metric name. -`localization_metrics(gold, predicted)` accepts equally sized, nonempty sequences. -Here `None` explicitly labels absence. Apply it to resolved outcomes and report -review counts separately: +```python +from jev_cookbook.vision import localization_report + +G = {"top": 0.0, "bottom": 0.5, "left": 0.0, "right": 0.5} +HALF = {"top": 0.0, "bottom": 0.25, "left": 0.0, "right": 0.5} # IoU 0.5 with G +present, absent = {"present": True, "box": G}, {"present": False} + +report = localization_report( + gold=[present, present, present, present, absent, absent, absent], + outcomes=[ + ("located", G), # IoU 1.0 + ("located", HALF), # IoU 0.5 + ("review", None), # a present target in review keeps its place in the denominator + ("absent", None), # a present target called absent: missed + ("located", G), # a box returned for an absent target: a false positive + ("absent", None), # correct absent + ("review", None), + ], +) +assert (report.n_total, report.n_present, report.n_absent) == (7, 4, 3) +assert report.present_table == {"located": 2, "absent": 1, "review": 1} +assert report.absent_table == {"located": 1, "absent": 1, "review": 1} +assert report.mean_iou_located == 0.75 # (1.0 + 0.5) / 2 located present targets +assert report.mean_iou_present == 0.375 # (1.0 + 0.5) / 4 present targets +assert report.hit_rate == 0.5 # 2 of 4 present targets at IoU >= 0.5 +print(report) +``` + +```text +Localization report: 7 examples (4 present + 3 absent), each counted once +Present targets (n = 4): box returned 2, said absent (missed) 1, sent to review 1 + mean IoU over located present targets: 0.7500 (denominator 2: located present targets) + mean IoU over all present targets, missed and review counted as 0: 0.3750 (denominator 4: all present targets) + hit rate at IoU >= 0.5: 2/4 = 0.5000 (denominator 4: all present targets) +Absent targets (n = 3): box returned (false positive) 1, correct absent 1, sent to review 1 + false positive rate: 1/3 = 0.3333 (denominator 3: all absent targets) + correct absent rate: 1/3 = 0.3333 (denominator 3: all absent targets) +``` + +`tests/test_vision.py` runs this example and compares the printed text. Malformed gold, +an unknown outcome kind, a `located` outcome without a valid box, a box attached to `absent` +or `review`, empty or unequal-length inputs and an `iou_threshold` outside (0, 1] raise +`ValueError` naming the example index. Choose `iou_threshold` and the presence policy on +validation examples and freeze them before scoring the test split. + +### The older `localization_metrics` + +`localization_metrics(gold, predicted)` is kept for compatibility. It takes `None` for an +absent target, so the caller has to drop reviewed examples before calling it, and a present +target sent to review then silently leaves the `mean_iou` denominator. It also reports no +correct-absent count and no hit rate. Use `localization_report` in new work. ```python from jev_cookbook.vision import localization_metrics diff --git a/src/jev_cookbook/fixtures/__init__.py b/src/jev_cookbook/fixtures/__init__.py index 1fd2cec1..ab4a52c7 100644 --- a/src/jev_cookbook/fixtures/__init__.py +++ b/src/jev_cookbook/fixtures/__init__.py @@ -553,6 +553,35 @@ def _check_examples( return problems +def _check_box_labels( + labels: dict[str, Any], label_lines: dict[str, int], name: str +) -> list[Problem]: + """A recipe whose labels are Box labels must use the gold Box label format everywhere. + + A top-level label object with a ``present`` key declares the format (``docs/fixtures.md``, + "Box labels"): every such label must decode with ``vision.gold_box``, and once one label + declares it, a label that does not (a bare box, a string) is an error, not a quiet skip. + A label that nests a Box label under another key is not seen here. + """ + declared = {i for i, v in labels.items() if isinstance(v, Mapping) and "present" in v} + if not declared: + return [] + from ..vision import gold_box # deferred: validating other recipes needs no Pillow + + problems: list[Problem] = [] + for ident, label in labels.items(): + if ident not in declared: + msg = "label is not a Box label ({'present': ...}) but this recipe's other labels are" + else: + try: + gold_box(label) + continue + except ValueError as exc: + msg = f"malformed Box label: {exc}" + problems.append(Problem(name, msg, label_lines[ident], ident)) + return problems + + def _check_responses( examples: list[Example], responses: Mapping[str, Any], name: str ) -> list[Problem]: @@ -692,6 +721,8 @@ def validate_recipe(recipe_dir: PathLike | None = None) -> Report: if inputs_ok: names = (_display(inputs_path), _display(labels_path)) problems += _check_examples(examples, labels if labels_ok else None, label_lines, names) + if labels_ok: + problems += _check_box_labels(labels, label_lines, names[1]) if replays and None not in responses_files: # Listed keys need the default file; with no key listed it is simply not needed. problems.append(Problem(_display(folder / RESPONSES_FILE), "file is missing")) diff --git a/src/jev_cookbook/fixtures/schema.json b/src/jev_cookbook/fixtures/schema.json index 7265f2c6..62d7d821 100644 --- a/src/jev_cookbook/fixtures/schema.json +++ b/src/jev_cookbook/fixtures/schema.json @@ -53,7 +53,7 @@ "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$" }, "label": { - "description": "The gold label. Any JSON value except null: a string, a number, a boolean, a list, or an object (for several labels).", + "description": "The gold label. Any JSON value except null: a string, a number, a boolean, a list, or an object (for several labels, or a Box label: see docs/fixtures.md).", "type": ["string", "number", "boolean", "array", "object"] } } diff --git a/src/jev_cookbook/vision.py b/src/jev_cookbook/vision.py index 6d5148c0..959bfe97 100644 --- a/src/jev_cookbook/vision.py +++ b/src/jev_cookbook/vision.py @@ -29,17 +29,20 @@ "AssetPathError", "ImageTile", "LocalizationMetrics", + "LocalizationReport", "box_confidence", "box_iou", "crop_box", "draw_boxes", "expand_box", "generate_tiles", + "gold_box", "image_asset", "image_assets", "image_reference", "load_frames", "localization_metrics", + "localization_report", "normalized_box", "normalized_to_pixels", "resolve_asset_path", @@ -314,6 +317,37 @@ def box_iou(first: Any, second: Any) -> float: return intersection / union +def gold_box(label: Any) -> dict[str, float] | None: + """Decode a gold Box label: a validated rectangle for a present target, ``None`` if absent. + + The label format (``docs/fixtures.md``) is ``{"present": true, "box": {top, bottom, left, + right}}`` for a target that is in the image and ``{"present": false}`` for one that is + not. ``present`` must be a JSON boolean; a present label has exactly ``present`` and + ``box``, an absent one exactly ``present`` (a ``box`` on an absent label is contradictory + and raises). The box must have exactly the four edges, each finite in [0, 1], positive + width and height, and an area that does not underflow to zero (such a gold box cannot + be scored, since :func:`box_iou` is undefined against it). Raises ``ValueError``. + """ + if not isinstance(label, Mapping): + raise ValueError("a gold Box label must be an object with a boolean 'present'") + present = label.get("present") + if not isinstance(present, bool): + raise ValueError("a gold Box label needs 'present' to be true or false") + if not present: + if set(label) != {"present"}: + raise ValueError("an absent gold Box label must be exactly {'present': false}") + return None + if set(label) != {"present", "box"}: + raise ValueError("a present gold Box label must be exactly 'present' and 'box'") + raw = label["box"] + if not isinstance(raw, Mapping) or set(raw) != set(_EDGES): + raise ValueError("a gold Box label's box needs exactly top, bottom, left and right") + box = normalized_box(raw) + if (box["right"] - box["left"]) * (box["bottom"] - box["top"]) <= 0.0: + raise ValueError("gold box area underflows to zero; it cannot be scored") + return box + + def expand_box(box: Any, fraction: float = 0.0) -> dict[str, float]: """Add ``fraction * box_size`` on each side, clipped to the image boundary.""" edges = normalized_box(box) @@ -479,9 +513,13 @@ class LocalizationMetrics: def localization_metrics(gold: Iterable[Any], predicted: Iterable[Any]) -> LocalizationMetrics: """Evaluate rectangles; ``None`` explicitly denotes an absent target. + Kept for compatibility; prefer :func:`localization_report`, the recommended entry for + a Box recipe. This function cannot see reviews: the caller must filter them out first, + so a present target sent to review silently leaves ``mean_iou``'s denominator, and it + reports no correct-absent count or hit rate. :func:`localization_report` takes review + outcomes itself and names every denominator. + Predictions must already have passed the recipe's presence/confidence policy. - A review outcome is not an absent prediction: report reviews separately through - ``evaluation.evaluate_outcomes`` and use this function on the resolved subset. Invalid rectangles raise, including predictions on absent gold examples. A ``ValueError`` from :func:`box_iou` (both areas underflow to zero) means a degenerate gold label, since the union is at least as large as the gold box's own area. @@ -512,6 +550,197 @@ def localization_metrics(gold: Iterable[Any], predicted: Iterable[Any]) -> Local ) +_OUTCOMES = ("located", "absent", "review") + + +def _rate(numerator: int, denominator: int) -> float: + return numerator / denominator if denominator else float("nan") + + +def _share(numerator: int, denominator: int, what: str) -> str: + if not denominator: + return f"n/a (denominator 0: {what})" + value = numerator / denominator + return f"{numerator}/{denominator} = {value:.4f} (denominator {denominator}: {what})" + + +def _mean(value: float, denominator: int, what: str) -> str: + if not denominator: + return f"n/a (denominator 0: {what})" + return f"{value:.4f} (denominator {denominator}: {what})" + + +@dataclass(frozen=True) +class LocalizationReport: + """Box outcomes against gold, counting every example exactly once. + + ``n_total`` equals ``n_present + n_absent`` and each side is split by outcome: + + * present gold: ``present_located`` (a box was returned after the presence check), + ``present_missed`` (the pipeline said absent), ``present_review``; + * absent gold: ``false_positives`` (a box was returned), ``correct_absents``, + ``absent_review``. + + ``mean_iou_located`` averages IoU over the ``present_located`` targets only (denominator + ``present_located``). ``mean_iou_present`` averages over all ``n_present`` with missed and + reviewed targets counted as 0 (denominator ``n_present``). ``hit_rate`` is ``hits`` (located + present targets with IoU >= ``iou_threshold``) over ``n_present``. A mean or rate with an + empty denominator is NaN. ``ious`` has one entry per example: the IoU of a located present + target, otherwise ``None``. Absent gold never contributes an IoU, and a review is never + counted as an absent prediction. ``str(report)`` names every denominator. + """ + + n_total: int + n_present: int + n_absent: int + present_located: int + present_missed: int + present_review: int + false_positives: int + correct_absents: int + absent_review: int + iou_threshold: float + hits: int + mean_iou_located: float + mean_iou_present: float + hit_rate: float + ious: tuple[float | None, ...] + + @property + def present_table(self) -> dict[str, int]: + """Present gold by outcome; the values sum to ``n_present``.""" + return { + "located": self.present_located, + "absent": self.present_missed, + "review": self.present_review, + } + + @property + def absent_table(self) -> dict[str, int]: + """Absent gold by outcome; the values sum to ``n_absent``.""" + return { + "located": self.false_positives, + "absent": self.correct_absents, + "review": self.absent_review, + } + + @property + def false_positive_rate(self) -> float: + """``false_positives / n_absent`` (NaN when no target is absent).""" + return _rate(self.false_positives, self.n_absent) + + @property + def correct_absent_rate(self) -> float: + """``correct_absents / n_absent`` (NaN when no target is absent).""" + return _rate(self.correct_absents, self.n_absent) + + @property + def denominators(self) -> dict[str, int]: + """The denominator of every mean and rate in the report, by name.""" + return { + "mean_iou_located": self.present_located, + "mean_iou_present": self.n_present, + "hit_rate": self.n_present, + "false_positive_rate": self.n_absent, + "correct_absent_rate": self.n_absent, + } + + def __str__(self) -> str: + located = _mean(self.mean_iou_located, self.present_located, "located present targets") + overall = _mean(self.mean_iou_present, self.n_present, "all present targets") + return "\n".join( + [ + f"Localization report: {self.n_total} examples " + f"({self.n_present} present + {self.n_absent} absent), each counted once", + f"Present targets (n = {self.n_present}): box returned {self.present_located}, " + f"said absent (missed) {self.present_missed}, sent to review {self.present_review}", + f" mean IoU over located present targets: {located}", + f" mean IoU over all present targets, missed and review counted as 0: {overall}", + f" hit rate at IoU >= {self.iou_threshold:g}: " + + _share(self.hits, self.n_present, "all present targets"), + f"Absent targets (n = {self.n_absent}): box returned (false positive) " + f"{self.false_positives}, correct absent {self.correct_absents}, " + f"sent to review {self.absent_review}", + " false positive rate: " + + _share(self.false_positives, self.n_absent, "all absent targets"), + " correct absent rate: " + + _share(self.correct_absents, self.n_absent, "all absent targets"), + ] + ) + + +def localization_report( + gold: Iterable[Any], outcomes: Iterable[Any], *, iou_threshold: float = 0.5 +) -> LocalizationReport: + """Score Box outcomes against gold Box labels without letting any example drop out. + + ``gold`` holds one label per example in the format :func:`gold_box` decodes (a malformed + or degenerate label raises). ``outcomes`` holds one pair per example, produced by the + recipe's presence and confidence policy: ``("located", box)``, ``("absent", None)`` or + ``("review", None)``. A review is never an absent prediction, and a returned rectangle + is never evidence the target exists: a ``located`` box on absent gold is a false + positive. Every example lands in exactly one cell of the present or absent table. An + example is a hit when its IoU is at least ``iou_threshold``, a number in (0, 1]. + Empty or unequal-length inputs, an unknown outcome kind, a ``located`` outcome without + a valid box, or a box attached to ``absent``/``review`` raise ``ValueError``. + """ + if isinstance(iou_threshold, bool) or not isinstance(iou_threshold, Real): + raise ValueError("iou_threshold must be a number in (0, 1]") + if not (math.isfinite(iou_threshold) and 0 < iou_threshold <= 1): + raise ValueError("iou_threshold must be a number in (0, 1]") + targets, results = list(gold), list(outcomes) + if not targets or len(targets) != len(results): + raise ValueError("gold and outcomes must be nonempty and have the same length") + present_table = dict.fromkeys(_OUTCOMES, 0) + absent_table = dict.fromkeys(_OUTCOMES, 0) + ious: list[float | None] = [] + for index, (label, outcome) in enumerate(zip(targets, results, strict=True)): + try: + target = gold_box(label) + if ( + isinstance(outcome, (str, bytes)) + or not isinstance(outcome, (tuple, list)) + or len(outcome) != 2 + ): + raise ValueError("an outcome must be a (kind, box-or-None) pair") + kind, box = outcome + if kind not in _OUTCOMES: + raise ValueError(f"outcome kind must be one of {_OUTCOMES}, not {kind!r}") + if kind == "located": + guess = normalized_box(box) + elif box is not None: + raise ValueError(f"a {kind!r} outcome must carry None, not a box") + except ValueError as exc: + raise ValueError(f"example {index}: {exc}") from exc + if target is None: + absent_table[kind] += 1 + ious.append(None) + else: + present_table[kind] += 1 + ious.append(box_iou(target, guess) if kind == "located" else None) + overlaps = [value for value in ious if value is not None] + n_present, n_absent = sum(present_table.values()), sum(absent_table.values()) + hits = sum(value >= iou_threshold for value in overlaps) + # Targets that are not located score 0 in the all-present mean, so its sum is the located sum. + return LocalizationReport( + n_total=len(targets), + n_present=n_present, + n_absent=n_absent, + present_located=present_table["located"], + present_missed=present_table["absent"], + present_review=present_table["review"], + false_positives=absent_table["located"], + correct_absents=absent_table["absent"], + absent_review=absent_table["review"], + iou_threshold=float(iou_threshold), + hits=hits, + mean_iou_located=math.fsum(overlaps) / len(overlaps) if overlaps else float("nan"), + mean_iou_present=math.fsum(overlaps) / n_present if n_present else float("nan"), + hit_rate=_rate(hits, n_present), + ious=tuple(ious), + ) + + def load_frames( frames: Iterable[tuple[float, str | Path]], *, root: str | Path | None = None ) -> tuple[list[dict[str, Any]], dict[str, dict[str, str]]]: diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py index 55a6aaa2..bc814fa3 100644 --- a/tests/test_fixtures.py +++ b/tests/test_fixtures.py @@ -335,6 +335,70 @@ def test_label_without_label_field(recipe): expect(recipe, "labels.jsonl:1 (id 't0'): missing required field 'label'") +# ------------------------------------------------------------------------- Box labels + +BOX_LABEL = {"present": True, "box": {"top": 0.25, "bottom": 0.75, "left": 0.125, "right": 0.625}} + + +def box_set(*labels) -> dict: + data = good_set() + for row, label in zip(data["labels"], labels, strict=True): + row["label"] = label + return data + + +def test_well_formed_box_labels_present_and_absent_are_valid(recipe): + write(recipe, box_set(BOX_LABEL, {"present": False}, BOX_LABEL, {"present": False})) + assert messages(recipe) == [] + assert load_labels(recipe)["t1"] == {"present": False} + + +@pytest.mark.parametrize( + ("bad", "fragment"), + [ + ({"present": "yes"}, "'present' to be true or false"), + ({"present": 1}, "'present' to be true or false"), + ({"present": None}, "'present' to be true or false"), + ({"present": False, "box": BOX_LABEL["box"]}, "exactly {'present': false}"), + ({"present": False, "box": None}, "exactly {'present': false}"), + ({"present": True}, "exactly 'present' and 'box'"), + ({"present": True, "box": None}, "exactly top, bottom, left and right"), + ({**BOX_LABEL, "extra": 1}, "exactly 'present' and 'box'"), + ({"present": True, "box": {"top": 0.1, "bottom": 0.5, "left": 0.1}}, "exactly top"), + ( + {"present": True, "box": {"top": 0.1, "bottom": 0.5, "left": 0.1, "right": 1.5}}, + "right must be in [0, 1]", + ), + ( + {"present": True, "box": {"top": 0.5, "bottom": 0.5, "left": 0.1, "right": 0.9}}, + "positive width and height", + ), + ( + {"present": True, "box": {"top": 0, "bottom": 1e-200, "left": 0, "right": 1e-200}}, + "underflows to zero", + ), + ], +) +def test_malformed_box_labels_are_rejected(recipe, bad, fragment): + write(recipe, box_set(BOX_LABEL, bad, BOX_LABEL, {"present": False})) + expect(recipe, "labels.jsonl:2 (id 't1'): malformed Box label", fragment) + + +def test_a_non_box_label_among_box_labels_is_rejected(recipe): + write(recipe, box_set(BOX_LABEL, "billing", BOX_LABEL, BOX_LABEL["box"])) + expect( + recipe, + "labels.jsonl:2 (id 't1'): label is not a Box label", + "labels.jsonl:4 (id 't3'): label is not a Box label", + ) + + +def test_labels_without_a_present_key_are_not_treated_as_box_labels(recipe): + data = box_set({"box": BOX_LABEL["box"]}, "x", ["y"], {"tags": []}) + write(recipe, data) + assert messages(recipe) == [] + + # --------------------------------------------------------------- responses and linking diff --git a/tests/test_vision.py b/tests/test_vision.py index e2a31f62..219d4039 100644 --- a/tests/test_vision.py +++ b/tests/test_vision.py @@ -3,6 +3,7 @@ from __future__ import annotations import base64 +import contextlib import hashlib import io import json @@ -869,3 +870,344 @@ def test_box_confidence_docstring_matches_its_behaviour(): assert vision.box_confidence({"confidence": 0.0}) == 0.0 assert vision.box_confidence({"confidence": 1.0}) == 1.0 assert vision.box_confidence({}) is None + + +# ------------------------------------------------------------------ gold Box labels and the report +# +# Every expected value below is computed by hand from these rectangles (all edges are dyadic +# fractions, so the IoUs are exact). GOLD has area 0.25. +# SAME identical to GOLD: intersection 0.25, union 0.25 -> 1 +# HALF the top half of GOLD: intersection 0.125, union 0.25 -> 1/2 +# QUARTER a 0.125-wide strip of GOLD: intersection 0.0625, union 0.25 -> 1/4 +# FAR the opposite corner, touching at a point: no intersection -> 0 +GOLD = {"top": 0.0, "bottom": 0.5, "left": 0.0, "right": 0.5} +SAME = dict(GOLD) +HALF = {"top": 0.0, "bottom": 0.25, "left": 0.0, "right": 0.5} +QUARTER = {"top": 0.0, "bottom": 0.5, "left": 0.0, "right": 0.125} +FAR = {"top": 0.5, "bottom": 1.0, "left": 0.5, "right": 1.0} +PRESENT = {"present": True, "box": GOLD} +ABSENT = {"present": False} +LOCATED_TINY = {"top": 0.0, "bottom": 1e-200, "left": 0.0, "right": 1e-200} + + +def test_gold_box_decodes_present_and_absent_labels(): + decoded = vision.gold_box(PRESENT) + assert decoded == GOLD + assert decoded is not PRESENT["box"] # a validated copy + assert vision.gold_box(ABSENT) is None + assert vision.gold_box(json.loads(json.dumps(PRESENT))) == GOLD + + +@pytest.mark.parametrize( + "label", + [ + None, + "present", + [PRESENT], + {}, + {"box": GOLD}, + {"present": None}, + {"present": 1}, + {"present": 0}, + {"present": "false"}, + {"present": True}, + {"present": True, "box": None}, + {"present": True, "box": [0, 0, 1, 1]}, + {"present": True, "box": {"top": 0, "bottom": 1, "left": 0}}, + {"present": True, "box": {**GOLD, "extra": 0.5}}, + {"present": True, "box": GOLD, "confidence": 0.9}, + {"present": False, "box": GOLD}, + {"present": False, "box": None}, + {"present": False, "extra": 1}, + {"present": True, "box": {**GOLD, "right": 1.5}}, + {"present": True, "box": {**GOLD, "left": -0.1}}, + {"present": True, "box": {**GOLD, "right": float("nan")}}, + {"present": True, "box": {**GOLD, "right": 0.0}}, # zero width + {"present": True, "box": {**GOLD, "bottom": 0.0}}, # zero height + {"present": True, "box": {**GOLD, "left": 0.5}}, # left == right + {"present": True, "box": {**GOLD, "left": "0"}}, + {"present": True, "box": {**GOLD, "left": True}}, + {"present": True, "box": LOCATED_TINY}, # valid edges, but the area underflows to 0.0 + ], +) +def test_gold_box_rejects_malformed_labels(label): + with pytest.raises(ValueError): + vision.gold_box(label) + + +def test_gold_box_rejects_a_degenerate_gold_box_before_any_score_is_computed(): + with pytest.raises(ValueError, match="underflows to zero"): + vision.gold_box({"present": True, "box": LOCATED_TINY}) + with pytest.raises(ValueError, match=r"example 1: .*underflows to zero"): + vision.localization_report( + [PRESENT, {"present": True, "box": LOCATED_TINY}], + [("located", SAME), ("located", LOCATED_TINY)], + ) + + +def counted_once(report) -> None: + """The invariant every report must keep: each example is in exactly one table cell.""" + assert sum(report.present_table.values()) == report.n_present + assert sum(report.absent_table.values()) == report.n_absent + assert report.n_present + report.n_absent == report.n_total + assert len(report.ious) == report.n_total + + +def test_report_mixed_outcomes_by_hand(): + # Present gold (5): IoU 1, IoU 1/2, IoU 1/4, said absent, sent to review. + # Absent gold (4): a box was returned, correct absent, correct absent, sent to review. + gold = [PRESENT] * 5 + [ABSENT] * 4 + outcomes = [ + ("located", SAME), + ("located", HALF), + ("located", QUARTER), + ("absent", None), + ("review", None), + ("located", SAME), + ("absent", None), + ("absent", None), + ("review", None), + ] + report = vision.localization_report(gold, outcomes) + counted_once(report) + assert (report.n_total, report.n_present, report.n_absent) == (9, 5, 4) + assert report.present_table == {"located": 3, "absent": 1, "review": 1} + assert report.absent_table == {"located": 1, "absent": 2, "review": 1} + assert (report.present_located, report.present_missed, report.present_review) == (3, 1, 1) + assert (report.false_positives, report.correct_absents, report.absent_review) == (1, 2, 1) + # (1 + 1/2 + 1/4) / 3 located present targets = 7/12; the false positive adds no IoU. + assert report.mean_iou_located == pytest.approx(7 / 12) + # (1 + 1/2 + 1/4 + 0 + 0) / 5 present targets = 7/20. + assert report.mean_iou_present == pytest.approx(0.35) + # Hits at IoU >= 0.5 are the first two present targets: 2 of 5. + assert report.hits == 2 + assert report.hit_rate == pytest.approx(0.4) + assert report.false_positive_rate == pytest.approx(0.25) # 1 of 4 absent targets + assert report.correct_absent_rate == pytest.approx(0.5) # 2 of 4 absent targets + assert report.ious == (1.0, 0.5, 0.25, None, None, None, None, None, None) + assert report.denominators == { + "mean_iou_located": 3, + "mean_iou_present": 5, + "hit_rate": 5, + "false_positive_rate": 4, + "correct_absent_rate": 4, + } + + +def test_report_all_absent_has_no_iou_and_no_invented_zero(): + gold = [ABSENT, ABSENT, ABSENT] + outcomes = [("absent", None), ("located", FAR), ("review", None)] + report = vision.localization_report(gold, outcomes) + counted_once(report) + assert (report.n_total, report.n_present, report.n_absent) == (3, 0, 3) + assert report.absent_table == {"located": 1, "absent": 1, "review": 1} + assert report.present_table == {"located": 0, "absent": 0, "review": 0} + assert math.isnan(report.mean_iou_located) + assert math.isnan(report.mean_iou_present) + assert math.isnan(report.hit_rate) + assert report.hits == 0 + assert report.false_positive_rate == pytest.approx(1 / 3) + assert report.correct_absent_rate == pytest.approx(1 / 3) + text = str(report) + assert "3 examples (0 present + 3 absent)" in text + assert "mean IoU over located present targets: n/a (denominator 0" in text + assert "hit rate at IoU >= 0.5: n/a (denominator 0" in text + assert "false positive rate: 1/3 = 0.3333 (denominator 3: all absent targets)" in text + + +def test_report_all_absent_and_all_correct(): + report = vision.localization_report([ABSENT, ABSENT], [("absent", None)] * 2) + assert (report.correct_absents, report.false_positives) == (2, 0) + assert report.correct_absent_rate == 1.0 + assert report.false_positive_rate == 0.0 + + +def test_report_all_review_keeps_every_example_in_the_denominator(): + gold = [PRESENT, ABSENT, PRESENT] + report = vision.localization_report(gold, [("review", None)] * 3) + counted_once(report) + assert report.n_total == 3 + assert report.present_table == {"located": 0, "absent": 0, "review": 2} + assert report.absent_table == {"located": 0, "absent": 0, "review": 1} + # No located target: the located mean has nothing to average, but the all-present mean is + # 0/2, not undefined: both reviewed present targets are in its denominator at 0. + assert math.isnan(report.mean_iou_located) + assert report.mean_iou_present == 0.0 + assert report.hit_rate == 0.0 + assert report.hits == 0 + # A review is not an absent prediction: nothing counts as a correct absent or a miss. + assert (report.correct_absents, report.false_positives, report.present_missed) == (0, 0, 0) + assert report.correct_absent_rate == 0.0 + assert report.false_positive_rate == 0.0 + + +def test_a_reviewed_present_target_changes_the_denominator_instead_of_vanishing(): + gold = [PRESENT, PRESENT] + resolved = vision.localization_report(gold, [("located", SAME), ("located", HALF)]) + reviewed = vision.localization_report(gold, [("located", SAME), ("review", None)]) + for report in (resolved, reviewed): + counted_once(report) + assert (report.n_total, report.n_present) == (2, 2) + assert resolved.mean_iou_located == 0.75 # (1 + 1/2) / 2 + assert resolved.mean_iou_present == 0.75 + assert resolved.denominators["mean_iou_located"] == 2 + # The reviewed target is not located (denominator 1) but still present (denominator 2). + assert reviewed.mean_iou_located == 1.0 # 1 / 1 + assert reviewed.mean_iou_present == 0.5 # (1 + 0) / 2 + assert reviewed.denominators["mean_iou_located"] == 1 + assert reviewed.denominators["mean_iou_present"] == 2 + assert reviewed.hit_rate == 0.5 # 1 hit of 2 present targets + text = str(reviewed) + assert "1.0000 (denominator 1: located present targets)" in text + assert "0.5000 (denominator 2: all present targets)" in text + assert "sent to review 1" in text + + +def test_box_returned_for_an_absent_target_is_a_false_positive_never_an_iou(): + # The box is identical to a rectangle that would score IoU 1 on a present target. + report = vision.localization_report([ABSENT], [("located", SAME)]) + counted_once(report) + assert report.absent_table == {"located": 1, "absent": 0, "review": 0} + assert report.false_positives == 1 + assert report.false_positive_rate == 1.0 + assert report.correct_absent_rate == 0.0 + assert report.ious == (None,) + assert math.isnan(report.mean_iou_present) + # Mixed with a present target, the false positive leaves the IoU means untouched. + both = vision.localization_report([PRESENT, ABSENT], [("located", HALF), ("located", SAME)]) + assert both.mean_iou_located == 0.5 + assert both.mean_iou_present == 0.5 + assert both.hit_rate == 1.0 + + +def test_a_present_target_called_absent_is_a_miss_and_not_a_correct_absent(): + report = vision.localization_report([PRESENT], [("absent", None)]) + assert report.present_table == {"located": 0, "absent": 1, "review": 0} + assert report.correct_absents == 0 + assert report.mean_iou_present == 0.0 + assert math.isnan(report.mean_iou_located) + + +def test_report_uses_each_examples_own_gold_box(): + other = {"present": True, "box": {"top": 0.25, "bottom": 0.75, "left": 0.25, "right": 0.75}} + # GOLD against `other`: intersection 0.25 * 0.25 = 1/16, union 1/4 + 1/4 - 1/16 = 7/16. + report = vision.localization_report([other], [("located", GOLD)]) + assert report.ious[0] == pytest.approx(1 / 7) + assert report.hits == 0 + + +def test_report_hit_threshold_is_inclusive_and_configurable(): + gold = [PRESENT] * 3 + outcomes = [("located", HALF), ("located", QUARTER), ("located", FAR)] + default = vision.localization_report(gold, outcomes) + assert default.iou_threshold == 0.5 + assert default.hits == 1 # IoU exactly 1/2 counts + strict = vision.localization_report(gold, outcomes, iou_threshold=0.6) + assert (strict.hits, strict.hit_rate) == (0, 0.0) + loose = vision.localization_report(gold, outcomes, iou_threshold=0.25) + assert (loose.hits, loose.hit_rate) == (2, pytest.approx(2 / 3)) # IoU 1/2 and 1/4 + assert vision.localization_report(gold, outcomes, iou_threshold=1).hits == 0 + assert "hit rate at IoU >= 0.25" in str(loose) + # FAR touches GOLD at a corner: IoU 0, and its zero is in the mean over 3 located targets. + assert default.mean_iou_located == pytest.approx((0.5 + 0.25 + 0.0) / 3) + + +def test_report_prints_every_denominator_for_the_documented_example(): + text = str( + vision.localization_report( + [PRESENT] * 4 + [ABSENT] * 3, + [ + ("located", SAME), + ("located", HALF), + ("review", None), + ("absent", None), + ("located", SAME), + ("absent", None), + ("review", None), + ], + ) + ) + assert text.splitlines() == [ + "Localization report: 7 examples (4 present + 3 absent), each counted once", + "Present targets (n = 4): box returned 2, said absent (missed) 1, sent to review 1", + " mean IoU over located present targets: 0.7500 (denominator 2: located present targets)", + " mean IoU over all present targets, missed and review counted as 0: " + "0.3750 (denominator 4: all present targets)", + " hit rate at IoU >= 0.5: 2/4 = 0.5000 (denominator 4: all present targets)", + "Absent targets (n = 3): box returned (false positive) 1, correct absent 1, " + "sent to review 1", + " false positive rate: 1/3 = 0.3333 (denominator 3: all absent targets)", + " correct absent rate: 1/3 = 0.3333 (denominator 3: all absent targets)", + ] + + +def test_the_documented_localization_example_runs_and_prints_what_the_docs_show(): + text = (Path(__file__).resolve().parent.parent / "docs" / "vision.md").read_text("utf-8") + start = text.index("from jev_cookbook.vision import localization_report") + code = text[start : text.index("```", start)] + printed = text[text.index("```text\n", start) + len("```text\n") :] + printed = printed[: printed.index("```")] + out = io.StringIO() + with contextlib.redirect_stdout(out): + exec(compile(code, "docs/vision.md", "exec"), {}) + assert out.getvalue() == printed + + +@pytest.mark.parametrize( + ("gold", "outcomes", "match"), + [ + ([], [], "nonempty"), + ([PRESENT], [], "same length"), + ([PRESENT], [("located", SAME)] * 2, "same length"), + ([{"present": "yes"}], [("absent", None)], "example 0: .*'present'"), + ([PRESENT, {"present": False, "box": GOLD}], [("absent", None)] * 2, "example 1"), + ([PRESENT], ["located"], "pair"), + ([PRESENT], [("located",)], "pair"), + ([PRESENT], [None], "pair"), + ([PRESENT], [("maybe", None)], "outcome kind"), + ([PRESENT], [("LOCATED", SAME)], "outcome kind"), + ([PRESENT], [("located", None)], "example 0"), + ([ABSENT], [("located", None)], "example 0"), + ([PRESENT], [("located", {**SAME, "right": 2.0})], "example 0"), + ([ABSENT], [("located", {**SAME, "right": 2.0})], "example 0"), # absent gold still checked + ([PRESENT], [("absent", SAME)], "must carry None"), + ([ABSENT], [("review", SAME)], "must carry None"), + ], +) +def test_report_rejects_malformed_inputs(gold, outcomes, match): + with pytest.raises(ValueError, match=match): + vision.localization_report(gold, outcomes) + + +@pytest.mark.parametrize( + "threshold", [0, -0.1, 1.01, float("nan"), float("inf"), True, "0.5", None] +) +def test_report_rejects_a_bad_threshold(threshold): + with pytest.raises(ValueError, match="iou_threshold"): + vision.localization_report([PRESENT], [("located", SAME)], iou_threshold=threshold) + + +def test_report_accepts_lists_a_box_answer_object_and_any_iterable(): + answer = SimpleNamespace(box=HALF, confidence=0.9) + report = vision.localization_report( + iter([PRESENT, ABSENT]), iter([["located", answer], ["absent", None]]) + ) + assert report.ious == (0.5, None) + assert report.correct_absents == 1 + + +def test_a_tiny_returned_box_on_a_present_target_scores_zero_without_raising(): + report = vision.localization_report([PRESENT], [("located", LOCATED_TINY)]) + assert report.ious == (0.0,) + assert report.mean_iou_located == 0.0 + + +def test_report_is_a_frozen_dataclass_and_localization_metrics_remains(): + report = vision.localization_report([ABSENT], [("absent", None)]) + assert is_dataclass(report) + with pytest.raises(AttributeError): + report.n_total = 5 + assert "localization_report" in vision.__all__ + assert "gold_box" in vision.__all__ + assert "localization_report" in (vision.localization_metrics.__doc__ or "") + assert vision.localization_metrics([GOLD, None], [GOLD, None]).mean_iou == 1.0 From 865dfccf26a4db633c2fa7900052dddac6e2c139 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 21 Oct 2026 16:53:28 +0000 Subject: [PATCH 2/2] Foundation: fix the nested Box label example and say how tiled recipes report (#311 review) Refs #311 Co-Authored-By: Claude Sonnet 5.5 --- docs/evaluation.md | 6 ++++ docs/fixtures.md | 17 ++++++++-- docs/vision.md | 18 ++++++++++ tests/test_vision.py | 80 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 3 deletions(-) diff --git a/docs/evaluation.md b/docs/evaluation.md index 443006df..f9d257db 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -762,6 +762,12 @@ false positive, because a rectangle never proves the target exists. The worked e the test that runs it, are in [vision](vision.md#the-localization-report). Use `evaluate_outcomes` above for the review rate of the presence question alone. +In a tiled or hierarchical recipe the headline report is per image: gold is the full-image +Box label and each outcome is the final rectangle after `tile_to_image`, with a stage-1 "no +match" on present gold reported as `absent` and an uncertain choice as `review`. A per-tile +report counts tiles, is a stage diagnostic and says so, and states its rule for a target a +tile edge cuts; see [vision](vision.md#tiled-and-hierarchical-recipes). + ### Calibration `reliability_table(probability, outcome, n_bins=10)` bins stated probability in equal widths diff --git a/docs/fixtures.md b/docs/fixtures.md index 67ab790f..c7530aba 100644 --- a/docs/fixtures.md +++ b/docs/fixtures.md @@ -127,9 +127,20 @@ The validator treats a label that is an object with a `present` key as a Box lab how a recipe declares the format, so `present` is reserved at the top level of a label object. Every such label must pass `gold_box`, and once any label in the file is a Box label, a label that is not (a bare box, a string) is an error. A recipe whose label holds a Box label among -other values nests it under its own key (for example `{"tile": "b3", "target": {"present": -false}}`) and calls `gold_box(label["target"])` in its own test; the validator does not look -inside other labels. +other values nests it under its own key and calls `gold_box(label["target"])` in its own test; +the validator does not look inside other labels, so that test must cover every split the +recipe ships (`train` included), not only the one it reports: + +```json +{"id": "t12", "label": {"target": {"present": true, "box": {"top": 0.25, "bottom": 0.75, "left": 0.125, "right": 0.625}}, "rubric": 3}} +``` + +Store the gold box once, for the whole image, as a full-image Box label. A recipe that tiles +the image does not store a tile choice beside it: a stored tile name can disagree with the box +(for an absent target there is no right tile at all, only "no match"), and with overlapping +tiles a target near a seam lies in more than one tile, so one string cannot hold the gold. +Python derives which tiles count as correct from the box and the tile bounds, and the +headline report stays per image ([vision](vision.md#tiled-and-hierarchical-recipes)). ## Nesting limit diff --git a/docs/vision.md b/docs/vision.md index b4ed11ee..45e24158 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -241,6 +241,24 @@ or `review`, empty or unequal-length inputs and an `iou_threshold` outside (0, 1 `ValueError` naming the example index. Choose `iou_threshold` and the presence policy on validation examples and freeze them before scoring the test split. +### Tiled and hierarchical recipes + +When a recipe tiles the image (a Choice picks the tile, then a Box locates the target inside +it), one example is one image, not one tile. A target can be present in the full image and +absent from most tiles, so counting tiles would make `n_total` a tile count and +`correct_absent_rate` mostly a measure of empty tiles. + +- The headline report is per image. `gold` is the full-image gold Box label (stored once, + see [Box labels](fixtures.md#box-labels)) and each outcome is the final rectangle after + `tile_to_image`. +- Map the stage-1 result to an outcome before reporting: a stage-1 "no match" on an image + whose gold is present is `("absent", None)` (a miss); an uncertain tile choice, or an + uncertain Box in the chosen tile, is `("review", None)`. +- A per-tile report is a stage diagnostic. Label it as counting tiles, never quote it as the + recipe's result, and state its rule for a target a tile edge cuts (the minimum visible + fraction of the box that makes the target "present" in that tile), because that rule + changes the per-tile denominators. + ### The older `localization_metrics` `localization_metrics(gold, predicted)` is kept for compatibility. It takes `None` for an diff --git a/tests/test_vision.py b/tests/test_vision.py index 219d4039..fe139f1d 100644 --- a/tests/test_vision.py +++ b/tests/test_vision.py @@ -9,6 +9,7 @@ import json import math import random +import re from dataclasses import is_dataclass from pathlib import Path from types import SimpleNamespace @@ -1153,6 +1154,85 @@ def test_the_documented_localization_example_runs_and_prints_what_the_docs_show( assert out.getvalue() == printed +_DOCS = Path(__file__).resolve().parent.parent / "docs" + + +def _section(text: str, heading: str, level: str = "###") -> str: + start = text.index(f"{level} {heading}\n") + following = re.search(rf"\n#{{1,{len(level)}}} ", text[start + 1 :]) + return text[start : start + 1 + following.start()] if following else text[start:] + + +def _box_label_objects(value): + """Every dict under `value` (itself included) that has a `present` key.""" + found = [] + if isinstance(value, dict): + if "present" in value: + found.append(value) + for child in value.values(): + found.extend(_box_label_objects(child)) + elif isinstance(value, list): + for child in value: + found.extend(_box_label_objects(child)) + return found + + +def _contains_tile_key(value) -> bool: + if isinstance(value, dict): + return any("tile" in str(key).lower() or _contains_tile_key(v) for key, v in value.items()) + if isinstance(value, list): + return any(_contains_tile_key(v) for v in value) + return False + + +def test_every_label_example_in_the_box_labels_docs_is_valid_and_consistent(): + """Each JSON example in docs/fixtures.md "Box labels" must decode with gold_box, and no + example stores a tile choice beside the Box label (the gold lives once, full-image).""" + section = _section((_DOCS / "fixtures.md").read_text("utf-8"), "Box labels") + lines = [ + line + for block in section.split("```json\n")[1:] + for line in block[: block.index("```")].splitlines() + if line.strip() + ] + labels = [json.loads(line)["label"] for line in lines] + nested = [label for label in labels if "present" not in label] + assert len(labels) >= 3 and nested, "the section must show flat and nested examples" + for label in labels: + boxes = _box_label_objects(label) + assert boxes, label + for box_label in boxes: + vision.gold_box(box_label) # raises on a self-contradictory example + assert not _contains_tile_key(label), f"a tile choice stored with the gold: {label}" + flat = " ".join(section.split()) + assert '"tile":' not in flat, "an inline example stores a tile choice beside the gold" + assert "Store the gold box once, for the whole image" in flat + assert "does not store a tile choice" in flat + assert "must cover every split" in flat + + +def test_the_docs_say_a_tiled_recipe_reports_per_image_not_per_tile(): + vision_md = " ".join( + _section((_DOCS / "vision.md").read_text("utf-8"), "Tiled and hierarchical recipes").split() + ) + for required in ( + "one example is one image, not one tile", + "The headline report is per image", + "after `tile_to_image`", + '"no match" on an image whose gold is present is `("absent", None)`', + '`("review", None)`', + "stage diagnostic", + "a target a tile edge cuts", + ): + assert required in vision_md, required + evaluation = " ".join((_DOCS / "evaluation.md").read_text("utf-8").split()) + assert "In a tiled or hierarchical recipe the headline report is per image" in evaluation + assert "vision.md#tiled-and-hierarchical-recipes" in evaluation + assert "vision.md#tiled-and-hierarchical-recipes" in " ".join( + (_DOCS / "fixtures.md").read_text("utf-8").split() + ) + + @pytest.mark.parametrize( ("gold", "outcomes", "match"), [