Share the generic parts of a node

.github/workflows/pytest.yml

@@ -3,96 +3,107 @@ name: Run Tests
 on: [push]
 
 jobs:
-  pytest_3_10:
+  unit:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 10
     strategy:
       fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
     steps:
       - uses: actions/checkout@v4
-      - name: set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-      - name: set up uv
-        uses: astral-sh/setup-uv@v3
+      - uses: astral-sh/setup-uv@v3
+      - name: install dependencies
+        # --locked fails on a lockfile that no longer matches pyproject.toml
+        run: uv sync --extra dev --locked --python ${{ matrix.python-version }}
+      - name: check interpreter
+        run: uv run --no-sync python -c "import sys; assert sys.version_info[:2] == tuple(map(int, '${{ matrix.python-version }}'.split('.'))), sys.version"
+      - name: test_unit
+        # no Learning Loop and no secrets needed, so this suite gates the slow ones
+        run: uv run --no-sync python -m pytest learning_loop_node/tests/unit -v
+
+  pytest:
+    needs:
+      - unit
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      # these suites share one Learning Loop instance, and test_general creates and deletes
+      # the project zauberzeug/pytest_nodelib_general by a fixed name -- two entries at once
+      # would delete each other's project
+      max-parallel: 1
+      matrix:
+        # the floor requires-python promises, and the version every node ships with; the unit
+        # job covers 3.11 and 3.13 too
+        python-version: ["3.10", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
       - name: install test dependencies
         run: |
           sudo apt-get update
           sudo apt-get install libcurl4-openssl-dev libssl-dev jpeginfo
       - name: install dependencies
-        run: |
-          uv sync --extra dev --frozen
+        run: uv sync --extra dev --locked --python ${{ matrix.python-version }}
       - name: test_general
         env:
           LOOP_HOST: "preview.learning-loop.ai"
           LOOP_USERNAME: "admin"
           LOOP_PASSWORD: ${{ secrets.LEARNING_LOOP_ADMIN_PASSWORD }}
-        run: |
-          uv run python -m pytest "learning_loop_node/tests/general" -v
+        run: uv run --no-sync python -m pytest learning_loop_node/tests/general -v
       - name: test_detector
         env:
           LOOP_HOST: "preview.learning-loop.ai"
           LOOP_USERNAME: "admin"
           LOOP_PASSWORD: ${{ secrets.LEARNING_LOOP_ADMIN_PASSWORD }}
-        run: |
-          uv run python -m pytest learning_loop_node/tests/detector -v
+        run: uv run --no-sync python -m pytest learning_loop_node/tests/detector -v
       - name: test_mock_detector
         env:
           LOOP_HOST: "preview.learning-loop.ai"
           LOOP_USERNAME: "admin"
           LOOP_PASSWORD: ${{ secrets.LEARNING_LOOP_ADMIN_PASSWORD }}
-        run: |
-          uv run python -m pytest mock_detector -v
-
-  pytest_3_13:
-    needs:
-      - pytest_3_10
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v4
-      - name: set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-      - name: set up uv
-        uses: astral-sh/setup-uv@v3
-      - name: install test dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install libcurl4-openssl-dev libssl-dev jpeginfo
-      - name: install dependencies
-        run: |
-          uv sync --extra dev --frozen
+        run: uv run --no-sync python -m pytest mock_detector -v
       - name: test_annotator
         env:
           LOOP_HOST: "preview.learning-loop.ai"
           LOOP_USERNAME: "admin"
           LOOP_PASSWORD: ${{ secrets.LEARNING_LOOP_ADMIN_PASSWORD }}
-        run: |
-          uv run python -m pytest learning_loop_node/tests/annotator -v
+        run: uv run --no-sync python -m pytest learning_loop_node/tests/annotator -v
       - name: test_trainer
         env:
           LOOP_HOST: "preview.learning-loop.ai"
           LOOP_USERNAME: "admin"
           LOOP_PASSWORD: ${{ secrets.LEARNING_LOOP_ADMIN_PASSWORD }}
-        run: |
-          uv run python -m pytest learning_loop_node/tests/trainer -v
+        run: uv run --no-sync python -m pytest learning_loop_node/tests/trainer -v
       - name: test_mock_trainer
         env:
           LOOP_HOST: "preview.learning-loop.ai"
           LOOP_USERNAME: "admin"
           LOOP_PASSWORD: ${{ secrets.LEARNING_LOOP_ADMIN_PASSWORD }}
+        run: uv run --no-sync python -m pytest mock_trainer -v
+
+  all-green:
+    # the one status check the branch ruleset requires: a matrix job's check run is named
+    # "<job> (<entry>)", so requiring the entries themselves ties the ruleset to the matrix
+    # and every change to it leaves the ruleset waiting for a name nothing reports any more
+    needs:
+      - unit
+      - pytest
+    if: always() # a required check that is skipped reports nothing and waits forever
+    runs-on: ubuntu-latest
+    steps:
+      - name: check the jobs it gates
+        # skipped and cancelled have to fail too -- only success is green
         run: |
-          uv run python -m pytest mock_trainer -v
+          echo "unit: ${{ needs.unit.result }}, pytest: ${{ needs.pytest.result }}"
+          [ "${{ needs.unit.result }}" = success ] || exit 1
+          [ "${{ needs.pytest.result }}" = success ] || exit 1
 
   slack:
     needs:
-      - pytest_3_10
-      - pytest_3_13
+      - unit
+      - pytest
     if: always() # also execute when pytest fails
     runs-on: ubuntu-latest
     steps:

AGENTS.md

@@ -53,15 +53,63 @@ A `DataExchanger` sits on top of the communicator to move images and model zips
 - **Annotator** — thin: it forwards the loop frontend's `handle_user_input` events into
   `AnnotatorLogic` and keeps a per-frontend history.
 
+`trainer/subprocess.py`, `trainer/batch_size.py` and `trainer/metrics.py` hold the parts of a
+trainer that are not framework-specific. `iterator_cpu_bound` runs a training generator in a
+spawned process and yields its progress through a `maxsize=1` queue, so the event loop stays
+responsive, CUDA state stays out of the node process, and the training can never run more than
+one item ahead of the bookkeeping. `find_batch_size` probes for the largest power-of-two batch
+that fits, around a `fits` predicate the trainer supplies — none of it imports torch, so a node
+brings its own way of running a step. `macro_f1` scores the confusion matrix
+`_get_new_best_training_state` returns.
+
+`trainer/cuda.py` is the one exception to that framework independence, and holds everything
+about a batch-size probe that torch has to answer. `usable_memory_bytes` and `limit_cuda_memory`
+turn a `--vram-limit-gb` setting into the budget a probe measures against and the cap that holds
+the process to it, and capping an allocator has no NVML equivalent. `probe_batch_size` is the
+whole probe for a node whose measurement is a single call — it resolves the limit, falls back
+without a card, holds the safety margin and runs the search. A node that must build a throwaway
+model first reserves the margin before building it, and so composes the same pieces itself:
+`reserve_margin`, `measured_fits` and `find_batch_size`. `measured_fits` is where an
+out-of-memory failure is told from a bug — both arrive as the same exception types, and a probe
+that confuses them reports the smallest batch size as the card's fault.
+
+It imports torch, the package does **not** declare it, and only a trainer imports the module — so
+the library keeps working where nothing trains. Its unit test installs a stand-in under the name
+`torch`, which covers the arithmetic, the guards and the search; whether the cap holds, and what
+a real step costs, can only be seen on a card.
+
+`helpers/entrypoint.py` holds what every node's `main.py` repeats: `node_parser` builds a
+configargparse parser with `--host`/`--port`, `run_node` starts uvicorn. A setting is a flag
+*and* an environment variable from one declaration — `--conf-threshold` reads
+`CONF_THRESHOLD`. The exception is `--host`/`--port`, which read `NODE_HOST`/`NODE_PORT`: the
+bare `HOST` already means the loop's address, and a node adopting it would hand it to uvicorn
+and fail to bind. A node that used to require a prefix passes `legacy_env_prefix`, and the
+prefixed names keep working with a warning.
+
+`detector/postprocess.py` and `detector/geometry.py` hold the parts of a detector that do *not*
+depend on the model: confidence filtering, per-class NMS, box/point clipping, and turning
+predictions into the loop's dataclasses. A node should import them rather than write its own —
+every node repository had grown its own drifting copy, which is why they live here. Note the two
+containers: `to_image_metadata` builds what a **detector** node reports, `to_detections` what a
+**trainer**'s auto-detection pass reports. Both go through one routine, so the two paths cannot
+drift apart again.
+
 All node state lives under `GLOBALS.data_folder` (`DATA_FOLDER`, default `/data`): `uuids.json`
 (the node uuid is derived from its name and reused across restarts), `models/` plus the
 `current_model` symlink, `outbox/`, and the per-project training folders.
 
 ## Running and testing
 
-The suites talk to a real Learning Loop instance and read their credentials from a local `.env`
-(`LOOP_HOST`, `LOOP_USERNAME`, `LOOP_PASSWORD`). Without a reachable loop they cannot pass — do not
-treat their failure as a regression you introduced.
+The `unit` suite is self-contained — no Learning Loop, no credentials, no network — so it is the
+one to run while iterating, and the one CI gates the others on:
+
+```bash
+python -m pytest learning_loop_node/tests/unit -v
+```
+
+Every other suite talks to a real Learning Loop instance and reads its credentials from a local
+`.env` (`LOOP_HOST`, `LOOP_USERNAME`, `LOOP_PASSWORD`). Without a reachable loop those cannot pass —
+do not treat their failure as a regression you introduced.
 
 ```bash
 ./run_tests.sh              # all suites
@@ -92,7 +140,8 @@ uvx ruff check .
 A clean tree already reports several hundred ruff findings, so a clean run is not a reachable goal.
 Compare the count on the files you touched, before and after.
 
-`.github/workflows/pytest.yml` runs the suites, `publish.yml` releases to PyPI on a tagged release.
+`.github/workflows/pytest.yml` runs the suites (the `unit` job first, without secrets),
+`publish.yml` releases to PyPI on a tagged release.
 
 ## Working in this repository
 

README.md

@@ -36,6 +36,10 @@ You can configure connection to our Learning Loop by specifying the following en
 
 Note that organization and project IDs are always lower case and may differ from the names in the Learning Loop which can have uppercase letters.
 
+Where a name has an alias, either spelling works. If both are set to **different** values the
+prefixed name wins and a warning names the value used — the variable is never treated as unset,
+which would otherwise let `LOOP_HOST` fall back to its default of `learning-loop.ai`.
+
 #### Testing
 
 We use github actions for CI. Tests can also be executed locally by running

learning_loop_node/detector/categories.py

@@ -0,0 +1,27 @@
+"""Resolving the class indices or names a model emits against ``ModelInformation.categories``."""
+
+from ..data_classes import Category, ModelInformation
+
+
+def category_by_index(model_information: ModelInformation, index: int) -> Category:
+    """Resolve the category a model's class index refers to.
+
+    :raises ValueError: If the index is outside the model's category list.
+    """
+    categories = model_information.categories
+    if not 0 <= index < len(categories):
+        raise ValueError(
+            f'category index {index} is out of range for a model with {len(categories)} categories')
+    return categories[index]
+
+
+def category_by_name(model_information: ModelInformation, name: str) -> Category:
+    """Resolve a category by name, for models whose outputs are named rather than indexed.
+
+    :raises ValueError: If no category of that name exists.
+    """
+    for category in model_information.categories:
+        if category.name == name:
+            return category
+    known = ', '.join(category.name for category in model_information.categories)
+    raise ValueError(f'unknown category name {name!r}; the model knows: {known}')

learning_loop_node/detector/detector_node.py

@@ -18,7 +18,6 @@
 
 from ..data_classes import (
     AboutResponse,
-    Category,
     Context,
     DetectorStatus,
     ImageMetadata,
@@ -33,6 +32,7 @@
 from ..helpers import background_tasks, environment_reader, run
 from ..helpers.misc import numpy_image_from_dict
 from ..node import Node
+from .categories import category_by_name
 from .detector_logic import DetectorLogic, DetectorLogicFactory
 from .exceptions import NodeNeedsRestartError
 from .inbox_filter.relevance_filter import RelevanceFilter
@@ -709,26 +709,28 @@ async def upload_images(
             await self.outbox.save(image, image_metadata, upload_priority)
 
     def add_category_id_to_detections(self, model_info: ModelInformation, image_metadata: ImageMetadata):
-        def find_category_id_by_name(categories: List[Category], category_name: str):
-            category_id = [category.id for category in categories if category.name == category_name]
-            return category_id[0] if category_id else ''
-
-        for box_detection in image_metadata.box_detections:
-            category_name = box_detection.category_name
-            category_id = find_category_id_by_name(model_info.categories, category_name)
-            box_detection.category_id = category_id
-        for point_detection in image_metadata.point_detections:
-            category_name = point_detection.category_name
-            category_id = find_category_id_by_name(model_info.categories, category_name)
-            point_detection.category_id = category_id
-        for segmentation_detection in image_metadata.segmentation_detections:
-            category_name = segmentation_detection.category_name
-            category_id = find_category_id_by_name(model_info.categories, category_name)
-            segmentation_detection.category_id = category_id
-        for classification_detection in image_metadata.classification_detections:
-            category_name = classification_detection.category_name
-            category_id = find_category_id_by_name(model_info.categories, category_name)
-            classification_detection.category_id = category_id
+        """Resolve each detection's category id from its name, in metadata a client uploaded.
+
+        A name the model does not know costs that one detection its id, not the whole upload.
+        """
+        unknown_names: set[str] = set()
+
+        def category_id_by_name(category_name: str) -> str:
+            try:
+                return category_by_name(model_info, category_name).id
+            except ValueError:
+                unknown_names.add(category_name)
+                return ''
+
+        for detection in (*image_metadata.box_detections,
+                          *image_metadata.point_detections,
+                          *image_metadata.segmentation_detections,
+                          *image_metadata.classification_detections):
+            detection.category_id = category_id_by_name(detection.category_name)
+
+        if unknown_names:
+            self.log.warning('Model %s knows no category named %s', model_info.version,
+                             ', '.join(sorted(unknown_names)))
         return image_metadata
 
     def register_sio_events(self, sio_client: AsyncClient):

learning_loop_node/detector/geometry.py

@@ -0,0 +1,39 @@
+"""Box and point clipping shared by every detector node.
+
+The loop stores a box as its top-left corner plus a size, which is the form :func:`clip_box`
+takes and produces.
+"""
+
+
+def clip_box(
+    *,
+    x1: float,
+    y1: float,
+    width: float,
+    height: float,
+    img_width: int,
+    img_height: int,
+) -> tuple[int, int, int, int]:
+    """Clip a top-left-anchored box to the image bounds.
+
+    :return: The clipped ``(x1, y1, width, height)``; the size is never negative.
+    """
+    x2 = x1 + width
+    y2 = y1 + height
+
+    clipped_x1 = round(max(0.0, x1))
+    clipped_y1 = round(max(0.0, y1))
+    clipped_x2 = round(min(float(img_width), x2))
+    clipped_y2 = round(min(float(img_height), y2))
+
+    clipped_width = max(clipped_x2 - clipped_x1, 0)
+    clipped_height = max(clipped_y2 - clipped_y1, 0)
+
+    return clipped_x1, clipped_y1, clipped_width, clipped_height
+
+
+def clip_point(x: float, y: float, img_width: int, img_height: int) -> tuple[float, float]:
+    """Clamp a point into the image bounds."""
+    x = min(max(0, x), img_width)
+    y = min(max(0, y), img_height)
+    return x, y