From 9d1071c8656ab96d9de4e2ca5bf50c479bcae0ec Mon Sep 17 00:00:00 2001 From: ashish-aesthisia Date: Sat, 22 Aug 2026 07:03:49 +0000 Subject: [PATCH 1/3] feat: minimax-h3 character support --- core/src/inline_core/characters/apply.py | 31 ++-- core/src/inline_core/characters/encode.py | 36 ++++- .../inline_core/models/minimaxh3/runner.py | 86 ++++++++++- core/src/inline_core/studio/characters.py | 107 +++++++++++++- core/src/inline_core/studio/generation.py | 10 +- core/src/inline_core/studio/recipe.py | 12 +- core/tests/test_h3_characters.py | 135 ++++++++++++++++++ core/tests/test_minimaxh3_nodes.py | 14 +- core/tests/test_recipe.py | 67 +++++++++ core/uv.lock | 2 +- src/renderer/views/Moodboard/graphExport.ts | 22 ++- 11 files changed, 490 insertions(+), 32 deletions(-) create mode 100644 core/tests/test_h3_characters.py diff --git a/core/src/inline_core/characters/apply.py b/core/src/inline_core/characters/apply.py index 7f9f5cc..31de78e 100644 --- a/core/src/inline_core/characters/apply.py +++ b/core/src/inline_core/characters/apply.py @@ -42,17 +42,27 @@ def __init__( #: turned down and the character wire carries no controls of its own. self.lora_strength = lora_strength - def prompt_prefix(self, first_position: int) -> str: - """Text naming the positions the character lands on, so ordinal prompting resolves.""" + def prompt_prefix(self, first_position: int, style: str = "ordinal") -> str: + """Text naming the positions the character lands on, so positional prompting resolves. + + ``style`` because a model only resolves the form it was trained on: FLUX.2 reads the ordinal + prose below, MiniMax H3 reads ```` tokens (``models/references.py``), and handing + either the other one names positions it cannot see. + """ if not self.refs: # A LoRA carries the likeness, so the description is all the prompt needs. detail = " ".join(self.description.split()) return f"{detail} " if detail else "" - positions = [str(first_position + i) for i in range(len(self.refs))] - if len(positions) == 1: + positions = [first_position + i for i in range(len(self.refs))] + if style == "token": + tokens = " ".join(f"" for n in positions) + plural = "" if len(positions) == 1 else "each" + which = f"{tokens} {'shows' if not plural else 'show'}" + elif len(positions) == 1: which = f"Image {positions[0]} shows" else: - which = f"Images {', '.join(positions[:-1])} and {positions[-1]} show" + ordinals = [str(n) for n in positions] + which = f"Images {', '.join(ordinals[:-1])} and {ordinals[-1]} show" line = f"{which} {self.name}, the same character in every image." detail = " ".join(self.description.split()) if not detail: @@ -84,10 +94,11 @@ def char_apply(chosen: str, arch: str = encode.FLUX2_KLEIN_ARCH) -> AppliedChara doc = cf.read(path) digest = library.content_hash(path) - references = arch == encode.FLUX2_KLEIN_ARCH + # Whether this arch reads references at all, which is exactly the archs with a policy for them. + references = arch in encode.REFERENCE_POLICIES if references and not cf.payload_valid(doc.manifest, arch, encode.PAYLOAD_ENCODER_VERSION): - doc = _recompile(doc, path) + doc = _recompile(doc, path, arch) digest = library.content_hash(path) description = _description(doc) @@ -137,9 +148,9 @@ def _description(doc: cf.CharDoc) -> str: return raw.decode("utf-8", errors="replace") if raw else "" -def _recompile(doc: cf.CharDoc, path: Path) -> cf.CharDoc: +def _recompile(doc: cf.CharDoc, path: Path, arch: str = encode.FLUX2_KLEIN_ARCH) -> cf.CharDoc: """Rebuild a stale payload from ``refs/``. Payloads are cache, so this always works.""" - logger.info("Recompiling the %s payload for %s", encode.FLUX2_KLEIN_ARCH, path.name) + logger.info("Recompiling the %s payload for %s", arch, path.name) import io from PIL import Image @@ -153,7 +164,7 @@ def _recompile(doc: cf.CharDoc, path: Path) -> cf.CharDoc: ) images.append(Image.open(io.BytesIO(raw)).convert("RGB")) - encode.build_payload(doc.manifest, doc.members, images) + encode.build_payload(doc.manifest, doc.members, images, arch) cf.write(path, doc) return doc diff --git a/core/src/inline_core/characters/encode.py b/core/src/inline_core/characters/encode.py index 5c48437..1f0d36f 100644 --- a/core/src/inline_core/characters/encode.py +++ b/core/src/inline_core/characters/encode.py @@ -41,8 +41,14 @@ def payload_key(arch: str, kind: str = PAYLOAD_REF) -> str: #: What each model accepts as a reference. A video model has its own frame grid, so the policy #: cannot stay one constant - and it rides in the payload entry, because the fingerprint is taken #: against the policy the payload was built with, not against whatever is current. +#: H3 resizes a reference to a 2048 short edge, upscaling included, rounding each axis to 32 on its +#: own, with no area cap - so a 4:1 reference is 8192x2048 (vendor/packing_ref2va.py). +MINIMAX_H3_ARCH = "minimax-h3" +MINIMAX_H3_POLICY: dict[str, Any] = {"short_edge": 2048, "multiple_of": 32, "max_aspect": 4.0} + REFERENCE_POLICIES: dict[str, dict[str, Any]] = { FLUX2_KLEIN_ARCH: PAYLOAD_POLICY, + MINIMAX_H3_ARCH: MINIMAX_H3_POLICY, } @@ -71,16 +77,32 @@ def _png_bytes(image: Any) -> bytes: def normalise_reference(image: Any, policy: dict[str, Any] | None = None) -> Any: - """A reference resized into a model's budget, on its grid, preserving aspect.""" + """A reference resized into a model's budget, on its grid, preserving aspect. + + Two policy shapes, because two models mean two things by "budget": `max_pixels` is an area cap + that only ever shrinks, `short_edge` is a target the smaller side is scaled *onto*, up or down. + """ rules = policy or PAYLOAD_POLICY - max_pixels = int(rules["max_pixels"]) grid = int(rules["multiple_of"]) width, height = image.width, image.height - if width * height > max_pixels: - scale = (max_pixels / (width * height)) ** 0.5 - width, height = int(width * scale), int(height * scale) - width = max(grid, (width // grid) * grid) - height = max(grid, (height // grid) * grid) + limit = rules.get("max_aspect") + if limit and max(width / height, height / width) > float(limit): + raise ValueError( + f"A reference must be within 1:{limit:g} and {limit:g}:1 for this model, " + f"got {width}x{height}." + ) + if "short_edge" in rules: + # Rounded, not floored: flooring a scaled-up edge can land back under the target. + scale = int(rules["short_edge"]) / min(width, height) + width = max(grid, round(width * scale / grid) * grid) + height = max(grid, round(height * scale / grid) * grid) + else: + max_pixels = int(rules["max_pixels"]) + if width * height > max_pixels: + scale = (max_pixels / (width * height)) ** 0.5 + width, height = int(width * scale), int(height * scale) + width = max(grid, (width // grid) * grid) + height = max(grid, (height // grid) * grid) if (width, height) == (image.width, image.height): return image from PIL import Image diff --git a/core/src/inline_core/models/minimaxh3/runner.py b/core/src/inline_core/models/minimaxh3/runner.py index 5a5bbee..e48f41c 100644 --- a/core/src/inline_core/models/minimaxh3/runner.py +++ b/core/src/inline_core/models/minimaxh3/runner.py @@ -37,6 +37,9 @@ #: Names this node in the error a mis-wired handle raises. _LABEL = "MiniMax H3" +#: The key a character files its H3 payloads under, matching `training/arch.py`. +ARCH = "minimax-h3" + #: 24 fps, decodable in blocks of 17 frames plus 5, between 5 and 15 seconds: 124 to 345 frames. GRID = VideoGrid(fps=24.0, grid=17, offset=5, min_seconds=5.0, max_seconds=15.0) @@ -135,6 +138,9 @@ def _inputs(variant: Variant) -> tuple[Port, ...]: # Adapters fuse into each block as it streams, before factorisation and quantisation. A # LoRA trained on either partition loads on both: they are the same architecture. Port("lora", "LoRA", PortKind.LORA, required=False), + # Every variant takes one: the reference partition applies it by compiled references, the + # rest by its trained adapter, which is the only route on a node with no reference channel. + Port("character", "Character", PortKind.CHARACTER, required=False), ] if variant.first_frame: ports.append(Port("image", "First frame", PortKind.IMAGE, required=False)) @@ -186,6 +192,8 @@ class Request: seed: int partition: str references: tuple[Any, ...] = () + #: The wired character's adapter, appended to the user's own LoRAs rather than replacing them. + character_loras: tuple[Any, ...] = () @property def seconds(self) -> float: @@ -206,14 +214,25 @@ def build_request( multiple=CANVAS_MULTIPLE, minimum=CANVAS_MULTIPLE, ) + character = _apply_character(inputs, variant) + loras: tuple[Any, ...] = () references: tuple[Any, ...] = () if variant.references: + wired = list(inputs.get("references") or []) + if character is not None and character.refs: + # Fed through the collector rather than appended after it, so the character's images are + # numbered and limit-checked as images - appending would land them behind the videos. + inputs = {**inputs, "references": [*wired, *character.refs]} references = collect_references(inputs, limits=REFERENCE_LIMITS) if not references: raise ComponentError( f"{variant.title} needs at least one reference wired to its References, " "Reference video or Reference audio input." ) + if character is not None: + prompt = character.prefix + prompt + if character.lora is not None: + loras = (character.lora,) return Request( prompt=prompt, num_frames=frames, @@ -223,6 +242,70 @@ def build_request( seed=rt.resolve_seed(params.get("seed")), partition=variant.partition, references=references, + character_loras=loras, + ) + + +@dataclass(frozen=True) +class _Character: + refs: list[Any] + prefix: str + lora: Any = None + + +def _character_file(inputs: dict[str, list[Any]]) -> str: + """The wired character's filename. Applying resolves payloads through a content-keyed cache, so + an identity that has not been written yet cannot be applied.""" + wired = (inputs.get("character") or [None])[0] + if wired is None: + return "" + name = str(getattr(wired, "file", "") or "") + if not name: + raise ComponentError( + "That character has not been saved yet. Wire it through Write .char first." + ) + return name + + +def _apply_character(inputs: dict[str, list[Any]], variant: Variant) -> _Character | None: + """A wired character as references or as its adapter, or None when none is wired.""" + chosen = _character_file(inputs) + if not chosen: + return None + from ...characters import apply as characters + from ...graph.loader_runners import LoraRef + + applied = characters.char_apply(chosen, ARCH) + if applied is None: + return None + if not variant.references: + # No reference channel on this partition, so the adapter is the only route it has. + if applied.lora is None: + raise ComponentError( + f"{chosen} has no {ARCH} adapter, and {variant.title} has no reference channel. " + "Train one and attach it, or use MiniMax H3 Reference to Video." + ) + logger.info("Applying character %s by adapter", applied.name) + return _Character( + refs=[], + prefix=applied.prompt_prefix(1), + lora=LoraRef(file=str(applied.lora), strength=applied.lora_strength), + ) + if not applied.refs and applied.lora is None: + return None + how = "adapter" if applied.lora is not None else f"{len(applied.refs)} reference(s)" + logger.info("Applying character %s by %s", applied.name, how) + # H3 resolves ``, not FLUX.2's ordinal prose, and the character's images land after + # whatever the user already wired. + wired = len([v for v in (inputs.get("references") or []) if v is not None]) + return _Character( + refs=list(applied.refs), + prefix=applied.prompt_prefix(wired + 1, style="token"), + lora=( + LoraRef(file=str(applied.lora), strength=applied.lora_strength) + if applied.lora is not None + else None + ), ) @@ -320,7 +403,8 @@ def run(self, node: Node, inputs: dict[str, list[Any]], ctx: ExecutionContext) - transformer=rt.component_ref(inputs, "model", "diffusion", _LABEL), video_vae=rt.component_ref(inputs, "vae", "vae", _LABEL), text_encoder=rt.component_ref(inputs, "text_encoder", "text_encoder", _LABEL), - loras=rt.lora_stack(inputs, _LABEL), + # Appended, so a user's own wired LoRAs still apply alongside the character's. + loras=(*rt.lora_stack(inputs, _LABEL), *request.character_loras), ) call = call_kwargs(request, self._variant, inputs) diff --git a/core/src/inline_core/studio/characters.py b/core/src/inline_core/studio/characters.py index 27918be..2c05f0c 100644 --- a/core/src/inline_core/studio/characters.py +++ b/core/src/inline_core/studio/characters.py @@ -25,6 +25,108 @@ CHANGED_EVENT = "events:charactersChanged" PROGRESS_EVENT = "events:characterProgress" +#: Takes whose score has to come from frames rather than from the file itself. +_VIDEO_SUFFIXES = {".mp4", ".mov", ".webm", ".mkv"} + +#: How many frames a video take is measured on. Each one costs a full SFace + DINOv2 pass, and this +#: runs inline while the take is saved, so it buys robustness rather than precision. +SCORE_FRAMES = 5 + +#: Skipped at each end, where a video is least settled and a low score would say more about the +#: first moments than about the character. +_EDGE_SECONDS = 0.5 + + +def _sample_frames(src: Path, count: int = SCORE_FRAMES) -> list[Any]: + """Evenly spaced frames as PIL images, or [] when ffmpeg cannot read the file.""" + import io + import subprocess + + from PIL import Image + + from ..ffmpeg import ffmpeg_exe + + exe = ffmpeg_exe() + if exe is None or not src.is_file(): + return [] + frames: list[Any] = [] + duration = _duration_seconds(src) + if duration is None or duration <= 0: + return [] + span = max(duration - 2 * _EDGE_SECONDS, 0.0) + # One decode per frame: seeking is cheaper than decoding the whole clip for five stills. + for index in range(count): + offset = _EDGE_SECONDS + (span * (index + 0.5) / count if span else 0.0) + try: + proc = subprocess.run( + [exe, "-v", "quiet", "-ss", f"{offset:.3f}", "-i", str(src), + "-frames:v", "1", "-f", "image2pipe", "-vcodec", "png", "-"], + capture_output=True, + timeout=60, + ) + except (OSError, subprocess.SubprocessError): + continue + if not proc.stdout: + continue + try: + with Image.open(io.BytesIO(proc.stdout)) as handle: + frames.append(handle.convert("RGB")) + except Exception: # noqa: BLE001 - a frame that will not decode is one fewer sample + continue + return frames + + +def _duration_seconds(src: Path) -> float | None: + """The clip's length via ffprobe, or None. ffprobe is often absent, so this is best-effort.""" + import subprocess + + from ..ffmpeg import ffprobe_exe + + exe = ffprobe_exe() + if exe is None: + return None + try: + proc = subprocess.run( + [exe, "-v", "quiet", "-show_entries", "format=duration", + "-of", "default=nw=1:nk=1", str(src)], + capture_output=True, + timeout=30, + ) + return float(proc.stdout.decode().strip()) + except (OSError, subprocess.SubprocessError, ValueError): + return None + + +def _score_video( + src: Path, + centroids: dict[str, list[float]], + face_refs: list[list[float]], + subject_refs: list[list[float]], + framings: list[float], +) -> dict[str, Any] | None: + """One score for a clip: the median across the frames that measured, never a mean. + + A frame where no face was found returns None from ``score`` and drops out rather than counting + as a zero, and the median survives one blurred frame and one lucky one alike. ``frames`` rides + along so a number from two samples is not read as a number from five. + """ + from statistics import median + + measured = [ + result + for frame in _sample_frames(src) + if (result := scoring.score(frame, centroids, face_refs, subject_refs, framings)) + ] + if not measured: + return None + out = dict(measured[len(measured) // 2]) + out["score"] = round(median(float(m["score"]) for m in measured), 1) + # Face-only if any sampled frame could not be spoken to by the subject term. + out["subjectCounted"] = all(m.get("subjectCounted", True) for m in measured) + out["frames"] = len(measured) + return out + + class Characters: """The `characters:*` channels, backed by ``models/characters/``.""" @@ -102,7 +204,10 @@ def score_take(self, image_path: Path | str, chosen: str) -> dict[str, Any] | No framings = [float(f) for f in (doc.manifest.scoring.get("refFramings") or [])] from PIL import Image - with Image.open(image_path) as handle: + path_in = Path(image_path) + if path_in.suffix.lower() in _VIDEO_SUFFIXES: + return _score_video(path_in, centroids, face_refs, subject_refs, framings) + with Image.open(path_in) as handle: return scoring.score( handle.convert("RGB"), centroids, face_refs, subject_refs, framings ) diff --git a/core/src/inline_core/studio/generation.py b/core/src/inline_core/studio/generation.py index 12ce161..f5e3357 100644 --- a/core/src/inline_core/studio/generation.py +++ b/core/src/inline_core/studio/generation.py @@ -252,12 +252,17 @@ def _continuity(self, take: Any, path: Path) -> dict[str, Any]: if result is None: # Measured nothing. Record the character anyway so the UI can say which one was used. return {"characterId": chosen} - return { + out = { "characterId": chosen, "continuityScore": result["score"], # False when the number is the face alone, so a dropped term is never hidden. "continuityFaceOnly": not result.get("subjectCounted", True), } + # Only a video carries this: a score from two sampled frames is not the same claim as one + # from five, and the reader cannot tell them apart from the number. + if result.get("frames"): + out["continuityFrames"] = result["frames"] + return out def _save_take(self, item_id: str, take: Any, ref: Any) -> None: """Copy a take's bytes into the project's takes/ dir and set its Core node's output. Image @@ -304,7 +309,8 @@ def _save_take(self, item_id: str, take: Any, ref: Any) -> None: "createdAt": int(time.time() * 1000), "params": dict(getattr(take, "params", {}) or {}), "prompt": recipe.get("prompt", ""), - **(self._continuity(take, dst) if kind == "image" else {}), + # Video too: a take is scored on sampled frames rather than not at all. + **(self._continuity(take, dst) if kind in ("image", "video") else {}), }, ) diff --git a/core/src/inline_core/studio/recipe.py b/core/src/inline_core/studio/recipe.py index 9619fad..0a7f933 100644 --- a/core/src/inline_core/studio/recipe.py +++ b/core/src/inline_core/studio/recipe.py @@ -12,6 +12,7 @@ import logging import sqlite3 +from collections import Counter from typing import Any from . import frames as fr @@ -80,12 +81,17 @@ def _typed_params( # names its own file, so counting this one too listed the wrong checkpoint beside the right. if kinds.get(key) in ("model", "character") and str(value) and key not in wired: wanted.setdefault(str(value).replace("\\", "/").rsplit("/", 1)[-1], "") + declared = _declared(node_type, params) + per_category = Counter(category for _f, category in declared) # A node's declared requirements name its *default* build, so a node set to klein-9b would - # otherwise export klein-4b beside it. Whatever a param already named speaks for its folder. - covered = _folders_of(list(wanted)) - for filename, category in _declared(node_type, params): + # otherwise export klein-4b beside it. Only where the folder holds one required file, though: a + # param names one file, so it cannot stand in for both of MiniMax H3's VAEs, and excusing the + # folder dropped the audio one from the export entirely. + covered = {c for c in _folders_of(list(wanted)) if per_category.get(c) == 1} + for filename, category in declared: if category in covered: continue + # Overwrites a param-named file's empty folder with the real one, which is an improvement. wanted[filename] = category return typed, _coordinates(wanted) diff --git a/core/tests/test_h3_characters.py b/core/tests/test_h3_characters.py new file mode 100644 index 0000000..5e059fb --- /dev/null +++ b/core/tests/test_h3_characters.py @@ -0,0 +1,135 @@ +"""Characters on MiniMax H3: the reference policy, the prompt form, and scoring a video take.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from inline_core.characters import encode +from inline_core.characters.apply import AppliedCharacter + +# --- the reference policy ------------------------------------------------------------------------- + + +def _resize(size: tuple[int, int], arch: str) -> tuple[int, int]: + pytest.importorskip("PIL") + from PIL import Image + + out = encode.normalise_reference(Image.new("RGB", size), encode.reference_policy(arch)) + return out.width, out.height + + +def test_h3_is_offered_a_reference_payload_at_all() -> None: + """One entry in this table was what forced every model but FLUX.2 down the adapter route.""" + assert encode.MINIMAX_H3_ARCH in encode.REFERENCE_POLICIES + + +def test_h3_scales_onto_a_short_edge_rather_than_under_an_area_cap() -> None: + """FLUX.2 caps area and only ever shrinks; H3 scales the short side onto 2048 either way.""" + assert _resize((3000, 2000), encode.MINIMAX_H3_ARCH) == (3072, 2048) + assert _resize((640, 640), encode.MINIMAX_H3_ARCH) == (2048, 2048) + assert _resize((3000, 2000), encode.FLUX2_KLEIN_ARCH) == (1248, 832) + + +def test_h3_has_no_area_cap() -> None: + """The vendored packer says a 4:1 reference is 8192x2048, so the payload must agree.""" + assert _resize((8000, 2000), encode.MINIMAX_H3_ARCH) == (8192, 2048) + + +def test_an_out_of_range_aspect_is_refused_while_compiling() -> None: + """The vendored blocks raise on this mid-run; catching it here names the reference instead.""" + pytest.importorskip("PIL") + from PIL import Image + + with pytest.raises(ValueError, match="within 1:4"): + encode.normalise_reference(Image.new("RGB", (5000, 900)), encode.MINIMAX_H3_POLICY) + + +def test_both_policies_land_on_their_own_grid() -> None: + for arch, grid in ((encode.FLUX2_KLEIN_ARCH, 16), (encode.MINIMAX_H3_ARCH, 32)): + width, height = _resize((1234, 987), arch) + assert width % grid == 0 and height % grid == 0 + + +# --- the prompt form ------------------------------------------------------------------------------ + + +def _character(refs: int) -> AppliedCharacter: + return AppliedCharacter("Ada", ["ref"] * refs, "freckles, dark hair") + + +def test_h3_names_references_as_tokens_not_as_ordinal_prose() -> None: + """H3 resolves ``; FLUX.2's "Images 1 and 2 show" names positions H3 cannot see.""" + prefix = _character(2).prompt_prefix(3, style="token") + assert prefix.startswith(" show Ada") + assert "Images" not in prefix + + +def test_the_offset_is_where_the_character_lands_not_where_it_starts() -> None: + """The character's images are appended after whatever the user wired, so the numbering has to + continue rather than restart.""" + assert _character(1).prompt_prefix(5, style="token").startswith(" shows Ada") + + +def test_flux2_keeps_its_prose() -> None: + assert _character(2).prompt_prefix(1).startswith("Images 1 and 2 show Ada") + + +def test_an_adapter_only_character_is_the_description_alone() -> None: + """No references to name, so there is nothing positional to say in either style.""" + assert AppliedCharacter("Ada", [], "freckles").prompt_prefix(1, style="token") == "freckles " + + +# --- scoring a video take ------------------------------------------------------------------------- + + +def _scored(value: float, subject: bool = True) -> dict[str, Any]: + return {"score": value, "subjectCounted": subject} + + +def test_a_video_score_is_the_median_of_the_frames_that_measured(monkeypatch) -> None: + """A mean lets one blurred frame drag the number; a median survives one bad and one lucky.""" + from inline_core.studio import characters as mod + + monkeypatch.setattr(mod, "_sample_frames", lambda *_a, **_k: ["f"] * 5) + scores = iter([_scored(80), _scored(12), _scored(78), _scored(82), _scored(79)]) + monkeypatch.setattr(mod.scoring, "score", lambda *_a, **_k: next(scores)) + + out = mod._score_video(Path("clip.mp4"), {}, [], [], []) + assert out is not None and out["score"] == 79.0 + assert out["frames"] == 5 + + +def test_a_frame_with_no_face_drops_out_rather_than_scoring_zero(monkeypatch) -> None: + """`score` returns None for unmeasurable, which is not the same claim as a score of nothing.""" + from inline_core.studio import characters as mod + + monkeypatch.setattr(mod, "_sample_frames", lambda *_a, **_k: ["f"] * 3) + scores = iter([_scored(90), None, _scored(70)]) + monkeypatch.setattr(mod.scoring, "score", lambda *_a, **_k: next(scores)) + + out = mod._score_video(Path("clip.mp4"), {}, [], [], []) + assert out is not None and out["frames"] == 2, "the unmeasurable frame is not counted" + assert out["score"] == 80.0 + + +def test_one_face_only_frame_makes_the_whole_score_face_only(monkeypatch) -> None: + """Reporting a blended number when a frame's subject term was noise hides the dropped term.""" + from inline_core.studio import characters as mod + + monkeypatch.setattr(mod, "_sample_frames", lambda *_a, **_k: ["f"] * 2) + scores = iter([_scored(80), _scored(70, subject=False)]) + monkeypatch.setattr(mod.scoring, "score", lambda *_a, **_k: next(scores)) + + out = mod._score_video(Path("clip.mp4"), {}, [], [], []) + assert out is not None and out["subjectCounted"] is False + + +def test_a_clip_that_cannot_be_read_scores_nothing(monkeypatch) -> None: + """A missing ffmpeg means no score, never a failed render.""" + from inline_core.studio import characters as mod + + monkeypatch.setattr(mod, "_sample_frames", lambda *_a, **_k: []) + assert mod._score_video(Path("clip.mp4"), {}, [], [], []) is None diff --git a/core/tests/test_minimaxh3_nodes.py b/core/tests/test_minimaxh3_nodes.py index e053643..fd4aa6d 100644 --- a/core/tests/test_minimaxh3_nodes.py +++ b/core/tests/test_minimaxh3_nodes.py @@ -63,7 +63,10 @@ def test_every_node_outputs_video_plus_a_separate_audio_port(node_type: str) -> def test_the_inputs_are_what_each_node_is_for() -> None: def media(node_type: str) -> list[str]: - return [p.id for p in DESCRIPTORS[node_type].inputs if p.id not in COMPONENTS] + # `character` is an identity handle, not media: it is on every node, so it says nothing + # about what a given node is for. + skip = {*COMPONENTS, "character"} + return [p.id for p in DESCRIPTORS[node_type].inputs if p.id not in skip] assert media(T2V) == ["prompt"] assert media(I2V) == ["prompt", "image"] @@ -71,6 +74,15 @@ def media(node_type: str) -> list[str]: assert media(REF) == ["prompt", "references", "video", "audio"] +@pytest.mark.parametrize("node_type", [T2V, I2V, FLF, REF]) +def test_every_node_takes_a_character(node_type: str) -> None: + """The reference partition applies one by compiled references and the rest by its trained + adapter, so one `.char` serves the whole family rather than only the node that reads images.""" + port = {p.id: p for p in DESCRIPTORS[node_type].inputs}.get("character") + assert port is not None and port.kind is PortKind.CHARACTER + assert not port.required + + @pytest.mark.parametrize("node_type", [T2V, I2V, FLF, REF]) def test_every_node_carries_the_component_handles(node_type: str) -> None: """They were missing at first, which left H3 the only model family on the canvas with no way to diff --git a/core/tests/test_recipe.py b/core/tests/test_recipe.py index 7f505c8..116bf9d 100644 --- a/core/tests/test_recipe.py +++ b/core/tests/test_recipe.py @@ -404,3 +404,70 @@ def test_a_path_shaped_pick_exports_under_its_bare_name(tmp_path) -> None: names = [m["name"] for m in built["graph"]["items"][0]["data"]["core"]["models"]] assert names == ["krea2_turbo_bf16.safetensors"] + +def test_two_required_files_in_one_folder_both_reach_the_export(tmp_path) -> None: + """MiniMax H3 needs a video VAE and an audio VAE, both under `vae/`, and has one `vae` param. + Excusing the whole folder because a param named one of them dropped the other, and the graph + then rebuilt on another machine without a file it cannot run without.""" + from inline_core.studio import recipe as studio_recipe + + store = _store(tmp_path) + conn = store.conn() + node = mb.add_core_node(conn, "minimax/h3-reference-to-video", 400, 200) + mb.update_item(conn, node["id"], {"data": {"core": { + "type": "minimax/h3-reference-to-video", + "params": {"vae": "minimax_h3_video_vae_fp16.safetensors"}, + }}}) + + studio_recipe.set_kind_resolver(lambda _t: {"vae": "model"}) + studio_recipe.set_model_resolver( + lambda _t, _p=None: [ + ("minimax_h3_video_vae_fp16.safetensors", "vae"), + ("minimax_h3_audio_vae_fp32.safetensors", "vae"), + ("MiniMax-H3-text-encoder", "text_encoders"), + ] + ) + try: + built = studio_recipe.build_recipe(conn, node["id"]) + finally: + studio_recipe.set_kind_resolver(None) + studio_recipe.set_model_resolver(None) + + models = built["graph"]["items"][0]["data"]["core"]["models"] + names = sorted(m["name"] for m in models) + assert names == [ + "MiniMax-H3-text-encoder", + "minimax_h3_audio_vae_fp32.safetensors", + "minimax_h3_video_vae_fp16.safetensors", + ] + # And the param-named file still gets its folder rather than an empty one. + by_name = {m["name"]: m for m in models} + assert by_name["minimax_h3_video_vae_fp16.safetensors"]["directory"] == "vae" + + +def test_a_folder_with_one_required_file_is_still_excused_by_a_param(tmp_path) -> None: + """The rule this replaces existed for a reason: a node set to klein-9b must not export the + klein-4b default beside it.""" + from inline_core.studio import recipe as studio_recipe + + store = _store(tmp_path) + conn = store.conn() + node = mb.add_core_node(conn, "black-forest-labs/flux-2", 400, 200) + mb.update_item(conn, node["id"], {"data": {"core": { + "type": "black-forest-labs/flux-2", + "params": {"model": "flux-2-klein-9b.safetensors"}, + }}}) + + studio_recipe.set_kind_resolver(lambda _t: {"model": "model"}) + studio_recipe.set_model_resolver( + lambda _t, _p=None: [("flux-2-klein-4b.safetensors", "diffusion_models")] + ) + try: + built = studio_recipe.build_recipe(conn, node["id"]) + finally: + studio_recipe.set_kind_resolver(None) + studio_recipe.set_model_resolver(None) + + names = [m["name"] for m in built["graph"]["items"][0]["data"]["core"]["models"]] + assert names == ["flux-2-klein-9b.safetensors"], "the default build is not exported beside it" + diff --git a/core/uv.lock b/core/uv.lock index d27d2ed..484b3f8 100644 --- a/core/uv.lock +++ b/core/uv.lock @@ -610,7 +610,7 @@ wheels = [ [[package]] name = "inline-core" -version = "1.2.74" +version = "1.3.0" source = { editable = "." } dependencies = [ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, diff --git a/src/renderer/views/Moodboard/graphExport.ts b/src/renderer/views/Moodboard/graphExport.ts index c74c9d4..5849478 100644 --- a/src/renderer/views/Moodboard/graphExport.ts +++ b/src/renderer/views/Moodboard/graphExport.ts @@ -167,16 +167,26 @@ function coreEntry( } } const registry = useModelRegistryStore.getState().entries + const required = (useModelRequirementsStore.getState().byType[type]?.components ?? []).filter( + (c) => !c.optional, + ) + const perCategory = new Map() + for (const c of required) perCategory.set(c.category, (perCategory.get(c.category) ?? 0) + 1) // Which folders the params already spoke for. A node's declared requirements name its *default* - // build, so a node set to klein-9b would otherwise export klein-4b beside it. + // build, so a node set to klein-9b would otherwise export klein-4b beside it. Only where the + // folder holds one required file, though: a param names one file, so it cannot stand in for both + // of MiniMax H3's VAEs, and excusing the folder dropped the audio one from the export entirely. const covered = new Set( - [...wanted.keys()].map( - (name) => registry.find((e) => e.filename.toLowerCase() === name.toLowerCase())?.category, - ), + [...wanted.keys()] + .map( + (name) => registry.find((e) => e.filename.toLowerCase() === name.toLowerCase())?.category, + ) + .filter((category): category is string => !!category && perCategory.get(category) === 1), ) // What the node needs without naming it, which no param can carry. - for (const component of useModelRequirementsStore.getState().byType[type]?.components ?? []) { - if (component.optional || covered.has(component.category)) continue + for (const component of required) { + if (covered.has(component.category)) continue + // Overwrites a param-named file's empty folder with the real one, which is an improvement. wanted.set(component.localPath.split('/').pop() ?? component.localPath, component.category) } const models = [...wanted].map(([name, directory]) => { From 6367882f6f38e27ac21f32be7c37f06147460620 Mon Sep 17 00:00:00 2001 From: ashish-aesthisia Date: Sun, 23 Aug 2026 17:04:11 +0000 Subject: [PATCH 2/3] a2i-loop --- core/src/inline_core/characters/apply.py | 9 +- core/src/inline_core/characters/charfile.py | 33 +- core/src/inline_core/characters/encode.py | 366 +++++++++++++++++- core/src/inline_core/characters/library.py | 6 +- core/src/inline_core/characters/scoring.py | 98 ++++- core/src/inline_core/characters/verify.py | 172 ++++++++ .../inline_core/models/character/runner.py | 266 +++++++++++-- core/src/inline_core/models/characterreqs.py | 4 +- core/src/inline_core/models/loaders.py | 21 + .../src/inline_core/models/minimaxh3/nvfp4.py | 161 ++++++++ .../inline_core/models/minimaxh3/pipeline.py | 234 ++++++++++- .../models/minimaxh3/requirements.py | 115 +++++- .../inline_core/models/minimaxh3/runner.py | 82 +++- .../inline_core/models/pipeline_runtime.py | 17 + core/src/inline_core/models/trainingreqs.py | 3 +- core/src/inline_core/studio/characters.py | 48 +-- core/src/inline_core/studio/models.py | 36 +- core/tests/test_character_nodes.py | 305 +++++++++++++++ core/tests/test_characters_encode.py | 150 +++++++ core/tests/test_characters_rpc.py | 51 +++ core/tests/test_characters_scoring.py | 53 +++ core/tests/test_h3_characters.py | 246 ++++++++++++ core/tests/test_minimaxh3_nodes.py | 57 ++- core/tests/test_minimaxh3_nvfp4.py | 124 ++++++ core/tests/test_model_download_queue.py | 40 ++ core/tests/test_studio_rpc.py | 2 + core/tests/test_training_requirements.py | 19 + .../lib/starterGraph.character.test.ts | 15 +- src/renderer/lib/starterGraph.ts | 44 ++- src/renderer/store/moodboardStore.ts | 28 ++ src/renderer/views/Moodboard/AddNodeMenu.tsx | 17 +- .../views/Moodboard/MoodboardPanel.tsx | 6 +- src/renderer/views/Moodboard/nodeKinds.ts | 43 ++ .../views/Moodboard/nodeRendering.test.ts | 18 + src/shared/types.ts | 7 + 35 files changed, 2712 insertions(+), 184 deletions(-) create mode 100644 core/src/inline_core/characters/verify.py create mode 100644 core/src/inline_core/models/minimaxh3/nvfp4.py create mode 100644 core/tests/test_minimaxh3_nvfp4.py create mode 100644 src/renderer/views/Moodboard/nodeKinds.ts create mode 100644 src/renderer/views/Moodboard/nodeRendering.test.ts diff --git a/core/src/inline_core/characters/apply.py b/core/src/inline_core/characters/apply.py index 31de78e..e0984be 100644 --- a/core/src/inline_core/characters/apply.py +++ b/core/src/inline_core/characters/apply.py @@ -77,7 +77,9 @@ def _cache_root() -> Path: return data_dir() / "characters" -def char_apply(chosen: str, arch: str = encode.FLUX2_KLEIN_ARCH) -> AppliedCharacter | None: +def char_apply( + chosen: str, arch: str = encode.FLUX2_KLEIN_ARCH, prefer: str | None = None +) -> AppliedCharacter | None: """How a character applies on ``arch``, or None when none is picked. An unreadable pick raises rather than silently generating the wrong person. @@ -105,8 +107,9 @@ def char_apply(chosen: str, arch: str = encode.FLUX2_KLEIN_ARCH) -> AppliedChara lora = _extract_lora(doc, digest, arch) strength = encode.lora_strength(doc.manifest, arch) # A trained adapter wins unless the character says otherwise: the user asked for it explicitly, - # and loading both would apply the identity twice. - mode = doc.manifest.apply.get(arch) or ("lora" if lora else "reference") + # and loading both would apply the identity twice. `prefer` overrides both, for a node that can + # only run one way: H3's reference partition needs a reference and cannot use an adapter alone. + mode = prefer or doc.manifest.apply.get(arch) or ("lora" if lora else "reference") # No reference channel on this arch, so the adapter is the only way it can apply at all. if not references: mode = "lora" diff --git a/core/src/inline_core/characters/charfile.py b/core/src/inline_core/characters/charfile.py index d9e7f1b..2c0782d 100644 --- a/core/src/inline_core/characters/charfile.py +++ b/core/src/inline_core/characters/charfile.py @@ -41,6 +41,18 @@ ) +#: Where a reference came from. Nested inside `refs` rather than a new top-level key on purpose: +#: `from_json` models only the keys it knows, so a top-level addition is destroyed by any older +#: build that rewrites the file, and three code paths rewrite one routinely. +ORIGIN_ORIGINAL = "original" +ORIGIN_HARVESTED = "harvested" + + +def origin_of(ref: dict[str, Any]) -> str: + """Absent means original: every reference written before harvesting existed is one.""" + return str(ref.get("origin") or ORIGIN_ORIGINAL) + + class CharFileError(Exception): """A ``.char`` that cannot be trusted. The message is shown to the user.""" @@ -190,12 +202,29 @@ def write(path: Path | str, doc: CharDoc) -> Path: def refs_fingerprint(manifest: Manifest, policy: dict[str, Any]) -> str: - """Ordered ref hashes plus policy, so a reorder or a resize-budget change both invalidate.""" - parts = [str(ref.get("sha256", "")) for ref in manifest.refs] + """Ordered ref hashes plus policy, so a reorder or a resize-budget change both invalidate. + + Originals only. A harvested reference in here would mark the trained adapter stale the moment + one was taken, and `_extract_lora` drops a stale adapter with an INFO log - so the loop whose + whole point is a better adapter would silently switch off the adapter the user already has. + """ + kept = [ref for ref in manifest.refs if origin_of(ref) == ORIGIN_ORIGINAL] + parts = [str(ref.get("sha256", "")) for ref in kept] parts.append(json.dumps(policy, sort_keys=True, separators=(",", ":"))) return hashlib.sha256("\x1f".join(parts).encode()).hexdigest() +def refs_identity(manifest: Manifest) -> str: + """Which reference set this is, order included and policy excluded. + + Separate from `refs_fingerprint`, which answers "is this payload still valid" and so covers + only the originals and the policy. This answers "were two nodes handed the same character", + which a payload node needs because it compiles from a doc it is not the one holding. + """ + parts = [f"{ref.get('sha256', '')}:{origin_of(ref)}" for ref in manifest.refs] + return hashlib.sha256("\x1f".join(parts).encode()).hexdigest() + + def payload_valid(manifest: Manifest, arch: str, encoder_version: str) -> bool: """Whether ``payloads//`` can be used as-is, or must be recompiled from ``refs/``.""" payload = manifest.payloads.get(arch) diff --git a/core/src/inline_core/characters/encode.py b/core/src/inline_core/characters/encode.py index 1f0d36f..03c8f31 100644 --- a/core/src/inline_core/characters/encode.py +++ b/core/src/inline_core/characters/encode.py @@ -44,7 +44,13 @@ def payload_key(arch: str, kind: str = PAYLOAD_REF) -> str: #: H3 resizes a reference to a 2048 short edge, upscaling included, rounding each axis to 32 on its #: own, with no area cap - so a 4:1 reference is 8192x2048 (vendor/packing_ref2va.py). MINIMAX_H3_ARCH = "minimax-h3" -MINIMAX_H3_POLICY: dict[str, Any] = {"short_edge": 2048, "multiple_of": 32, "max_aspect": 4.0} +#: H3 encodes a reference at a 2048 short edge, which is 4096 vision tokens each through Qwen3-VL. +MINIMAX_H3_SHORT_EDGE = 2048 +MINIMAX_H3_POLICY: dict[str, Any] = { + "short_edge": MINIMAX_H3_SHORT_EDGE, + "multiple_of": 32, + "max_aspect": 4.0, +} REFERENCE_POLICIES: dict[str, dict[str, Any]] = { FLUX2_KLEIN_ARCH: PAYLOAD_POLICY, @@ -55,11 +61,57 @@ def payload_key(arch: str, kind: str = PAYLOAD_REF) -> str: def reference_policy(arch: str) -> dict[str, Any]: return REFERENCE_POLICIES.get(arch, PAYLOAD_POLICY) + +#: What "Resized Reference Resolution" means when left at -1: the model's own policy, uncapped. +NO_REFERENCE_CAP = -1 + + +def capped_policy(arch: str, resolution: int | None) -> dict[str, Any]: + """A model's reference policy, with its target lowered to ``resolution``. + + Capping the source image instead would do nothing for a model whose policy scales *up*: H3 + takes a 2048 short edge whatever it is handed, so a 4K reference and a 512 one cost the same + 36,864 vision tokens. The lever has to be the target the policy resizes onto. + """ + policy = dict(reference_policy(arch)) + if resolution is None or int(resolution) <= 0: + return policy + value = int(resolution) + if "short_edge" in policy: + policy["short_edge"] = min(int(policy["short_edge"]), value) + if "max_pixels" in policy: + policy["max_pixels"] = min(int(policy["max_pixels"]), value * value) + return policy + #: Enough pixels for SFace without carrying a full reference into the file. FACE_CROP_SIZE = 512 +#: 2: per-reference scoring lists are aligned with `manifest.refs`; a v1 file's are compacted, so +#: an index in one cannot be read as a reference position. +SCORING_VERSION = 2 + +ORIGIN_ORIGINAL = cf.ORIGIN_ORIGINAL +ORIGIN_HARVESTED = cf.ORIGIN_HARVESTED +origin_of = cf.origin_of + +#: The harvested pool never outgrows the originals, so they stay at least half of the reference +#: payload and half of the training mix. Absolute cap on top, for a character with many originals. +MAX_HARVESTED = 12 + + +def originals(manifest: cf.Manifest) -> list[dict[str, Any]]: + return [ref for ref in manifest.refs if origin_of(ref) == ORIGIN_ORIGINAL] + + +def harvested(manifest: cf.Manifest) -> list[dict[str, Any]]: + return [ref for ref in manifest.refs if origin_of(ref) == ORIGIN_HARVESTED] + _FACE_EMBEDS = f"scoring/embeds_{scoring.SFACE_ID}.json" _SUBJECT_EMBEDS = f"scoring/embeds_{scoring.DINOV2_ID}.json" +_ORIGINALS_FACE = f"scoring/originals_{scoring.SFACE_ID}.json" +_ORIGINALS_SUBJECT = f"scoring/originals_{scoring.DINOV2_ID}.json" +_HARVEST_FACE = f"scoring/harvested_{scoring.SFACE_ID}.json" +_HARVEST_SUBJECT = f"scoring/harvested_{scoring.DINOV2_ID}.json" def _open(path: Path) -> Any: @@ -76,6 +128,26 @@ def _png_bytes(image: Any) -> bytes: return buffer.getvalue() +def ref_images(doc: cf.CharDoc) -> list[Any]: + """The character's own references decoded, so a rebuild reads truth not the user's library. + + One image per entry in `manifest.refs`, always. Skipping a missing member would shorten the + list and silently shift every scoring position after it onto the wrong reference; a file whose + bytes have gone is unrebuildable either way, so it is reported rather than worked around. + """ + from PIL import Image + + out: list[Any] = [] + for ref in doc.manifest.refs: + member = str(ref.get("path") or "") + data = doc.members.get(member) + if data is None: + raise cf.CharFileError(f"Reference {member} is missing from this character.") + with Image.open(io.BytesIO(data)) as handle: + out.append(handle.convert("RGB").copy()) + return out + + def normalise_reference(image: Any, policy: dict[str, Any] | None = None) -> Any: """A reference resized into a model's budget, on its grid, preserving aspect. @@ -155,8 +227,11 @@ def flags_for(doc: cf.CharDoc) -> dict[str, Any]: def hints_for(manifest: cf.Manifest) -> list[str]: """Hints recomputed from the manifest, so they never go stale against the current rules.""" framings = [float(f) for f in (manifest.scoring.get("refFramings") or [])] + # Originals only, or harvesting three takes onto a one-reference character silences the very + # hints - another angle, a profile, a full-body shot - the harvest pool depends on being met. return strength_hints( - [(int(r.get("width") or 0), int(r.get("height") or 0)) for r in manifest.refs], framings + [(int(r.get("width") or 0), int(r.get("height") or 0)) for r in originals(manifest)], + framings, ) @@ -205,6 +280,7 @@ def char_encode( "width": image.width, "height": image.height, "source_name": path.name, + "origin": cf.ORIGIN_ORIGINAL, } ) @@ -213,12 +289,10 @@ def char_encode( members[text_member] = text_bytes manifest.text = {"path": text_member, "sha256": cf.sha256_bytes(text_bytes)} - crops: list[Any | None] = [] total = len(images) for index, image in enumerate(images): report(0.15 + 0.25 * index / total, f"Finding faces ({index + 1} of {total})…") crop = scoring.face_crop(image) - crops.append(crop) if crop is None: continue crop = crop.resize((FACE_CROP_SIZE, FACE_CROP_SIZE)) @@ -238,7 +312,7 @@ def char_encode( report(0.4, "Normalising reference set…") build_payload(manifest, members, images) - _build_centroids(manifest, members, images, crops, report) + _build_centroids(manifest, members, images, report) # From the manifest the scoring pass just wrote, so the hint and the score cannot disagree. manifest.hints = hints_for(manifest) report(1.0, "Done") @@ -266,8 +340,10 @@ def append_refs(doc: cf.CharDoc, paths: list[Path]) -> None: "width": image.width, "height": image.height, "source_name": path.name, + "origin": cf.ORIGIN_ORIGINAL, } ) + _invalidate_scoring(doc) doc.manifest.modified_at = int(time.time()) @@ -284,9 +360,40 @@ def drop_ref(doc: cf.CharDoc, index: int) -> None: for entry in [d for d in doc.manifest.derived if str(d.get("from") or "") == member]: doc.members.pop(str(entry.get("path") or ""), None) doc.manifest.derived.remove(entry) + # Or scoring keeps best-matching a take against the reference just deleted for being wrong. + _invalidate_scoring(doc) doc.manifest.modified_at = int(time.time()) +def rescore(doc: cf.CharDoc, on_progress: Progress | None = None) -> None: + """Recompute the scoring block from the character's own references, in place. + + Never a re-encode: `char_encode` builds a fresh manifest and members, so rebuilding a stale + character through it drops the trained adapter, every payload but flux2-klein, `apply` and + `reserved` - and the write that follows is what puts that loss on disk. + """ + report = on_progress or (lambda _fraction, _status: None) + _build_centroids(doc.manifest, doc.members, ref_images(doc), report) + doc.manifest.hints = hints_for(doc.manifest) + report(1.0, "Done") + doc.manifest.modified_at = int(time.time()) + + +def _invalidate_scoring(doc: cf.CharDoc) -> None: + """Drop scoring after the reference set changes; `_scoring_stale` then rebuilds it. + + Index surgery is not available: the stored per-reference lists are compacted, so a reference + without a face already shifts every index after it and there is nothing to renumber against. + """ + # The frozen identity survives: it is not derived from the set that just changed. + doc.manifest.scoring = { + key: value for key, value in doc.manifest.scoring.items() if key == "originals" + } + owned = ("scoring/centroid_", "scoring/embeds_") + for member in [m for m in doc.members if m.startswith(owned)]: + doc.members.pop(member, None) + + def payload_stale(manifest: cf.Manifest, key: str) -> bool: """Whether one payload was built from references that have since changed. @@ -317,15 +424,17 @@ def build_payload( members: dict[str, bytes], images: list[Any], arch: str = FLUX2_KLEIN_ARCH, + policy: dict[str, Any] | None = None, ) -> None: """(Re)compile one model's reference set. Public because ``apply`` rebuilds stale ones.""" - policy = reference_policy(arch) + policy = policy or reference_policy(arch) for stale in [m for m in members if m.startswith(f"payloads/{arch}/")]: members.pop(stale, None) files: list[dict[str, Any]] = [] - for index, image in enumerate(images): - member = f"payloads/{arch}/ref_{index:03d}.png" - data = _png_bytes(normalise_reference(image, policy)) + order = _originals_first(manifest, len(images)) + for slot, index in enumerate(order): + member = f"payloads/{arch}/ref_{slot:03d}.png" + data = _png_bytes(normalise_reference(images[index], policy)) members[member] = data files.append({"path": member, "sha256": cf.sha256_bytes(data)}) manifest.payloads[arch] = { @@ -334,10 +443,26 @@ def build_payload( "encoder": {"id": PAYLOAD_ENCODER_ID, "version": PAYLOAD_ENCODER_VERSION}, "source_sha256": cf.refs_fingerprint(manifest, policy), "policy": dict(policy), + # The fingerprint covers originals only, so this is the only record of what else went in. + "harvested_count": len(order) - len(originals(manifest)), "files": files, } +def _originals_first(manifest: cf.Manifest, count: int) -> list[int]: + """Reference positions with originals ahead of harvested ones. + + Position is meaning - FLUX.2 addresses a reference by its number - so the ones the user + vouched for take the leading slots. Sorted here rather than kept sorted in the manifest, + because `append_refs` adds at the end and `drop_ref` refuses to renumber. + """ + refs = manifest.refs[:count] + ordered = [i for i, ref in enumerate(refs) if cf.origin_of(ref) == cf.ORIGIN_ORIGINAL] + ordered += [i for i, ref in enumerate(refs) if cf.origin_of(ref) == cf.ORIGIN_HARVESTED] + # Images that do not line up with the manifest keep their order, not one read off other refs. + return ordered if len(ordered) == count else list(range(count)) + + def set_lora_payload( manifest: cf.Manifest, members: dict[str, bytes], @@ -393,48 +518,253 @@ def lora_payload(manifest: cf.Manifest, arch: str = FLUX2_KLEIN_ARCH) -> dict[st return entry if isinstance(entry, dict) else None +def can_freeze(manifest: cf.Manifest) -> bool: + """Whether there are enough originals for a frozen gallery to mean anything.""" + return len(originals(manifest)) >= scoring.MIN_REFS_TO_FLAG + + +def originals_frozen(manifest: cf.Manifest) -> bool: + return bool((manifest.scoring.get("originals") or {}).get("refs")) + + +def originals_stale(manifest: cf.Manifest) -> bool: + """Whether the frozen gallery's vectors were built by encoders that have since moved.""" + frozen = manifest.scoring.get("originals") or {} + recorded = {str(e.get("id")): str(e.get("version")) for e in (frozen.get("encoders") or [])} + return recorded != scoring.encoder_versions_by_id() + + +def freeze_originals(doc: cf.CharDoc) -> dict[str, Any]: + """Establish the identity target: which references are the originals, and their embeddings. + + The frozen thing is the membership. The vectors beside it are cache keyed by encoder version, + because cosine across two encoder builds is a number with no meaning while the pixels are + still in the file - the same rule payloads already follow. + """ + if not can_freeze(doc.manifest): + raise ValueError( + f"A character needs {scoring.MIN_REFS_TO_FLAG} original references before its identity " + "can be frozen: below that, the odd one out cannot be told from the rest." + ) + entries = originals(doc.manifest) + doc.manifest.scoring["originals"] = { + "refs": [ + {"path": str(ref.get("path") or ""), "sha256": str(ref.get("sha256") or "")} + for ref in entries + ], + "vectors": _ORIGINALS_FACE, + "subjectVectors": _ORIGINALS_SUBJECT, + "encoders": scoring.encoder_versions(), + "frozenAt": int(time.time()), + } + _embed_frozen(doc) + return doc.manifest.scoring["originals"] + + +def frozen_originals(doc: cf.CharDoc) -> tuple[list[list[float]], list[list[float]]]: + """The frozen gallery's face and subject vectors, re-embedded if the encoders have moved.""" + if not originals_frozen(doc.manifest): + raise ValueError("This character has no frozen identity yet.") + if originals_stale(doc.manifest): + _embed_frozen(doc) + doc.manifest.scoring["originals"]["encoders"] = scoring.encoder_versions() + face = scoring.load_keyed(doc.members, _ORIGINALS_FACE) + subject = scoring.load_keyed(doc.members, _ORIGINALS_SUBJECT) + paths = [str(r.get("path")) for r in doc.manifest.scoring["originals"]["refs"]] + return ([face.get(p) or [] for p in paths], [subject.get(p) or [] for p in paths]) + + +def _embed_frozen(doc: cf.CharDoc) -> None: + """(Re)build the frozen gallery's vectors from exactly the references it was frozen over.""" + from PIL import Image + + face: dict[str, list[float]] = {} + subject: dict[str, list[float]] = {} + for ref in doc.manifest.scoring["originals"]["refs"]: + path = str(ref.get("path") or "") + data = doc.members.get(path) + if data is None: + raise cf.CharFileError(f"The frozen reference {path} is missing, so identity is gone.") + # A changed reference means the frozen set was tampered with; re-embedding would launder it. + if cf.sha256_bytes(data) != str(ref.get("sha256") or ""): + raise cf.CharFileError(f"The frozen reference {path} has changed since it was frozen.") + with Image.open(io.BytesIO(data)) as handle: + image = handle.convert("RGB").copy() + if vector := scoring.embed_face(image): + face[path] = vector + if vector := scoring.embed_subject(image): + subject[path] = vector + doc.members[_ORIGINALS_FACE] = scoring.dump_keyed(face) + doc.members[_ORIGINALS_SUBJECT] = scoring.dump_keyed(subject) + + +def quarantine_ref(doc: cf.CharDoc, index: int) -> str: + """Take a reference out of the set but keep its bytes, so removal stays reversible. + + `drop_ref` pops the member outright, and refs are truth: the file it came from may be long + gone and Write persists over the character with no undo. + """ + if not 0 <= index < len(doc.manifest.refs): + raise ValueError("That reference is not in this character.") + data = doc.members.get(str(doc.manifest.refs[index].get("path") or "")) + drop_ref(doc, index) + if data is None: + return "" + member = cf.member_name("quarantined", _next_slot(doc, "quarantined"), ".png") + doc.members[member] = data + return member + + +def harvest_cap(manifest: cf.Manifest) -> int: + """Never more harvested than originals, so the user's own references stay at least half.""" + return min(MAX_HARVESTED, len(originals(manifest))) + + +def add_harvested( + doc: cf.CharDoc, image: Any, *, agreement: float | None, score: float, source_take: str = "" +) -> int: + """Add an approved take to the harvested pool. Returns its position in `manifest.refs`.""" + member = cf.member_name("harvested", _next_slot(doc, "harvested"), ".png") + data = _png_bytes(image) + doc.members[member] = data + doc.manifest.refs.append( + { + "path": member, + "sha256": cf.sha256_bytes(data), + "width": image.width, + "height": image.height, + "source_name": source_take or member, + "origin": cf.ORIGIN_HARVESTED, + "agreement": agreement, + "score": score, + "harvestedAt": int(time.time()), + "sourceTake": source_take, + } + ) + pool_face = scoring.load_keyed(doc.members, _HARVEST_FACE) + pool_subject = scoring.load_keyed(doc.members, _HARVEST_SUBJECT) + if vector := scoring.embed_face(image): + pool_face[member] = vector + if vector := scoring.embed_subject(image): + pool_subject[member] = vector + doc.members[_HARVEST_FACE] = scoring.dump_keyed(pool_face) + doc.members[_HARVEST_SUBJECT] = scoring.dump_keyed(pool_subject) + _invalidate_scoring(doc) + doc.manifest.modified_at = int(time.time()) + return len(doc.manifest.refs) - 1 + + +def prune_harvested(doc: cf.CharDoc) -> list[str]: + """Trim the pool to the cap, least distinctive first. Originals are never candidates. + + Coverage rather than score, because a pool of near-duplicates of the best-scoring angle is + worth less to a compile or a train than one that spans the angles the originals miss. + """ + removed: list[str] = [] + subject = scoring.load_keyed(doc.members, _HARVEST_SUBJECT) + frozen = [v for v in scoring.load_keyed(doc.members, _ORIGINALS_SUBJECT).values() if v] + while len(harvested(doc.manifest)) > harvest_cap(doc.manifest): + pool = [ + (index, str(ref.get("path"))) + for index, ref in enumerate(doc.manifest.refs) + if cf.origin_of(ref) == cf.ORIGIN_HARVESTED + ] + # Originals first in the list, so each candidate is measured against them as well as + # against the rest of the pool; only the pool's own slice is a candidate for dropping. + values = scoring.coverage_values(frozen + [subject.get(p) or [] for _i, p in pool])[ + len(frozen) : + ] + # Unmeasurable first: a candidate with no embedding has no coverage to argue for keeping it. + worst = min(range(len(pool)), key=lambda n: (values[n] is not None, values[n] or 0.0)) + index, member = pool[worst] + drop_ref(doc, index) + subject.pop(member, None) + removed.append(member) + if removed: + doc.members[_HARVEST_SUBJECT] = scoring.dump_keyed(subject) + face = scoring.load_keyed(doc.members, _HARVEST_FACE) + for member in removed: + face.pop(member, None) + doc.members[_HARVEST_FACE] = scoring.dump_keyed(face) + return removed + + +def _next_slot(doc: cf.CharDoc, prefix: str) -> int: + index = 0 + while cf.member_name(prefix, index, ".png") in doc.members: + index += 1 + return index + + def _build_centroids( manifest: cf.Manifest, members: dict[str, bytes], images: list[Any], - crops: list[Any | None], report: Progress = lambda _fraction, _status: None, ) -> None: centroids: dict[str, str] = {} + previous_scoring = dict(manifest.scoring) + # Only what this function owns: the frozen originals and the harvested pool outlive a rescore. + for stale in [m for m in members if m.startswith(("scoring/centroid_", "scoring/embeds_"))]: + members.pop(stale, None) + originals = [i for i, ref in enumerate(manifest.refs) if origin_of(ref) == ORIGIN_ORIGINAL] report(0.55, "Loading identity encoders…") # Whole frame, not the crop: SFace self-aligns, and mismatching the two sides costs ~10 points. - face_vectors = [v for image in images if (v := scoring.embed_face(image))] + # Aligned with `manifest.refs`, empty where no face was found, so a flag names a position. + face_slots = [scoring.embed_face(image) or [] for image in images] + face_vectors = [face_slots[i] for i in originals if face_slots[i]] face_centroid = scoring.mean_vector(face_vectors) if face_centroid: member = f"scoring/centroid_{scoring.SFACE_ID}.json" members[member] = scoring.dump_centroid(face_centroid, len(face_vectors)) centroids[scoring.SFACE_ID] = member # Every view kept, not just their mean: the face term matches the best-fitting one. - members[_FACE_EMBEDS] = scoring.dump_embeds(face_vectors) + members[_FACE_EMBEDS] = scoring.dump_embeds(_masked(face_slots, originals)) report(0.7, "Measuring the subject…") - subject_vectors = [v for image in images if (v := scoring.embed_subject(image))] + subject_slots = [scoring.embed_subject(image) or [] for image in images] + subject_vectors = [subject_slots[i] for i in originals if subject_slots[i]] subject_centroid = scoring.mean_vector(subject_vectors) if subject_centroid: member = f"scoring/centroid_{scoring.DINOV2_ID}.json" members[member] = scoring.dump_centroid(subject_centroid, len(subject_vectors)) centroids[scoring.DINOV2_ID] = member # Keep every view: a mean over chest-up refs matches none of them. - members[_SUBJECT_EMBEDS] = scoring.dump_embeds(subject_vectors) + members[_SUBJECT_EMBEDS] = scoring.dump_embeds(_masked(subject_slots, originals)) report(0.9, "Measuring reference coverage…") - # How wide each reference is, so scoring can tell whether the gallery covers a take's framing. - framings = [f for image in images if (f := scoring.face_fraction(image)) is not None] + # Compacted, not aligned: it is read as an unordered bag, and a null in it raises `hints_for`. + framings = [ + f for i in originals if (f := scoring.face_fraction(images[i])) is not None + ] manifest.scoring = { "encoders": scoring.encoder_versions(), + # 2: per-reference lists are aligned with `manifest.refs`; a v1 file's are compacted. + "version": SCORING_VERSION, + # So adding or dropping a reference is detectable without running an encoder to find out. + "refCount": len(manifest.refs), "centroids": centroids, "faceEmbeds": _FACE_EMBEDS if face_centroid else "", "subjectEmbeds": _SUBJECT_EMBEDS if subject_centroid else "", "refFramings": framings, - "refAgreement": scoring.reference_agreement(face_vectors), - "flaggedRefs": scoring.flagged_references(face_vectors), + "refAgreement": scoring.reference_agreement(face_slots), + "flaggedRefs": scoring.flagged_references(face_slots), "face_bearing": bool(face_centroid), "blend": {"face": scoring.FACE_WEIGHT, "subject": scoring.SUBJECT_WEIGHT}, } + # Carried across, because neither is derived from the set this pass just measured. + for key in ("originals", "harvested", "verification"): + if key in previous_scoring: + manifest.scoring[key] = previous_scoring[key] + + +def _masked(slots: list[list[float]], keep: list[int]) -> list[list[float]]: + """The aligned slots with everything outside `keep` blanked. + + This is what keeps the take-scoring gallery originals-only while staying ref-aligned: `score` + already drops empty vectors, so a blanked position simply is not a candidate to match against. + """ + allowed = set(keep) + return [vector if i in allowed else [] for i, vector in enumerate(slots)] diff --git a/core/src/inline_core/characters/library.py b/core/src/inline_core/characters/library.py index c3f2818..0ca2dfc 100644 --- a/core/src/inline_core/characters/library.py +++ b/core/src/inline_core/characters/library.py @@ -86,7 +86,11 @@ def summaries() -> list[dict[str, Any]]: "file": path.name, "charId": manifest.char_id, "name": manifest.name or path.stem, - "refs": len(manifest.refs), + # The references the user curated. Harvested ones are counted separately, or a + # character reads as stronger than it is because it absorbed its own output. + "refs": len(encode.originals(manifest)), + "harvested": len(encode.harvested(manifest)), + "flagged": list(manifest.scoring.get("flaggedRefs") or []), "createdAt": manifest.created_at, "modifiedAt": manifest.modified_at, "hints": encode.hints_for(manifest), diff --git a/core/src/inline_core/characters/scoring.py b/core/src/inline_core/characters/scoring.py index 3e594b5..5dcfdfc 100644 --- a/core/src/inline_core/characters/scoring.py +++ b/core/src/inline_core/characters/scoring.py @@ -74,6 +74,27 @@ def use_encoders( ) +def use_encoders_from(scoring_block: dict[str, Any]) -> None: + """Pin the encoders to the ones a character was scored with. + + `_chosen` is module state that nothing resets, so without this one Encode Character node run + with a picked annotator marks every other character's centroid stale for the rest of the + process - and each one is then rewritten on the next take it is scored against. + """ + picked: dict[str, str] = {} + for entry in scoring_block.get("encoders") or []: + if not isinstance(entry, dict): + continue + version = str(entry.get("version") or "") + name = version.split(":", 1)[1] if ":" in version else "" + if str(entry.get("id")) == SFACE_ID: + picked["sface"] = name + elif str(entry.get("id")) == DINOV2_ID: + picked["dinov2"] = name + # The detector is not version-tracked, so it can only go back to the shipped default here. + use_encoders(face_embedder=picked.get("sface", ""), subject_embedder=picked.get("dinov2", "")) + + def chosen(kind: str, default: str) -> str: """The picked filename for an encoder, or the shipped one.""" return _chosen.get(kind) or default @@ -349,13 +370,22 @@ def score( } -def reference_agreement(face_refs: list[list[float]]) -> list[float]: - """Each reference's mean SFace similarity to the others, 0-100.""" - if len(face_refs) < 2: - return [100.0] * len(face_refs) - out: list[float] = [] +def reference_agreement(face_refs: list[list[float]]) -> list[float | None]: + """Each reference's mean SFace similarity to the others, 0-100, aligned with the input. + + `None` is a reference with no face to compare, which is not a low score. `cosine` against an + empty vector is 0.0, so letting one into the average drags every genuine reference's mean + toward the floor - four references with two wide shots among them would all fall under it. + """ + measured = [i for i, vector in enumerate(face_refs) if vector] + if len(measured) < 2: + return [100.0 if vector else None for vector in face_refs] + out: list[float | None] = [] for i, vector in enumerate(face_refs): - others = [v for j, v in enumerate(face_refs) if j != i] + if not vector: + out.append(None) + continue + others = [face_refs[j] for j in measured if j != i] out.append(round(sum(to_percent(cosine(vector, o)) for o in others) / len(others), 1)) return out @@ -363,10 +393,42 @@ def reference_agreement(face_refs: list[list[float]]) -> list[float]: def flagged_references(face_refs: list[list[float]]) -> list[int]: """Indices of references that may not be the same person. Face identity only - a different outfit or setting is exactly what the user was asked for and must not trip this.""" - if len(face_refs) < MIN_REFS_TO_FLAG: + if sum(1 for vector in face_refs if vector) < MIN_REFS_TO_FLAG: return [] scores = reference_agreement(face_refs) - return [i for i, s in enumerate(scores) if s < REFERENCE_AGREEMENT_FLOOR] + return [i for i, s in enumerate(scores) if s is not None and s < REFERENCE_AGREEMENT_FLOOR] + + +def agreement_against(candidate: list[float], gallery: list[list[float]]) -> float | None: + """A candidate's mean SFace similarity to a fixed gallery, or None when it cannot be measured. + + Separate from `reference_agreement` because the candidate is not a member of the gallery, so + there is nothing to leave out. Below `MIN_REFS_TO_FLAG` genuine faces it declines to answer: + against one or two references this is a pairwise number, and the floor is a mean over a set. + """ + usable = [vector for vector in gallery if vector] + if not candidate or len(usable) < MIN_REFS_TO_FLAG: + return None + return round(sum(to_percent(cosine(candidate, v)) for v in usable) / len(usable), 1) + + +def coverage_values(subject_refs: list[list[float]]) -> list[float | None]: + """How much each reference shows that the others do not: 1 minus its closest cosine to them. + + DINOv2, never SFace: this measures framing and setting, which is the axis a gallery needs to + span, and is exactly the axis that must never decide identity. + """ + measured = [i for i, vector in enumerate(subject_refs) if vector] + if len(measured) < 2: + return [1.0 if vector else None for vector in subject_refs] + out: list[float | None] = [] + for i, vector in enumerate(subject_refs): + if not vector: + out.append(None) + continue + others = [subject_refs[j] for j in measured if j != i] + out.append(round(1.0 - max(cosine(vector, o) for o in others), 4)) + return out def load_centroids(members: dict[str, bytes], paths: dict[str, Any]) -> dict[str, list[float]]: @@ -394,6 +456,26 @@ def dump_centroid(vector: list[float], count: int) -> bytes: return json.dumps({"vector": vector, "count": count}, separators=(",", ":")).encode() +def dump_keyed(vectors: dict[str, list[float]]) -> bytes: + import json + + return json.dumps({"vectors": vectors}, separators=(",", ":")).encode() + + +def load_keyed(members: dict[str, bytes], path: str) -> dict[str, list[float]]: + """Vectors keyed by member name, for a pool where a position is not a stable identifier.""" + import json + + raw = members.get(str(path)) + if not raw: + return {} + try: + parsed = (json.loads(raw).get("vectors") or {}).items() + except (ValueError, TypeError, AttributeError): + return {} + return {str(k): [float(x) for x in v] for k, v in parsed} + + def dump_embeds(vectors: list[list[float]]) -> bytes: import json diff --git a/core/src/inline_core/characters/verify.py b/core/src/inline_core/characters/verify.py new file mode 100644 index 0000000..587ea8e --- /dev/null +++ b/core/src/inline_core/characters/verify.py @@ -0,0 +1,172 @@ +"""Checking a character's reference set before anything compiles a payload or trains on it. + +Face identity only. A different outfit, setting or framing is exactly what the user was asked for, +so the subject term never decides whether a reference belongs - it only measures coverage. +""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass, field +from typing import Any + +from . import charfile as cf +from . import encode, scoring + +logger = logging.getLogger("inline_core.characters") + +#: Verified against the character's own frozen gallery, or against nothing but itself. +MODE_EXISTING = "existing" +MODE_BOOTSTRAP = "bootstrap" + + +@dataclass +class Verdict: + """What a pass found, with every list holding positions into ``manifest.refs``.""" + + mode: str + floor: float + agreement: list[float | None] = field(default_factory=list) + flagged: list[int] = field(default_factory=list) + duplicates: list[int] = field(default_factory=list) + unchecked: list[int] = field(default_factory=list) + note: str = "" + + def to_json(self) -> dict[str, Any]: + return { + "mode": self.mode, + "floor": self.floor, + "agreement": self.agreement, + "flagged": self.flagged, + "duplicates": self.duplicates, + "unchecked": self.unchecked, + "note": self.note, + } + + +def duplicate_positions(manifest: cf.Manifest) -> list[int]: + """Later copies of a reference already in the set, by content hash. No encoder needed. + + A duplicate is not a judgement call, and it is not harmless: it doubles that image's weight in + a training mix and spends a reference slot the model addresses by position. + """ + seen: set[str] = set() + out: list[int] = [] + for index, ref in enumerate(manifest.refs): + digest = str(ref.get("sha256") or "") + if digest and digest in seen: + out.append(index) + seen.add(digest) + return out + + +def verify( + doc: cf.CharDoc, + *, + floor: float = scoring.REFERENCE_AGREEMENT_FLOOR, + on_progress: encode.Progress | None = None, +) -> Verdict: + """Score every reference against the character's identity, without changing anything.""" + report = on_progress or (lambda _fraction, _status: None) + duplicates = duplicate_positions(doc.manifest) + existing = encode.originals_frozen(doc.manifest) + verdict = Verdict(mode=MODE_EXISTING if existing else MODE_BOOTSTRAP, floor=floor) + verdict.duplicates = duplicates + + images = encode.ref_images(doc) + total = len(images) + slots: list[list[float]] = [] + for index, image in enumerate(images): + report(0.1 + 0.6 * index / max(1, total), f"Checking reference {index + 1} of {total}…") + slots.append(scoring.embed_face(image) or []) + verdict.unchecked = [i for i, vector in enumerate(slots) if not vector] + + live = [i for i in range(len(slots)) if i not in set(duplicates)] + if existing: + verdict.agreement = _against_frozen(doc, slots, live) + else: + # Dedup first: a duplicate agrees with its twin at 100 and lifts the mean it is judged by. + masked = [slots[i] if i in set(live) else [] for i in range(len(slots))] + verdict.agreement = scoring.reference_agreement(masked) + for index in duplicates: + verdict.agreement[index] = None + + measured = [i for i, value in enumerate(verdict.agreement) if value is not None] + if len(measured) < scoring.MIN_REFS_TO_FLAG: + verdict.note = ( + f"{len(measured)} reference(s) with a usable face: below {scoring.MIN_REFS_TO_FLAG} " + "there is no way to tell which one is the odd one out, so none were flagged." + ) + return verdict + verdict.flagged = [i for i in measured if (verdict.agreement[i] or 0.0) < floor] + return verdict + + +def _against_frozen( + doc: cf.CharDoc, slots: list[list[float]], live: list[int] +) -> list[float | None]: + """Each reference's mean similarity to the frozen originals, leaving out its own vector. + + Against the frozen gallery rather than the live set, so a harvested reference is measured by + what the user vouched for and can never drift the target it is being measured against. + """ + gallery, _subject = encode.frozen_originals(doc) + frozen_paths = [str(r.get("path")) for r in doc.manifest.scoring["originals"]["refs"]] + by_path = dict(zip(frozen_paths, gallery, strict=True)) + out: list[float | None] = [] + for index, vector in enumerate(slots): + path = str(doc.manifest.refs[index].get("path") or "") + if not vector or index not in set(live): + out.append(None) + continue + others = [v for p, v in by_path.items() if p != path and v] + out.append(scoring.agreement_against(vector, others)) + return out + + +def apply_verdict(doc: cf.CharDoc, verdict: Verdict, *, quarantine: bool) -> dict[str, list[str]]: + """Act on a verdict. Duplicates always go; a flagged reference only when asked. + + Removed from the back, so a position still names the reference the report named. + """ + before = [str(ref.get("path") or "") for ref in doc.manifest.refs] + removed: dict[str, list[str]] = {"duplicates": [], "quarantined": []} + targets = set(verdict.duplicates) + if quarantine: + targets |= set(verdict.flagged) + for index in sorted(targets, reverse=True): + if len(doc.manifest.refs) <= 1: + logger.info("Keeping reference 1: a character needs at least one.") + break + if index in set(verdict.duplicates): + # Byte-identical to one that stays, so there is nothing to preserve a copy of. + encode.drop_ref(doc, index) + removed["duplicates"].append(before[index]) + else: + removed["quarantined"].append(encode.quarantine_ref(doc, index)) + _reindex(verdict, before, [str(ref.get("path") or "") for ref in doc.manifest.refs]) + doc.manifest.scoring["verification"] = { + **verdict.to_json(), + "checkedAt": int(time.time()), + "removed": removed, + } + return removed + + +def _reindex(verdict: Verdict, before: list[str], after: list[str]) -> None: + """Rewrite the verdict's positions onto the set that survived. + + Every list it holds is a position into `manifest.refs`, and a removal shifts each position + after it - so a report stored as it was found would ring a reference that is now another one. + """ + if before == after: + return + landed = {path: index for index, path in enumerate(after)} + moved = {old: landed[path] for old, path in enumerate(before) if path in landed} + verdict.agreement = [ + verdict.agreement[old] for old in sorted(moved, key=lambda old: moved[old]) + ] + verdict.flagged = sorted(moved[i] for i in verdict.flagged if i in moved) + verdict.unchecked = sorted(moved[i] for i in verdict.unchecked if i in moved) + verdict.duplicates = sorted(moved[i] for i in verdict.duplicates if i in moved) diff --git a/core/src/inline_core/models/character/runner.py b/core/src/inline_core/models/character/runner.py index b33a562..f94bdce 100644 --- a/core/src/inline_core/models/character/runner.py +++ b/core/src/inline_core/models/character/runner.py @@ -15,7 +15,7 @@ from typing import Any from ...characters import charfile as cf -from ...characters import encode, library, scoring, weights +from ...characters import encode, library, scoring, verify, weights from ...graph.descriptor import NodeDescriptor, Option, ParamField, Port, Widget from ...graph.runners import NodeResult, NodeRunner from ...graph.schema import Node, PortKind @@ -47,6 +47,10 @@ class Payload: arch: str kind: str apply: Any + #: The reference set this was compiled against. A payload node uses its `character` input only + #: to read settings - the doc it compiles from is the one Write hands it - so without this a + #: graph that wires Write ahead of the verify node compiles the unverified set and says nothing. + source_sha256: str = "" ENCODE = NodeDescriptor( @@ -94,6 +98,83 @@ class Payload: "arch", "Model", Widget.SELECT, encode.FLUX2_KLEIN_ARCH, options=tuple(Option(value=a, label=a) for a in encode.REFERENCE_POLICIES), ), + # On the face: it decides both the file size and, for H3, whether the run fits the card. + ParamField( + "ref_resolution", "Resized Reference Resolution", Widget.NUMBER, 1024, + min=encode.NO_REFERENCE_CAP, max=8192, step=64, on_face=True, + ), + ), +) + +VERIFY_REFS = NodeDescriptor( + type="character/verify-refs", + title="Verify References", + category="Character", + icon="sparkles", + output_kind=None, + inputs=(Port("character", "Character", PortKind.CHARACTER, required=True),), + outputs=(Port("character", "Character", PortKind.CHARACTER),), + params=( + # On the face: whether a flagged reference is removed is the whole behaviour of the node. + ParamField( + "on_outlier", "When a reference looks wrong", Widget.SELECT, "flag", on_face=True, + options=( + Option(value="flag", label="Flag it (keep it)"), + Option(value="quarantine", label="Take it out (reversible)"), + ), + ), + # Surfaced, because it is measured rather than chosen and a set may sit close to it. + ParamField( + "floor", "Agreement floor", Widget.NUMBER, scoring.REFERENCE_AGREEMENT_FLOOR, + min=0.0, max=100.0, step=0.5, + ), + # The encoders are pickable, not just visible: a node that silently uses a file the user + # cannot see or change is the reason none of them showed up as missing. + ParamField( + "face_detector", "Face detector", Widget.SELECT, weights.YUNET_FILE, + options_from="annotators", + ), + ParamField( + "face_embedder", "Face embedder", Widget.SELECT, weights.SFACE_FILE, + options_from="annotators", + ), + ParamField( + "subject_embedder", "Subject embedder", Widget.SELECT, weights.DINOV2_DIR, + options_from="annotators", + ), + ), +) + +INGEST = NodeDescriptor( + type="character/ingest-approved", + title="Harvest Approved Take", + category="Character", + icon="sparkles", + output_kind=None, + inputs=( + Port("character", "Character", PortKind.CHARACTER, required=True), + Port("image", "Approved take", PortKind.IMAGE, required=True), + ), + outputs=(Port("character", "Character", PortKind.CHARACTER),), + params=( + # Provisional: the continuity numbers this is compared against were measured on real + # photographs, and nothing has yet measured a generated take against a frozen gallery. + ParamField( + "min_score", "Minimum continuity", Widget.NUMBER, 70.0, + min=0.0, max=100.0, step=1.0, on_face=True, + ), + ParamField( + "face_detector", "Face detector", Widget.SELECT, weights.YUNET_FILE, + options_from="annotators", + ), + ParamField( + "face_embedder", "Face embedder", Widget.SELECT, weights.SFACE_FILE, + options_from="annotators", + ), + ParamField( + "subject_embedder", "Subject embedder", Widget.SELECT, weights.DINOV2_DIR, + options_from="annotators", + ), ), ) @@ -235,6 +316,8 @@ def register_character_nodes(registry: Any) -> None: _dataset_runner = CharacterDatasetRunner() registry.register(DATASET, _dataset_runner) registry.register(EDIT, EditCharacterRunner()) + registry.register(VERIFY_REFS, VerifyReferencesRunner()) + registry.register(INGEST, IngestApprovedRunner()) registry.register(COMPILE_REFS, CompileReferencesRunner()) registry.register(ATTACH, AttachAdapterRunner()) registry.register(WRITE, WriteCharacterRunner()) @@ -248,14 +331,7 @@ class EncodeCharacterRunner(NodeRunner): """References plus a description into an identity: crops, embeddings, framings, hints.""" def run(self, node: Node, inputs: dict[str, list[Any]], ctx: ExecutionContext) -> NodeResult: - from ...characters import weights - - if not weights.present(): - raise ValueError( - "The character encoders are not downloaded yet: " - "face_detection_yunet_2023mar.onnx, face_recognition_sface_2021dec.onnx and " - "dinov2-base, about 385MB in models/annotators." - ) + _require_encoders() refs = list(inputs.get("images") or []) if not refs: raise ValueError("A character needs at least one reference image.") @@ -339,6 +415,133 @@ def _set_description(doc: cf.CharDoc, description: str) -> None: doc.manifest.text = {"path": member, "sha256": cf.sha256_bytes(data)} +def _resolution(raw: Any) -> int: + """The resolution param, defaulting to the cap rather than to uncapped: a graph saved before + this param existed carries no value, and silently compiling those at 2048 is what OOMs.""" + try: + value = int(raw) + except (TypeError, ValueError): + return 1024 + return value if value > 0 else encode.NO_REFERENCE_CAP + + +def _describe_policy(policy: dict[str, Any]) -> str: + """What the setting resolved to, which is not what was typed: a policy states a short edge or + an area cap, never both.""" + if "short_edge" in policy: + return f"a {policy['short_edge']}px short edge" + pixels = int(policy.get("max_pixels", 0)) + return f"at most {pixels:,} pixels (about {int(pixels ** 0.5)}px square)" + + +class VerifyReferencesRunner(NodeRunner): + """Check the reference set before a payload or a training set is built from it. + + Sits in front of both consumers because both read `manifest.refs` and neither looks: a + reference of the wrong person drags a reference payload, and a LoRA bakes it in. + """ + + def run(self, node: Node, inputs: dict[str, list[Any]], ctx: ExecutionContext) -> NodeResult: + identity = _first(inputs.get("character")) + if not isinstance(identity, Identity): + raise ValueError("Verify References needs a character.") + _require_encoders() + _use_encoders(node) + # Copied, because the upstream node's output is cached and every other reader shares it. + doc = copy.deepcopy(identity.doc) + floor = _as_float(node.params.get("floor"), scoring.REFERENCE_AGREEMENT_FLOOR) + + def report(fraction: float, status: str) -> None: + ctx.emitter.emit(progress_event(ctx, node, Phase.ENCODE, fraction, status=status)) + + verdict = verify.verify(doc, floor=floor, on_progress=report) + for index, value in enumerate(verdict.agreement): + if value is not None: + mark = " - flagged" if index in verdict.flagged else "" + report(0.75, f"Reference {index + 1}: {value}% agreement{mark}") + quarantine = str(node.params.get("on_outlier") or "flag") == "quarantine" + removed = verify.apply_verdict(doc, verdict, quarantine=quarantine) + + # Frozen from what survived, and only once: a set nothing has checked is not an identity. + if verdict.mode == verify.MODE_BOOTSTRAP and encode.can_freeze(doc.manifest): + report(0.9, "Freezing the original references…") + encode.freeze_originals(doc) + logger.info( + "Verified %s (%s): %d flagged, %d duplicate(s), %d unchecked, %d removed. %s", + doc.manifest.name, verdict.mode, len(verdict.flagged), len(verdict.duplicates), + len(verdict.unchecked), sum(len(v) for v in removed.values()), verdict.note, + ) + report(1.0, "Done") + return NodeResult(outputs={"character": Identity(doc=doc, file=identity.file)}) + + +class IngestApprovedRunner(NodeRunner): + """Add an approved take to the harvested pool, scored against the frozen originals. + + Never wired into the compile or train chain: harvesting is its own small graph, and the two + meet at the `.char` file rather than at a wire. + """ + + def run(self, node: Node, inputs: dict[str, list[Any]], ctx: ExecutionContext) -> NodeResult: + identity = _first(inputs.get("character")) + if not isinstance(identity, Identity): + raise ValueError("Harvest Approved Take needs a character.") + take = _first(inputs.get("image")) + if take is None: + raise ValueError("Harvest Approved Take needs an image.") + _require_encoders() + _use_encoders(node) + doc = copy.deepcopy(identity.doc) + if not encode.originals_frozen(doc.manifest): + raise ValueError( + "Run Verify References on this character first: harvesting is measured against " + "its frozen original references, and it has none yet." + ) + + from PIL import Image, ImageOps + + with Image.open(_image_path(take)) as handle: + image = ImageOps.exif_transpose(handle).convert("RGB") + + face_gallery, subject_gallery = encode.frozen_originals(doc) + centroids = scoring.load_centroids(doc.members, doc.manifest.scoring.get("centroids") or {}) + framings = [float(f) for f in (doc.manifest.scoring.get("refFramings") or [])] + result = scoring.score(image, centroids, face_gallery, subject_gallery, framings) + if result is None or not result.get("faceBearing"): + raise ValueError( + "That take has no face this character's encoders can measure, so there is nothing " + "to check it against. Only the originals are taken on trust." + ) + minimum = _as_float(node.params.get("min_score"), 70.0) + score = float(result["score"]) + if score < minimum: + raise ValueError( + f"That take scores {score} against {doc.manifest.name}'s original references, " + f"under the {minimum} this node asks for. Harvesting it would teach the drift." + ) + + agreement = scoring.agreement_against(scoring.embed_face(image) or [], face_gallery) + encode.add_harvested( + doc, image, agreement=agreement, score=score, source_take=str(getattr(take, "ref", "")) + ) + dropped = encode.prune_harvested(doc) + logger.info( + "Harvested a take into %s at %s: %d in the pool, cap %d, %d pruned", + doc.manifest.name, score, len(encode.harvested(doc.manifest)), + encode.harvest_cap(doc.manifest), len(dropped), + ) + return NodeResult(outputs={"character": Identity(doc=doc, file=identity.file)}) + + +def _require_encoders() -> None: + if not weights.present(): + raise ValueError( + "The character encoders are not downloaded yet: " + "face_detection_yunet_2023mar.onnx, face_recognition_sface_2021dec.onnx and " + "dinov2-base, about 385MB in models/annotators." + ) + + class CompileReferencesRunner(NodeRunner): """One model's reference set: each reference resized to what that model accepts.""" @@ -347,11 +550,20 @@ def run(self, node: Node, inputs: dict[str, list[Any]], ctx: ExecutionContext) - if not isinstance(identity, Identity): raise ValueError("Compile References needs a character.") arch = str(node.params.get("arch") or encode.FLUX2_KLEIN_ARCH) + policy = encode.capped_policy(arch, _resolution(node.params.get("ref_resolution"))) + logger.info( + "Compiling %s references at %s", arch, _describe_policy(policy) + ) def apply(doc: cf.CharDoc) -> None: - encode.build_payload(doc.manifest, doc.members, _ref_images(doc), arch=arch) + encode.build_payload(doc.manifest, doc.members, _ref_images(doc), arch, policy) - payload = Payload(arch=arch, kind=encode.PAYLOAD_REF, apply=apply) + payload = Payload( + arch=arch, + kind=encode.PAYLOAD_REF, + apply=apply, + source_sha256=cf.refs_identity(identity.doc.manifest), + ) return NodeResult(outputs={"payload": payload}) @@ -364,8 +576,17 @@ def run(self, node: Node, inputs: dict[str, list[Any]], ctx: ExecutionContext) - raise ValueError("Write .char needs a character.") doc = identity.doc for payload in inputs.get("payloads") or []: - if isinstance(payload, Payload): - payload.apply(doc) + if not isinstance(payload, Payload): + continue + # A payload node compiles from the doc Write hands it, not from its own input, so + # wiring Write ahead of a verify node would silently save the unchecked set. + if payload.source_sha256 and payload.source_sha256 != cf.refs_identity(doc.manifest): + raise ValueError( + f"The {payload.arch} payload was built from a different version of " + f"{doc.manifest.name or 'this character'}. Wire Write .char to the same node " + "the payload node reads from." + ) + payload.apply(doc) _apply_mode(doc, str(node.params.get("apply") or "auto")) # A typed name wins over the file it was loaded from: that is what Save as means. path = library.save(doc, _target_name(node.params.get("filename")) or identity.file or None) @@ -397,19 +618,8 @@ def _apply_mode(doc: cf.CharDoc, mode: str) -> None: doc.manifest.apply[key] = mode -def _ref_images(doc: cf.CharDoc) -> list[Any]: - """The character's own references decoded, so a payload compiles from truth not the library.""" - import io - - from PIL import Image - - images = [] - for ref in doc.manifest.refs: - data = doc.members.get(str(ref.get("path") or "")) - if data: - with Image.open(io.BytesIO(data)) as handle: - images.append(handle.convert("RGB").copy()) - return images +#: The character's own references decoded, so a payload compiles from truth not the library. +_ref_images = encode.ref_images def _image_path(ref: Any) -> Any: @@ -568,7 +778,9 @@ def _materialise(doc: cf.CharDoc) -> list[Path]: folder = Path(tempfile.mkdtemp(prefix="char-dataset-")) written: list[Path] = [] - for index, ref in enumerate(doc.manifest.refs): + # Originals first, so a capped or truncated training set keeps the ones the user vouched for. + ordered = encode.originals(doc.manifest) + encode.harvested(doc.manifest) + for index, ref in enumerate(ordered): data = doc.members.get(str(ref.get("path") or "")) if data: out = folder / f"{index:04d}.png" diff --git a/core/src/inline_core/models/characterreqs.py b/core/src/inline_core/models/characterreqs.py index 43e9dbc..feb6a50 100644 --- a/core/src/inline_core/models/characterreqs.py +++ b/core/src/inline_core/models/characterreqs.py @@ -15,7 +15,9 @@ from .requirements import ModelComponent #: Every node that runs an encoder. The rest of the character family only moves bytes around. -ENCODER_NODES = ("character/encode", "character/edit") +ENCODER_NODES = ( + "character/encode", "character/edit", "character/verify-refs", "character/ingest-approved", +) def _component( diff --git a/core/src/inline_core/models/loaders.py b/core/src/inline_core/models/loaders.py index 98f99ea..c0f5370 100644 --- a/core/src/inline_core/models/loaders.py +++ b/core/src/inline_core/models/loaders.py @@ -181,12 +181,33 @@ class ArchSpec: ), ) +#: MiniMax H3's conditioner. The repo lays its encoder out under `FL2VA/text_encoder/`, so each +#: file is re-homed to the `text_encoder/` subfolder the staged dir loads from. +_MINIMAX_H3 = ArchSpec( + key="minimax-h3", + assets_repo="MiniMaxAI/MiniMax-H3", + asset_files=tuple( + AssetFile(f"FL2VA/text_encoder/{name}", local=f"text_encoder/{name}") + for name in ( + "config.json", + "chat_template.json", + "preprocessor_config.json", + "tokenizer.json", + "tokenizer_config.json", + "video_preprocessor_config.json", + "merges.txt", + "vocab.json", + ) + ), +) + SPECS: dict[str, ArchSpec] = { _ZIMAGE.key: _ZIMAGE, _KREA2.key: _KREA2, _FLUX2_KLEIN_4B.key: _FLUX2_KLEIN_4B, _FLUX2_KLEIN_9B.key: _FLUX2_KLEIN_9B, _FLUX2_DEV.key: _FLUX2_DEV, + _MINIMAX_H3.key: _MINIMAX_H3, } diff --git a/core/src/inline_core/models/minimaxh3/nvfp4.py b/core/src/inline_core/models/minimaxh3/nvfp4.py new file mode 100644 index 0000000..8982ff2 --- /dev/null +++ b/core/src/inline_core/models/minimaxh3/nvfp4.py @@ -0,0 +1,161 @@ +"""NVFP4 (ComfyUI's ``comfy_quant``) weights, dequantised per layer at inference. + +The file's own marker says ``{"format": "nvfp4", "full_precision_matrix_mult": true}``: it is meant +to be unpacked into an ordinary matmul, not fed to FP4 tensor cores, so it runs on any card rather +than needing Blackwell. Weights stay 4-bit in VRAM and each layer is unpacked inside its own +forward, which is what makes a 15.7 GB file stand in for a 63 GB folder. + +The layout is decoded in ``docs/nvfp4-format.md`` and verified tensor-by-tensor against the bf16 +release of the same encoder. +""" + +from __future__ import annotations + +from typing import Any + +import torch +from torch import nn + +#: FP4 E2M1: sign in bit 3, exponent in bits 2-1, mantissa in bit 0. +E2M1 = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0) +E2M1_MAX = 6.0 +F8_E4M3_MAX = 448.0 +#: One FP8 scale per this many weights, along the input dimension. +BLOCK = 16 + + +def e2m1_table(device: Any = None, dtype: Any = torch.float32) -> torch.Tensor: + """The 16 representable FP4 values, indexed by nibble.""" + return torch.tensor(E2M1 + tuple(-v for v in E2M1), device=device, dtype=dtype) + + +def unpack_fp4(packed: torch.Tensor, table: torch.Tensor) -> torch.Tensor: + """``[..., n/2]`` uint8 to ``[..., n]`` values. High nibble first, then low, interleaved. + + The other three orderings score ~0 against the reference, so this is not a coin flip. + """ + codes = packed.to(torch.int64) + pairs = torch.stack((table[codes >> 4], table[codes & 0xF]), dim=-1) + return pairs.reshape(*packed.shape[:-1], packed.shape[-1] * 2) + + +def from_blocked(stored: torch.Tensor, rows: int, cols: int) -> torch.Tensor: + """Undo the cuBLAS block-scaling swizzle that ``comfy/float.py``'s ``to_blocked`` applies. + + Two stages, not one, which is why searching single reshape+permute patterns never finds it: + skipping this scores 0.899 against the reference where undoing it scores 0.991. + """ + row_blocks = (rows + 127) // 128 + col_blocks = (cols + 3) // 4 + tiles = stored.reshape(row_blocks * col_blocks, 32, 4, 4).transpose(1, 2) + flat = tiles.reshape(row_blocks, col_blocks, 128, 4).permute(0, 2, 1, 3) + return flat.reshape(row_blocks * 128, col_blocks * 4)[:rows, :cols] + + +def to_blocked(scales: torch.Tensor) -> torch.Tensor: + """The forward swizzle, kept so a test can prove ``from_blocked`` inverts the real thing.""" + rows, cols = scales.shape + row_blocks = (rows + 127) // 128 + col_blocks = (cols + 3) // 4 + padded = scales + if (rows, cols) != (row_blocks * 128, col_blocks * 4): + padded = torch.zeros( + (row_blocks * 128, col_blocks * 4), device=scales.device, dtype=scales.dtype + ) + padded[:rows, :cols] = scales + tiles = padded.view(row_blocks, 128, col_blocks, 4).permute(0, 2, 1, 3) + return tiles.reshape(-1, 4, 32, 4).transpose(1, 2).reshape(row_blocks * 128, col_blocks * 4) + + +def dequantize( + packed: torch.Tensor, + block_scale: torch.Tensor, + global_scale: torch.Tensor | float, + *, + out_features: int, + in_features: int, + dtype: torch.dtype = torch.bfloat16, +) -> torch.Tensor: + """One NVFP4 weight back to ``dtype``: ``fp4 * block_scale * global_scale``.""" + table = e2m1_table(packed.device, torch.float32) + values = unpack_fp4(packed, table) + rows, blocks = values.shape[0], values.shape[1] // BLOCK + scales = from_blocked(block_scale.to(torch.float32), rows, blocks) + weight = (values.reshape(rows, blocks, BLOCK) * scales.unsqueeze(-1)).reshape(values.shape) + weight = weight * float(global_scale) + # Padded up to a multiple of 16 on the way in, so trim back to the layer's real shape. + return weight[:out_features, :in_features].to(dtype) + + +class NVFP4Linear(nn.Module): + """A ``nn.Linear`` whose weight lives packed and is unpacked inside the forward. + + Holding the unpacked weight would defeat the point: the packed tensors are the whole reason the + encoder fits. The transient bf16 copy is one layer's worth (~262 MB at the widest) and is freed + on the way out. + """ + + #: Annotated because `register_buffer` is untyped, so strict mode reads these as `Module`. + weight: torch.Tensor + weight_scale: torch.Tensor + weight_scale_2: torch.Tensor + pre_quant_scale: torch.Tensor | None + + def __init__( + self, in_features: int, out_features: int, bias: bool, dtype: torch.dtype + ) -> None: + super().__init__() + self.in_features = in_features + self.out_features = out_features + self.compute_dtype = dtype + packed_cols = ((in_features + 15) // 16 * 16) // 2 + padded_rows = (out_features + 15) // 16 * 16 + self.register_buffer( + "weight", torch.empty(padded_rows, packed_cols, dtype=torch.uint8), persistent=True + ) + # The swizzle pads to 128 rows by 4 blocks, so the scales are stored larger than the weight + # they describe; sizing this from the weight's own shape truncates every small layer. + scale_rows = (padded_rows + 127) // 128 * 128 + scale_cols = ((packed_cols * 2 // BLOCK) + 3) // 4 * 4 + self.register_buffer( + "weight_scale", + torch.empty(scale_rows, scale_cols, dtype=torch.float8_e4m3fn), + persistent=True, + ) + self.register_buffer("weight_scale_2", torch.empty((), dtype=torch.float32)) + # AWQ smoothing, on the layers that carry it. Where it is absent the factor is folded into + # the preceding norm in the same checkpoint, so there is nothing to undo. + self.register_buffer("pre_quant_scale", None, persistent=True) + self.bias = nn.Parameter(torch.empty(out_features, dtype=dtype)) if bias else None + + def forward(self, x: torch.Tensor) -> torch.Tensor: + smooth = self.pre_quant_scale + if smooth is not None: + x = x * smooth.to(x.dtype) + weight = dequantize( + self.weight, + self.weight_scale, + self.weight_scale_2, + out_features=self.out_features, + in_features=self.in_features, + dtype=x.dtype, + ) + return torch.nn.functional.linear(x, weight, self.bias) + + def extra_repr(self) -> str: + return f"in_features={self.in_features}, out_features={self.out_features}, nvfp4" + + +def quantize_reference(weight: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ComfyUI's quantiser, for tests only: it is the definition the loader has to invert.""" + global_scale = weight.abs().amax() / (F8_E4M3_MAX * E2M1_MAX) + blocks = weight.reshape(weight.shape[0], -1, BLOCK) + block_scale = torch.clamp( + blocks.abs().amax(dim=-1) / E2M1_MAX / global_scale, max=F8_E4M3_MAX + ).to(torch.float8_e4m3fn) + normalised = blocks / (global_scale * block_scale.to(torch.float32)).unsqueeze(-1) + table = e2m1_table(weight.device, torch.float32) + flat = normalised.reshape(weight.shape).nan_to_num() + codes = (flat.unsqueeze(-1) - table).abs().argmin(dim=-1).to(torch.uint8) + packed = (codes[..., 0::2] << 4) | codes[..., 1::2] + return packed, to_blocked(block_scale), global_scale diff --git a/core/src/inline_core/models/minimaxh3/pipeline.py b/core/src/inline_core/models/minimaxh3/pipeline.py index 4085fa6..e2dfe8e 100644 --- a/core/src/inline_core/models/minimaxh3/pipeline.py +++ b/core/src/inline_core/models/minimaxh3/pipeline.py @@ -35,6 +35,7 @@ ) from . import requirements as reqs from .load import load_transformer +from .vendor.packing import MINIMAX_H3_TEXT_ENCODER_LAYER logger = logging.getLogger("inline_core.minimaxh3") @@ -290,16 +291,22 @@ def _build( encoder_quant = _encoder_config(recipe) _placement_kwargs = _encoder_placement(placement) if encoder_quant is not None else {} _encoder_on_card = _placement_kwargs.get("device_map", {}).get("") not in (None, "cpu") - text_encoder = Qwen3VLForConditionalGeneration.from_pretrained( - str(encoder_dir), - dtype=dtype, - local_files_only=True, - **( - {"quantization_config": encoder_quant, **_placement_kwargs} - if encoder_quant is not None - else {} - ), - ) + if _is_nvfp4(encoder_dir): + # Already 4-bit on disk, so the NF4 rung would be quantising a quantised file - the same + # rule that turns quantization off for any prequantized source. + _encoder_on_card = False + text_encoder = _load_nvfp4_encoder(encoder_dir, dtype) + else: + text_encoder = Qwen3VLForConditionalGeneration.from_pretrained( + str(_encoder_source(encoder_dir)), + dtype=dtype, + local_files_only=True, + **( + {"quantization_config": encoder_quant, **_placement_kwargs} + if encoder_quant is not None + else {} + ), + ) # The conditioner's shards stage through ~21 GB of shared memory that is not reclaimed on its # own, and the denoiser needs that space. Forcing a collection here is what actually frees it. @@ -544,6 +551,213 @@ def _encoder_placement(placement: Any) -> dict[str, Any]: _ENCODER_RESIDENT_GB = 20.5 +#: The `models/loaders.py` spec key for this family's one-time config assets. +ASSETS_ARCH = "minimax-h3" + +#: Encoder builds we cannot read, with the reason, rather than failing deep inside transformers. +_ENCODER_REFUSED = { + "int8_convrot": "an int8 rotation repack this loader cannot undo", + "nvfp4_awq": "", +} + + +def _is_nvfp4(path: Path) -> bool: + """Whether a single-file encoder carries ComfyUI's nvfp4 marker.""" + if path.is_dir() or path.suffix.lower() not in (".safetensors", ".sft"): + return False + from ..checkpoint import prequantized_kind + + return prequantized_kind(path) is not None and _nvfp4_marked(path) + + +def _nvfp4_marked(path: Path) -> bool: + from safetensors import safe_open + + with safe_open(str(path), framework="pt") as handle: + keys: list[str] = list(handle.keys()) + return any(key.endswith("comfy_quant") for key in keys) + + +def _encoder_source(path: Path) -> Path: + """A folder as-is, or a single file staged into a tiny dir transformers can load. + + Streaming from a directory is what keeps peak host RAM at about one tensor; handing + ``from_pretrained`` a materialised state dict would pull the whole encoder into RAM first. + """ + if path.is_dir(): + return path + for marker, reason in _ENCODER_REFUSED.items(): + if reason and marker in path.name: + raise ComponentError(f"{path.name} is {reason}. Use the bf16 or nvfp4 build instead.") + from ..loaders import _staged_encoder_dir + + return _staged_encoder_dir(ASSETS_ARCH, str(path)) + + +def _nvfp4_key(key: str) -> str: + """The file's flat layout to the module tree transformers builds. + + ComfyUI writes the pre-Qwen3VL-split names (`model.layers.*`, `visual.*`); transformers nests + both under `model.` with the language stack under `language_model`. + """ + if key.startswith("visual."): + return f"model.{key}" + if key.startswith("model."): + return f"model.language_model.{key[len('model.'):]}" + return key + + +def _nvfp4_layers(keys: set[str]) -> int: + import re + + found = {int(m.group(1)) for k in keys if (m := re.match(r"model\.layers\.(\d+)\.", k))} + return max(found) + 1 if found else 0 + + +def _load_nvfp4_encoder(path: Path, dtype: Any) -> Any: + """Qwen3-VL from an NVFP4 file, with every quantised Linear left packed. + + Built on ``meta`` and populated by hand because transformers has no reader for this format; + materialising it first would want the 51 GB the packed file exists to avoid. The build is + truncated to the layers H3 actually reads (``hidden_states[50]``) and carries no language-model + head, which the vendored encoder never calls - so the config is cut to match the file rather + than the file being padded to match the config. + """ + from accelerate import init_empty_weights + from safetensors import safe_open + from transformers import AutoConfig, Qwen3VLForConditionalGeneration + + from ..loaders import ensure_assets + + with safe_open(str(path), framework="pt") as handle: + keys: set[str] = set(handle.keys()) + layers = _nvfp4_layers(keys) + if layers < MINIMAX_H3_TEXT_ENCODER_LAYER: + raise ComponentError( + f"{path.name} carries {layers} encoder layers, and MiniMax H3 reads hidden state " + f"{MINIMAX_H3_TEXT_ENCODER_LAYER}. This is not the H3 conditioner." + ) + + config = AutoConfig.from_pretrained(str(ensure_assets(ASSETS_ARCH) / "text_encoder")) + text_config = getattr(config, "text_config", config) + text_config.num_hidden_layers = layers + with init_empty_weights(): + model = Qwen3VLForConditionalGeneration._from_config(config, dtype=dtype) + # The trailing norm is absent from this build on purpose. At full depth H3 reads the *un-normed* + # state after layer 50, because the norm only lands at the final index; cut to 50 layers that + # index becomes the normed one, so keeping a real norm here would change the conditioning. + if "model.norm.weight" not in keys: + model.model.language_model.norm = torch.nn.Identity() + + with safe_open(str(path), framework="pt") as handle: + formats = _packed_formats(handle, keys) + linears = {n for n, fmt in formats.items() if fmt == "nvfp4"} + plain = {n for n, fmt in formats.items() if fmt == "int8_tensorwise"} + unknown = set(formats) - linears - plain + if unknown: + raise ComponentError( + f"{path.name} quantises {len(unknown)} layers as " + f"{formats[sorted(unknown)[0]]!r}, which this loader cannot read." + ) + _swap_packed_linears(model, linears, dtype) + # Tensor-wise int8 is the embedding, read once per prompt: keeping it packed would trade + # about a gigabyte for dequantising the whole vocabulary on every forward. + for name in plain: + _unpack_int8(model, handle, name, dtype) + missing = _load_into(model, handle, keys, skip={f"{n}.weight" for n in plain}) + available_keys = {_nvfp4_key(key) for key in keys} + if missing: + raise ComponentError( + f"{path.name} is missing {len(missing)} tensors this encoder needs, " + f"starting with {sorted(missing)[0]}." + ) + # Dropped rather than left on meta: this build ships no head, and any `.to(device)` over a meta + # parameter raises rather than being skipped. The vendored encoder calls `.model` directly. + if "lm_head.weight" not in available_keys: + model.lm_head = torch.nn.Identity() + logger.info( + "MiniMax H3 conditioner: %d layers, %d nvfp4 linears, %d int8 tables, from %s", + layers, len(linears), len(plain), path.name, + ) + return model + + +def _swap_packed_linears(model: Any, packed: set[str], dtype: Any) -> None: + """Replace each quantised ``nn.Linear`` with its packed counterpart, in place.""" + from .nvfp4 import NVFP4Linear + + for name in packed: + parent = model.get_submodule(name.rsplit(".", 1)[0]) + attr = name.rsplit(".", 1)[-1] + old = getattr(parent, attr) + setattr( + parent, + attr, + NVFP4Linear(old.in_features, old.out_features, old.bias is not None, dtype), + ) + + +def _packed_formats(handle: Any, keys: set[str]) -> dict[str, str]: + """Each quantised module mapped to the format its own ``comfy_quant`` marker names. + + Read per layer rather than assumed for the file: this one mixes nvfp4 linears with a + tensor-wise int8 embedding, and guessing from the module type would have silently mis-read it. + """ + import json + + out: dict[str, str] = {} + for key in keys: + if not key.endswith(".comfy_quant"): + continue + raw = bytes(handle.get_tensor(key).tolist()).decode("utf-8", errors="replace") + try: + marker = json.loads(raw) + except ValueError: + marker = {} + out[_nvfp4_key(key[: -len(".comfy_quant")])] = str(marker.get("format") or raw) + return out + + +def _unpack_int8(model: Any, handle: Any, name: str, dtype: Any) -> None: + """A tensor-wise int8 weight back to ``dtype``, in place: ``int8 * per-row scale``.""" + import torch + + module = model.get_submodule(name) + source = name.replace("model.language_model.", "model.", 1) + weight = handle.get_tensor(f"{source}.weight").to(torch.float32) + weight = weight * handle.get_tensor(f"{source}.weight_scale").to(torch.float32) + module.weight = torch.nn.Parameter(weight.to(dtype), requires_grad=False) + + +def _load_into(model: Any, handle: Any, keys: set[str], *, skip: set[str]) -> set[str]: + """Copy every tensor in, materialising the meta parameters. Returns what the file lacked.""" + import torch + + available = {_nvfp4_key(key): key for key in keys} + wanted: set[str] = {str(name) for name in model.state_dict()} + missing: set[str] = set() + for name in sorted(wanted - skip): + source = available.get(name) + if source is None: + # The head is the one correct absence: H3 reads a hidden state and never runs it. + if not name.startswith("lm_head."): + missing.add(name) + continue + parent = model.get_submodule(name.rsplit(".", 1)[0]) if "." in name else model + attr = name.rsplit(".", 1)[-1] + value = handle.get_tensor(source) + if isinstance(getattr(parent, attr, None), torch.nn.Parameter): + setattr(parent, attr, torch.nn.Parameter(value, requires_grad=False)) + else: + parent.register_buffer(attr, value, persistent=True) + # AWQ smoothing is not in `state_dict` until it exists, so attach it from the file directly. + for mapped, source in available.items(): + if source.endswith(".pre_quant_scale"): + module = model.get_submodule(mapped[: -len(".pre_quant_scale")]) + module.pre_quant_scale = handle.get_tensor(source) + return missing + + def _encoder_config(recipe: Any) -> Any: """The conditioner's quantisation, decided independently of the denoiser's. diff --git a/core/src/inline_core/models/minimaxh3/requirements.py b/core/src/inline_core/models/minimaxh3/requirements.py index e99a8d4..6d5323f 100644 --- a/core/src/inline_core/models/minimaxh3/requirements.py +++ b/core/src/inline_core/models/minimaxh3/requirements.py @@ -32,7 +32,12 @@ #: pruned build does not ship, and it saves nothing in VRAM because the base is quantised anyway. FL2VA_FP8_FILE = "minimax_h3_fl2va_pruned_fp8_scaled.safetensors" REF2VA_FILE = "minimax_h3_ref2va_bf16.safetensors" +REF2VA_FP8_FILE = "minimax_h3_ref2va_pruned_fp8_scaled.safetensors" TEXT_ENCODER_DIR = "FL2VA/text_encoder" +#: Single-file conditioners. nvfp4 is 4-bit on disk and the default: the folder is quantised to NF4 +#: on load anyway, so this lands at the same resident size for a quarter of the download. +ENCODER_NVFP4_FILE = "qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors" +ENCODER_BF16_FILE = "qwen3vl_32b_minimax_h3_bf16.safetensors" VIDEO_VAE_FILE = "minimax_h3_video_vae_fp16.safetensors" AUDIO_VAE_FILE = "minimax_h3_audio_vae_fp32.safetensors" @@ -245,28 +250,79 @@ def record_provenance(partition: str, filename: str) -> None: path.write_text(json.dumps(current, indent=2)) -def components(partition: str = "fl2va") -> list[ModelComponent]: +def components(partition: str = "fl2va", *, fp8_substitutes: bool = True) -> list[ModelComponent]: """What this node needs, with live presence. Sizes in the labels because the totals are large enough that a user deserves to know before pressing Download.""" ref2va_required = partition == "ref2va" + + def pair(needed: bool) -> tuple[bool, bool]: + """``(bf16 optional, fp8 optional)`` for a partition this node does or does not use. + + The pruned fp8 build is what generation asks for: same render, 21 GB against 66.3, and it + fits cards that cannot hold the bf16 at all. Training inverts it - fp8 renders but does not + fine-tune - which is what ``fp8_substitutes=False`` selects. + """ + if not needed: + return True, True + return fp8_substitutes, not fp8_substitutes + + fl2va_bf16, fl2va_fp8 = pair(not ref2va_required) + ref2va_bf16, ref2va_fp8 = pair(ref2va_required) entries: list[ModelComponent] = [ - _file("h3-fl2va", "FL2VA transformer (66.3 GB)", "diffusion_models", FL2VA_FILE, - COMFY_REPO, f"diffusion_models/{FL2VA_FILE}", optional=ref2va_required), - _folder("h3-text-encoder", "Text encoder, Qwen3-VL-32B (66.7 GB)", "text_encoders", - "MiniMax-H3-text-encoder", MINIMAX_REPO, TEXT_ENCODER_DIR), + _file("h3-fl2va", "FL2VA transformer, bf16 (66.3 GB, needed to train)", + "diffusion_models", FL2VA_FILE, + COMFY_REPO, f"diffusion_models/{FL2VA_FILE}", optional=fl2va_bf16), + _file("h3-text-encoder-nvfp4", "Text encoder, Qwen3-VL-32B nvfp4 (15.7 GB)", + "text_encoders", ENCODER_NVFP4_FILE, + COMFY_REPO, f"text_encoders/{ENCODER_NVFP4_FILE}"), + _folder("h3-text-encoder", "Text encoder, Qwen3-VL-32B folder (66.7 GB)", "text_encoders", + "MiniMax-H3-text-encoder", MINIMAX_REPO, TEXT_ENCODER_DIR, optional=True), + _file("h3-text-encoder-bf16", "Text encoder, Qwen3-VL-32B bf16 single file (51.5 GB)", + "text_encoders", ENCODER_BF16_FILE, + COMFY_REPO, f"text_encoders/{ENCODER_BF16_FILE}", optional=True), _file("h3-video-vae", "Video VAE (5.2 GB)", "vae", VIDEO_VAE_FILE, COMFY_REPO, f"vae/{VIDEO_VAE_FILE}"), _file("h3-audio-vae", "Audio VAE (0.6 GB)", "vae", AUDIO_VAE_FILE, COMFY_REPO, f"vae/{AUDIO_VAE_FILE}"), _folder("h3-processor", "Tokenizer and processor (12 MB)", "text_encoders", "MiniMax-H3-processor", MINIMAX_REPO, "FL2VA/processor"), - _file("h3-ref2va", "Ref2VA transformer (66.3 GB)", "diffusion_models", REF2VA_FILE, - COMFY_REPO, f"diffusion_models/{REF2VA_FILE}", optional=not ref2va_required), + _file("h3-ref2va", "Ref2VA transformer, bf16 (66.3 GB, needed to train)", + "diffusion_models", REF2VA_FILE, + COMFY_REPO, f"diffusion_models/{REF2VA_FILE}", optional=ref2va_bf16), _file("h3-fl2va-fp8", "FL2VA transformer, fp8 (21.0 GB, generation only)", "diffusion_models", FL2VA_FP8_FILE, - COMFY_REPO, f"diffusion_models/{FL2VA_FP8_FILE}", optional=True), + COMFY_REPO, f"diffusion_models/{FL2VA_FP8_FILE}", optional=fl2va_fp8), + _file("h3-ref2va-fp8", "Ref2VA transformer, fp8 (21.0 GB, generation only)", + "diffusion_models", REF2VA_FP8_FILE, + COMFY_REPO, f"diffusion_models/{REF2VA_FP8_FILE}", optional=ref2va_fp8), ] - return entries + # A partition needs *a* transformer, not a particular one. Without this a box holding only the + # fp8 build - the one that fits most cards - was told its 66.3 GB bf16 twin was missing. Off for + # training, where a pruned fp8 build is not a substitute: it generates, it does not fine-tune. + if not fp8_substitutes: + return entries + pairs = ( + ("h3-fl2va", "h3-fl2va-fp8"), + ("h3-ref2va", "h3-ref2va-fp8"), + ("h3-text-encoder-nvfp4", "h3-text-encoder", "h3-text-encoder-bf16"), + ) + return _satisfy_alternatives(entries, pairs) + + +def _satisfy_alternatives( + entries: list[ModelComponent], pairs: tuple[tuple[str, ...], ...] +) -> list[ModelComponent]: + """Mark both members of an either-or pair optional once either one is on disk.""" + from dataclasses import replace + + by_id = {entry.id: entry for entry in entries} + relaxed = { + component_id + for pair in pairs + if any(by_id.get(other) and by_id[other].present for other in pair) + for component_id in pair + } + return [replace(e, optional=True) if e.id in relaxed and not e.optional else e for e in entries] def _file( @@ -281,12 +337,13 @@ def _file( def _folder( - component_id: str, label: str, category: str, folder: str, repo: str, repo_folder: str + component_id: str, label: str, category: str, folder: str, repo: str, repo_folder: str, + *, optional: bool = False, ) -> ModelComponent: return ModelComponent( id=component_id, label=label, category=category, filename=folder, present=(models_dir() / category / folder).is_dir(), - repo=repo, repo_file="", repo_folder=repo_folder, + repo=repo, repo_file="", repo_folder=repo_folder, optional=optional, ) @@ -323,12 +380,43 @@ def resident_bytes(path: Path) -> int: return total +def resolve_encoder(pick: str | None = None) -> Path | None: + """The conditioner this node would load: an explicit pick, else the smallest build present.""" + if pick: + return resolve("text_encoders", pick) + for name in (ENCODER_NVFP4_FILE, "MiniMax-H3-text-encoder", ENCODER_BF16_FILE): + found = resolve("text_encoders", name) + if found is not None: + return found + return None + + +def encoder_resident_bytes(path: Path | None) -> int: + """What the conditioner occupies once placed. + + An nvfp4 build is the one case where resident is about what it weighs: its linears are never + unpacked, so sizing it like a bf16 file that will be quantised on load doubles it and refuses + machines it runs on. Everything else is sized from its bytes, as before. + """ + if path is None: + return 0 + if path.is_dir(): + return sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) + try: + size = path.stat().st_size + except OSError: + return 0 + # The packed file plus the one table it does unpack; measured at 16.46 GB against 15.69 on disk. + return int(size * 1.05) if "nvfp4" in path.name else size + + def footprint_bytes( partition: str = "fl2va", *, factorised: bool = True, transformer: Path | None = None, video_vae: Path | None = None, + text_encoder: Path | None = None, ) -> dict[str, int]: """Sizes for the fit estimate: what will actually be placed, not what is on disk. @@ -343,10 +431,7 @@ def size(path: Path | None) -> int: except OSError: return 0 - encoder = models_dir() / "text_encoders" / "MiniMax-H3-text-encoder" - encoder_bytes = sum(f.stat().st_size for f in encoder.rglob("*") if f.is_file()) if ( - encoder.is_dir() - ) else 0 + encoder_bytes = encoder_resident_bytes(text_encoder or resolve_encoder()) chosen = transformer if transformer is not None else resolve_transformer(partition) diffusion = size(chosen) if chosen is not None and (candidate := inspect_file(chosen)).is_h3: diff --git a/core/src/inline_core/models/minimaxh3/runner.py b/core/src/inline_core/models/minimaxh3/runner.py index e48f41c..b2314ea 100644 --- a/core/src/inline_core/models/minimaxh3/runner.py +++ b/core/src/inline_core/models/minimaxh3/runner.py @@ -102,13 +102,6 @@ def _params(variant: Variant) -> tuple[ParamField, ...]: ParamField("num_inference_steps", "Steps", Widget.NUMBER, 50, min=1, max=200, step=1), ParamField("seed", "Seed (-1 = random)", Widget.SEED, -1), ] - if variant.references: - fields.append( - ParamField( - "ref_image_size", "Reference detail", Widget.SELECT, "match", - options_from=None, advanced=True, - ) - ) fields.append( ParamField( "model", "Diffusion model", Widget.SELECT, "", @@ -218,10 +211,11 @@ def build_request( loras: tuple[Any, ...] = () references: tuple[Any, ...] = () if variant.references: - wired = list(inputs.get("references") or []) + wired = [v for v in (inputs.get("references") or []) if v is not None] if character is not None and character.refs: # Fed through the collector rather than appended after it, so the character's images are # numbered and limit-checked as images - appending would land them behind the videos. + # Already trimmed to fit by `_apply_character`, which owns the numbering. inputs = {**inputs, "references": [*wired, *character.refs]} references = collect_references(inputs, limits=REFERENCE_LIMITS) if not references: @@ -275,7 +269,11 @@ def _apply_character(inputs: dict[str, list[Any]], variant: Variant) -> _Charact from ...characters import apply as characters from ...graph.loader_runners import LoraRef - applied = characters.char_apply(chosen, ARCH) + # The reference partition cannot run on an adapter alone, so it asks for references outright + # rather than taking the adapter a character prefers by default. + applied = characters.char_apply( + chosen, ARCH, prefer="reference" if variant.references else None + ) if applied is None: return None if not variant.references: @@ -291,15 +289,28 @@ def _apply_character(inputs: dict[str, list[Any]], variant: Variant) -> _Charact prefix=applied.prompt_prefix(1), lora=LoraRef(file=str(applied.lora), strength=applied.lora_strength), ) - if not applied.refs and applied.lora is None: - return None + if not applied.refs: + raise ComponentError( + f"{chosen} has no {ARCH} references, so {variant.title} has nothing to condition on. " + "Wire it through Compile References with Model set to minimax-h3 and write it again, " + "or wire images into this node's References input." + ) how = "adapter" if applied.lora is not None else f"{len(applied.refs)} reference(s)" logger.info("Applying character %s by %s", applied.name, how) # H3 resolves ``, not FLUX.2's ordinal prose, and the character's images land after # whatever the user already wired. wired = len([v for v in (inputs.get("references") or []) if v is not None]) + # Trimmed here rather than by the caller, so the prefix can never name a position that was + # dropped: a character is a library artefact and H3's 9 images is not every model's limit. + keep = list(applied.refs)[: max(0, REFERENCE_LIMITS.max_images - wired)] + if len(keep) < len(applied.refs): + logger.info( + "%s: using %d of %s's %d references, the most it takes beside %d wired", + variant.title, len(keep), chosen, len(applied.refs), wired, + ) + applied.refs = keep return _Character( - refs=list(applied.refs), + refs=keep, prefix=applied.prompt_prefix(wired + 1, style="token"), lora=( LoraRef(file=str(applied.lora), strength=applied.lora_strength) @@ -408,6 +419,7 @@ def run(self, node: Node, inputs: dict[str, list[Any]], ctx: ExecutionContext) - ) call = call_kwargs(request, self._variant, inputs) + call["generator"] = torch.Generator(device="cpu").manual_seed(request.seed) def on_step(done: int, total: int) -> None: @@ -490,13 +502,57 @@ def _result( return NodeResult(outputs=outputs, takes=takes) +def _reference_tokens(request: Request) -> tuple[int, int]: + """Wired image references and what they cost the vision tower, measured from the pixels. + + Read off the files rather than a setting, because the size that matters was decided when the + character was compiled and nothing on this node records it. + """ + images = [r for r in request.references if getattr(r, "kind", None) == ReferenceKind.IMAGE] + tokens = 0 + for ref in images: + try: + from PIL import Image + + with Image.open(getattr(ref.value, "path", ref.value)) as handle: + width, height = handle.size + except Exception: # noqa: BLE001 - an error path must not raise a second error + continue + tokens += (width // 32) * (height // 32) + return len(images), tokens + + +#: Below this, another process on the card is noise; above it, it is the whole story. +_FOREIGN_VRAM_FLOOR = 2 * 1024**3 + + def _oom(request: Request, *, host: bool = False) -> str: where = "System RAM" if host else "VRAM" - return ( + # Asked first, because when it is true nothing on this node is the cause and every other hint + # below sends the user to change a setting that was never the problem. + foreign = 0 if host else rt.foreign_vram_bytes() + if foreign >= _FOREIGN_VRAM_FLOOR: + return ( + f"{where} ran out, but {foreign / 1024**3:.1f} GB of this card is held by another " + "process - a training run, another render, or another app. Wait for it to finish or " + "stop it, then run this again. Nothing on this node will free that memory." + ) + canvas = ( f"{where} ran out at {request.width}x{request.height} for {request.seconds:.1f}s. " "Canvas is the biggest lever: 960x544 needs far less than 1344x768 and renders about " "2.3x faster per step. A shorter duration helps too." ) + images, tokens = _reference_tokens(request) + if not images: + return canvas + cost = f", which is {tokens:,} vision tokens" if tokens else "" + # Named alone because references are encoded before a frame exists: the canvas cannot move this + # step at all, and a hint that leads with it sends the user to resize for nothing. + return ( + f"{where} ran out encoding {images} reference(s){cost}. The canvas does not affect this " + "step. Lower Resized Reference Resolution on the Compile References node and write the " + "character again - halving it quarters the tokens - or wire fewer references." + ) __all__ = [ diff --git a/core/src/inline_core/models/pipeline_runtime.py b/core/src/inline_core/models/pipeline_runtime.py index 556caa8..5a55053 100644 --- a/core/src/inline_core/models/pipeline_runtime.py +++ b/core/src/inline_core/models/pipeline_runtime.py @@ -489,6 +489,23 @@ def free_vram() -> None: pass +def foreign_vram_bytes(device: Any = None) -> int: + """VRAM held on this card by anyone but us - another render, a training run, another app. + + An OOM message that only knows its own allocation blames whatever the user last changed. On a + box that trains and generates at once the honest answer is usually that the card is already + two-thirds spoken for, and no setting on this node will fix that. + """ + try: + if not torch.cuda.is_available(): + return 0 + target = torch.device(str(device)) if device else None + free, total = torch.cuda.mem_get_info(target) + return max(0, int(total) - int(free) - int(torch.cuda.memory_reserved(target))) + except Exception: # noqa: BLE001 + return 0 + + def free_vram_bytes(device: Any = None) -> int: """What the driver says is unallocated right now, which is the only honest number once another model is already placed. 0 when there is no CUDA device to ask.""" diff --git a/core/src/inline_core/models/trainingreqs.py b/core/src/inline_core/models/trainingreqs.py index 1cfadfb..47de80c 100644 --- a/core/src/inline_core/models/trainingreqs.py +++ b/core/src/inline_core/models/trainingreqs.py @@ -55,7 +55,8 @@ def base_components(arch: str, base_mode: str) -> list[ModelComponent]: if arch == "minimax-h3": from .minimaxh3 import requirements as reqs - return _required(reqs.components()) + # The fp8 builds generate but do not fine-tune, so they cannot stand in here. + return _required(reqs.components(fp8_substitutes=False)) return [] diff --git a/core/src/inline_core/studio/characters.py b/core/src/inline_core/studio/characters.py index 2c05f0c..0fd20fd 100644 --- a/core/src/inline_core/studio/characters.py +++ b/core/src/inline_core/studio/characters.py @@ -180,6 +180,8 @@ def score_take(self, image_path: Path | str, chosen: str) -> dict[str, Any] | No if path is None: return None doc = cf.read(path) + # Before anything reads a version: a node's encoder pick must not decide this. + scoring.use_encoders_from(doc.manifest.scoring) # Refs are truth and scoring is cache, so a character written before the current # encoders is rebuilt here rather than scoring against a centroid nothing can compare. if self._scoring_stale(doc.manifest): @@ -229,36 +231,24 @@ def _scoring_stale(self, manifest: cf.Manifest) -> bool: manifest, encoder_id, version ): return True - return bool(manifest.refs) and not manifest.scoring.get("refFramings") + # A reference was added or dropped since, so every stored per-reference list is misaligned. + recorded = manifest.scoring.get("refCount") + if recorded is not None and int(recorded) != len(manifest.refs): + return True + # Key presence, not truthiness: a character whose references carry no detectable face has + # an honestly empty list, and reading that as stale rescored it on every take forever. + return bool(manifest.refs) and "refFramings" not in manifest.scoring def _rescore(self, path: Path, previous: cf.CharDoc) -> cf.CharDoc: - """Re-encode from its own refs, keeping char_id and filename so nothing unpicks.""" - doc = encode.char_encode( - self._ref_files(previous), - name=previous.manifest.name, - description=self._description(previous), - char_id=previous.manifest.char_id, - created_at=previous.manifest.created_at, - on_progress=self._progress(previous.manifest.name), - ) - cf.write(path, doc) - self._changed() - return doc - - def _ref_files(self, doc: cf.CharDoc) -> list[Path]: - """The character's own refs written out, so a rebuild reads truth not the user's library.""" - import tempfile + """Recompute scoring from the character's own refs, in place. - root = Path(tempfile.mkdtemp(prefix="inline-char-refs-")) - out: list[Path] = [] - for index, ref in enumerate(doc.manifest.refs): - data = doc.members.get(str(ref.get("path"))) - if data is None: - continue - target = root / f"{index:03d}.png" - target.write_bytes(data) - out.append(target) - return out + Never through `char_encode`: that builds a fresh manifest, so the write below would put the + loss of the trained adapter and every non-flux payload on disk. + """ + encode.rescore(previous, self._progress(previous.manifest.name)) + cf.write(path, previous) + self._changed() + return previous def _require(self, file: str) -> Path: path = library.resolve(file) @@ -280,7 +270,9 @@ def _summary(self, path: Path, doc: cf.CharDoc) -> dict[str, Any]: "file": path.name, "charId": manifest.char_id, "name": manifest.name or path.stem, - "refs": len(manifest.refs), + "refs": len(encode.originals(manifest)), + "harvested": len(encode.harvested(manifest)), + "flagged": list(manifest.scoring.get("flaggedRefs") or []), "createdAt": manifest.created_at, "modifiedAt": manifest.modified_at, "description": self._description(doc), diff --git a/core/src/inline_core/studio/models.py b/core/src/inline_core/studio/models.py index bc01de2..4b575a5 100644 --- a/core/src/inline_core/studio/models.py +++ b/core/src/inline_core/studio/models.py @@ -41,6 +41,22 @@ def download_target(self, _component: Any) -> Path: return self._target +def _with_xet(call: Callable[[], str]) -> str: + """Run ``call`` with XET enabled, whatever the process default is. + + ``is_xet_available`` reads the constant on every call rather than at import, so this is a real + switch and not a no-op. Restored afterwards so the next download keeps its progress bar. + """ + from huggingface_hub import constants + + previous = constants.HF_HUB_DISABLE_XET + constants.HF_HUB_DISABLE_XET = False + try: + return call() + finally: + constants.HF_HUB_DISABLE_XET = previous + + class DownloadCancelled(Exception): """Raised out of the progress callback, the only per-chunk seam a download has.""" @@ -333,8 +349,10 @@ def _download_component( into place under its basename, so a half-finished download never looks installed. Progress comes from huggingface_hub's own download counter via ``tqdm_class`` - real - per-chunk motion, and it still resumes a partial file. (XET is disabled at process start so - the plain HTTP path is used; XET reports nothing to tqdm.)""" + per-chunk motion, and it still resumes a partial file. XET is disabled at process start for + exactly that reason (it reports nothing to tqdm), but plain HTTP refuses any file over 50GB, + which is every H3 transformer - so a refusal turns XET back on and retries rather than + leaving the only route to those models closed.""" from huggingface_hub import hf_hub_download, snapshot_download from huggingface_hub.utils import HfHubHTTPError @@ -377,8 +395,18 @@ def _fetch(token: bool | None) -> str: tqdm_class=tqdm_class, ) + def _fetch_or_xet(token: bool | None) -> str: + """Plain HTTP first for its progress bar, XET only where HTTP will not go at all.""" + try: + return _fetch(token) + except ValueError as error: + if "too large" not in str(error): + raise + on_progress(0.0, f"Downloading {comp.label} over Xet…") + return _with_xet(lambda: _fetch(token)) + try: - path = _fetch(None) # ambient token, e.g. for a gated repo the user has access to + path = _fetch_or_xet(None) # ambient token, e.g. a gated repo the user has access to except HfHubHTTPError as error: # A stale/invalid cached HF token 401s even on a public repo (HF masks it as "not # found"). Retry anonymously so a bad token never blocks a public model download. @@ -386,7 +414,7 @@ def _fetch(token: bool | None) -> str: raise shutil.rmtree(staging, ignore_errors=True) try: - path = _fetch(False) + path = _fetch_or_xet(False) except HfHubHTTPError as anonymous: # Hugging Face reports a gated or missing repo identically, as 404 "Repository Not # Found", which reads as a broken link rather than a licence the user must accept. diff --git a/core/tests/test_character_nodes.py b/core/tests/test_character_nodes.py index d93637b..e1197f5 100644 --- a/core/tests/test_character_nodes.py +++ b/core/tests/test_character_nodes.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy from pathlib import Path import pytest @@ -53,6 +54,7 @@ def _ctx() -> object: def _image(path: Path) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) Image.new("RGB", (768, 1024), (180, 150, 140)).save(path) return path @@ -577,3 +579,306 @@ def test_save_as_keeps_the_character_inside_the_library(tmp_path: Path, encoders assert _target_name("../../escape.char") == "escape.char" assert _target_name("") is None + + +# --- verify-refs --------------------------------------------------------------------------------- + + +def _fake_faces(monkeypatch: pytest.MonkeyPatch, vectors: list[list[float]]) -> None: + """Deterministic identity, so the node's own logic is what is under test and not a detector. + + Keyed on the reference's own pixels, not call order: freezing re-decodes the members it froze + over, so an order-keyed stub would hand those a different vector than the pass that flagged. + """ + from inline_core.characters import scoring, weights + + monkeypatch.setattr(weights, "present", lambda: True) + by_colour = {_REF_COLOURS[i]: v for i, v in enumerate(vectors)} + + def face(image: object) -> list[float] | None: + return by_colour.get(image.getpixel((0, 0))) or None # type: ignore[attr-defined] + + monkeypatch.setattr(scoring, "embed_face", face) + monkeypatch.setattr(scoring, "embed_subject", lambda _image: [1.0, 0.0, 0.0]) + + +#: One flat colour per reference slot, so a stub can identify a reference by its own pixels. +_REF_COLOURS = [(index * 30 + 10, 90, 140) for index in range(12)] + + +def _character(tmp_path: Path, count: int, name: str = "Ada") -> Identity: + from inline_core.characters import charfile as cf + from inline_core.characters import encode + + manifest = cf.Manifest(char_id="c", name=name, created_at=0, modified_at=0) + members: dict[str, bytes] = {} + for index in range(count): + image = Image.new("RGB", (320, 320), _REF_COLOURS[index]) + member = cf.member_name("refs", index, ".png") + data = encode._png_bytes(image) + members[member] = data + manifest.refs.append( + {"path": member, "sha256": cf.sha256_bytes(data), "width": 320, "height": 320, + "origin": cf.ORIGIN_ORIGINAL} + ) + return Identity(doc=cf.CharDoc(manifest=manifest, members=members)) + + +def _verify(identity: Identity, **params: object): + from inline_core.graph.schema import Node + from inline_core.models.character.runner import VerifyReferencesRunner + + merged: dict[str, object] = {"on_outlier": "flag", "floor": 25.0} + merged.update(params) + return VerifyReferencesRunner().run( + Node(id="v", type="character/verify-refs", params=merged), {"character": [identity]}, _ctx() + ).outputs["character"] + + +_SAME = [[1.0, 0.0, 0.0], [0.99, 0.1, 0.0], [0.98, 0.0, 0.1], [0.99, 0.05, 0.05]] +_OTHER = [0.0, 1.0, 0.0] + + +def test_verify_flags_a_reference_of_someone_else( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Best-match scoring means one wrong reference is a backdoor into every take.""" + _fake_faces(monkeypatch, [*_SAME[:3], _OTHER]) + out = _verify(_character(tmp_path, 4)) + + verdict = out.doc.manifest.scoring["verification"] + assert verdict["mode"] == "bootstrap" + assert verdict["flagged"] == [3] + assert len(out.doc.manifest.refs) == 4, "flag is the default and must remove nothing" + + +def test_verify_freezes_the_originals_once_it_has_checked_them( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from inline_core.characters import encode + + _fake_faces(monkeypatch, [*_SAME[:3], _OTHER]) + first = _verify(_character(tmp_path, 4)) + assert encode.originals_frozen(first.doc.manifest) + frozen = dict(first.doc.manifest.scoring["originals"]) + + _fake_faces(monkeypatch, [*_SAME[:3], _OTHER]) + again = _verify(Identity(doc=first.doc)) + assert again.doc.manifest.scoring["verification"]["mode"] == "existing" + assert again.doc.manifest.scoring["originals"] == frozen, "the identity target moved" + + +def test_verify_quarantines_rather_than_deletes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Refs are truth and the file one came from may be long gone, so removal stays reversible.""" + _fake_faces(monkeypatch, [*_SAME[:3], _OTHER]) + out = _verify(_character(tmp_path, 4), on_outlier="quarantine") + + assert len(out.doc.manifest.refs) == 3 + kept = [m for m in out.doc.members if m.startswith("quarantined/")] + assert len(kept) == 1, "the removed reference's bytes were not kept" + + +def test_verify_never_removes_a_reference_with_no_face( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Those are the wide and full-body shots the hints ask for, and the only ones that let the + subject term speak to a wide take at all.""" + _fake_faces(monkeypatch, [*_SAME[:3], []]) + out = _verify(_character(tmp_path, 4), on_outlier="quarantine") + + verdict = out.doc.manifest.scoring["verification"] + assert verdict["unchecked"] == [3] + assert verdict["flagged"] == [] + assert len(out.doc.manifest.refs) == 4 + + +def test_verify_declines_to_flag_a_set_too_small_to_judge( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """With two, "agreement with the others" is one pairwise number and cannot say which is odd.""" + _fake_faces(monkeypatch, [_SAME[0], _OTHER]) + out = _verify(_character(tmp_path, 2), on_outlier="quarantine") + + verdict = out.doc.manifest.scoring["verification"] + assert verdict["flagged"] == [] + assert "odd one out" in verdict["note"] + assert len(out.doc.manifest.refs) == 2 + + +def test_verify_removes_a_byte_identical_duplicate_even_in_flag_mode( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A duplicate is not a judgement call: it doubles that image's weight in a training mix and + spends a reference slot the model addresses by position.""" + from inline_core.characters import charfile as cf + + # Three vectors for three colours: the twin shares reference 0's pixels, so it shares its face. + _fake_faces(monkeypatch, _SAME[:3]) + identity = _character(tmp_path, 3) + doc = identity.doc + twin = dict(doc.manifest.refs[0]) + twin["path"] = cf.member_name("refs", 9, ".png") + doc.members[twin["path"]] = doc.members[doc.manifest.refs[0]["path"]] + doc.manifest.refs.append(twin) + + out = _verify(identity) + + verdict = out.doc.manifest.scoring["verification"] + assert verdict["removed"]["duplicates"] == ["refs/009.png"] + # Emptied, not left at [3]: the positions describe the set that survived, not the one checked. + assert verdict["duplicates"] == [] + assert len(out.doc.manifest.refs) == 3 + + +def test_a_stored_verdict_names_positions_in_the_set_that_survived( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Every list in the verdict is a position into `manifest.refs`, and removing one shifts each + position after it - so a report stored as found would ring a reference that is now another.""" + _fake_faces(monkeypatch, [_SAME[0], _SAME[1], _SAME[2], _OTHER]) + identity = _character(tmp_path, 4) + doc = identity.doc + # A duplicate of reference 0 in the middle, so removing it shifts the flagged impostor down. + twin = dict(doc.manifest.refs[0]) + twin["path"] = cf.member_name("refs", 9, ".png") + doc.members[twin["path"]] = doc.members[doc.manifest.refs[0]["path"]] + doc.manifest.refs.insert(1, twin) + + verdict = _verify(identity).doc.manifest.scoring["verification"] + + assert len(verdict["agreement"]) == 4, "the report still describes five references" + assert verdict["flagged"] == [3], "the impostor kept the position it held before the dedup" + + +def test_write_refuses_a_payload_built_from_a_different_reference_set( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Compile References uses its `character` input only to read settings - it compiles from the + doc Write hands it - so wiring Write ahead of the verify node would save the unchecked set.""" + from inline_core.graph.schema import Node + from inline_core.models.character.runner import CompileReferencesRunner, WriteCharacterRunner + + _fake_faces(monkeypatch, [*_SAME[:3], _OTHER]) + unchecked = _character(tmp_path, 4) + verified = _verify(Identity(doc=copy.deepcopy(unchecked.doc)), on_outlier="quarantine") + + payload = CompileReferencesRunner().run( + Node(id="p", type="character/references", params={}), {"character": [verified]}, _ctx() + ).outputs["payload"] + + with pytest.raises(ValueError, match="different version"): + WriteCharacterRunner().run( + Node(id="w", type="character/write", params={"filename": "Ada"}), + {"character": [unchecked], "payloads": [payload]}, + _ctx(), + ) + + +def test_a_verified_drop_reaches_the_training_set( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The LoRA path matters more than the reference path: a bad reference bakes into the weights. + + `commit_staged` reconciles the dataset against exactly what was staged, so a reference the + verify node took out is a row the next run removes rather than one left behind. + """ + from inline_core.models.character.runner import CharacterDatasetRunner + from inline_core.models.training.runner import TrainingBridge + + _fake_faces(monkeypatch, [*_SAME[:3], _OTHER]) + identity = _character(tmp_path, 4) + identity.doc.members["text/description.md"] = b"green jacket" + identity.doc.manifest.text = {"path": "text/description.md", "sha256": ""} + verified = _verify(identity, on_outlier="quarantine") + + staged: dict[str, object] = {} + + class _Training: + def list_datasets(self) -> list[dict[str, object]]: + return [] + + def create_dataset(self, inp: dict[str, object]) -> dict[str, object]: + return {"id": "d1", "name": inp["name"]} + + def stage_from_path(self, path: str) -> list[dict[str, object]]: + staged["files"] = sorted(p.name for p in Path(path).iterdir()) + return [{"assetId": f"a{n}"} for n, _ in enumerate(staged["files"])] # type: ignore[arg-type] + + def commit_staged(self, _did: str, rows: list[dict[str, object]]) -> list[dict[str, str]]: + return [{"id": f"i{n}"} for n, _ in enumerate(rows)] + + def set_caption(self, item_id: str, caption: str) -> None: + return None + + CharacterDatasetRunner(TrainingBridge(_Training())).run( + _node({}), {"character": [verified]}, _ctx(), # type: ignore[arg-type] + ) + + assert staged["files"] == ["0000.png", "0001.png", "0002.png"], "the dropped ref still trained" + + +def test_a_character_that_never_harvested_is_unchanged_by_the_feature(tmp_path: Path) -> None: + """The loop is opt-in and additive: a character built without it must compile the same bytes + and stage the same training rows as one built before it existed.""" + manifest = cf.Manifest(char_id="c", name="Ada", created_at=0, modified_at=0) + members: dict[str, bytes] = {} + for index in range(3): + image = Image.new("RGB", (320, 320), (index * 30 + 10, 90, 140)) + member = cf.member_name("refs", index, ".png") + data = encode._png_bytes(image) + members[member] = data + # No origin field at all, the way every character written before this was. + manifest.refs.append({"path": member, "sha256": cf.sha256_bytes(data)}) + doc = cf.CharDoc(manifest=manifest, members=members) + + encode.build_payload(manifest, members, encode.ref_images(doc)) + entry = manifest.payloads[encode.FLUX2_KLEIN_ARCH] + + assert entry["harvested_count"] == 0 + assert [f["path"] for f in entry["files"]] == [ + f"payloads/flux2-klein/ref_{i:03d}.png" for i in range(3) + ] + assert encode.harvested(manifest) == [] + assert len(encode.originals(manifest)) == 3 + + +def test_the_harvest_canvas_graph_validates_against_the_registered_descriptors( + tmp_path: Path, +) -> None: + """The harvest chain is only real if the port ids the canvas emits are the ones the nodes + declare. A mismatch is a run that dies at submit, and neither side's unit tests would see it.""" + import sqlite3 + + from inline_core.graph.registry import build_default_registry + from inline_core.graph.schema import parse_graph + from inline_core.graph.validate import validate + from inline_core.models.character.runner import register_character_nodes + from inline_core.studio import moodboard as mb + from inline_core.studio.graph_build import build_workflow_graph + from inline_core.studio.schema import apply_schema + + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + apply_schema(conn) + conn.execute("INSERT INTO project (id, name, created_at, updated_at) VALUES ('p','P',0,0)") + + conn.execute( + "INSERT INTO assets (id, project_id, name, file_path, kind, created_at) " + "VALUES ('take', 'p', 'take', 'assets/take.png', 'image', 0)" + ) + _image(tmp_path / "assets" / "take.png") + asset = mb.add_asset(conn, "take", 0, 0) + load = mb.add_core_node(conn, "character/load", 0, 0) + ingest = mb.add_core_node(conn, "character/ingest-approved", 0, 0) + write = mb.add_core_node(conn, "character/write", 0, 0) + mb.create_connector(conn, load["id"], ingest["id"], "character", "character") + mb.create_connector(conn, asset["id"], ingest["id"], "image", "image") + mb.create_connector(conn, ingest["id"], write["id"], "character", "character") + + # The default registry, because the take reaches the node as an `input/image` source node. + registry = build_default_registry() + register_character_nodes(registry) + graph_dict, target = build_workflow_graph(conn, tmp_path, write["id"], lambda _t, _p: False) + validate(parse_graph(graph_dict), target, registry) diff --git a/core/tests/test_characters_encode.py b/core/tests/test_characters_encode.py index 7be098e..e796b9c 100644 --- a/core/tests/test_characters_encode.py +++ b/core/tests/test_characters_encode.py @@ -6,6 +6,7 @@ from __future__ import annotations +import io from pathlib import Path import pytest @@ -355,3 +356,152 @@ def test_a_payload_is_judged_against_the_policy_it_was_built_with(tmp_path: Path assert encode.payload_stale(doc.manifest, encode.FLUX2_KLEIN_ARCH) is True, ( "a payload built under another policy no longer matches this reference set" ) + + +# --- origins, the harvested pool, and what must survive it --------------------------------------- + + +def _harvest(doc: cf.CharDoc, colour: tuple[int, int, int] = (10, 200, 30)) -> None: + """Add a harvested reference without running an encoder: only the bookkeeping is under test.""" + image = Image.new("RGB", (512, 512), colour) + member = cf.member_name("harvested", len(encode.harvested(doc.manifest)), ".png") + data = encode._png_bytes(image) + doc.members[member] = data + doc.manifest.refs.append( + { + "path": member, + "sha256": cf.sha256_bytes(data), + "width": image.width, + "height": image.height, + "origin": cf.ORIGIN_HARVESTED, + } + ) + + +def test_a_reference_with_no_origin_is_an_original() -> None: + """Every character written before harvesting existed has references and no origin field.""" + manifest, _members, _images = _manifest_with_refs(2) + assert len(encode.originals(manifest)) == 2 + assert encode.harvested(manifest) == [] + + +def test_harvesting_does_not_invalidate_a_trained_adapter() -> None: + """The fingerprint covers originals only. A harvested reference in it would mark the adapter + stale, `_extract_lora` drops a stale adapter with an INFO log, and the loop whose whole point + is a better adapter would silently switch off the one the user has.""" + manifest, members, images = _manifest_with_refs(3) + encode.build_payload(manifest, members, images) + encode.set_lora_payload( + manifest, members, b"ADAPTER", base="flux2-klein-4b", rank=16, steps=500, resolution=512 + ) + doc = cf.CharDoc(manifest=manifest, members=members) + key = encode.payload_key(encode.FLUX2_KLEIN_ARCH, encode.PAYLOAD_LORA) + assert cf.payload_valid(manifest, key, encode.LORA_PAYLOAD_VERSION) + + _harvest(doc) + + assert cf.payload_valid(manifest, key, encode.LORA_PAYLOAD_VERSION), "the adapter went stale" + assert cf.payload_valid(manifest, encode.FLUX2_KLEIN_ARCH, encode.PAYLOAD_ENCODER_VERSION) + + +def test_dropping_or_adding_an_original_still_invalidates() -> None: + manifest, members, images = _manifest_with_refs(3) + encode.build_payload(manifest, members, images) + doc = cf.CharDoc(manifest=manifest, members=members) + encode.drop_ref(doc, 0) + assert encode.payload_stale(manifest, encode.FLUX2_KLEIN_ARCH) + + +def test_a_recompile_takes_harvested_references_in_behind_the_originals(tmp_path: Path) -> None: + """Position is meaning - FLUX.2 addresses a reference by number - so the ones the user vouched + for hold the leading slots however the manifest happens to be ordered.""" + manifest, members, _images = _manifest_with_refs(2) + doc = cf.CharDoc(manifest=manifest, members=members) + _harvest(doc) + # An original added after the harvest lands at the end of `manifest.refs`, not before it. + encode.append_refs(doc, [_image(tmp_path / "late.png", (64, 64), (1, 2, 3))]) + assert [cf.origin_of(r) for r in manifest.refs] == [ + cf.ORIGIN_ORIGINAL, cf.ORIGIN_ORIGINAL, cf.ORIGIN_HARVESTED, cf.ORIGIN_ORIGINAL + ] + + encode.build_payload(manifest, members, encode.ref_images(doc)) + + entry = manifest.payloads[encode.FLUX2_KLEIN_ARCH] + assert entry["harvested_count"] == 1 + assert len(entry["files"]) == 4 + # The harvested one is last in the payload, whatever position it holds in the manifest. + sizes = [Image.open(io.BytesIO(members[f["path"]])).size for f in entry["files"]] + assert sizes[3] == (512, 512), "the harvested reference did not land in the last slot" + + +def test_hints_count_the_originals_only() -> None: + """Otherwise harvesting silences the very prompts - another angle, a full-body shot - that the + pool depends on being met.""" + manifest, members, _images = _manifest_with_refs(1) + doc = cf.CharDoc(manifest=manifest, members=members) + assert encode.hints_for(manifest) == ["Add a second angle"] + _harvest(doc) + _harvest(doc, (200, 10, 30)) + assert encode.hints_for(manifest) == ["Add a second angle"] + + +def test_the_harvest_cap_never_lets_the_pool_outgrow_the_originals() -> None: + manifest, _members, _images = _manifest_with_refs(3) + assert encode.harvest_cap(manifest) == 3 + manifest.refs = manifest.refs[:1] + assert encode.harvest_cap(manifest) == 1 + + +def test_changing_the_reference_set_marks_scoring_for_a_rebuild() -> None: + """`drop_ref` leaves the stored per-reference lists describing a set that no longer exists, so + scoring kept best-matching a take against the reference just deleted for being the wrong + person. The lists are compacted, so there is no index surgery available - it has to go.""" + manifest, members, _images = _manifest_with_refs(3) + manifest.scoring = {"refFramings": [0.1, 0.1, 0.1], "refCount": 3, "originals": {"refs": []}} + members["scoring/embeds_sface.json"] = b'{"vectors":[]}' + doc = cf.CharDoc(manifest=manifest, members=members) + + encode.drop_ref(doc, 1) + + assert "refFramings" not in manifest.scoring, "a phantom reference survived in scoring" + assert "scoring/embeds_sface.json" not in members + # The frozen identity is not derived from the set that changed, so it is not collateral. + assert "originals" in manifest.scoring + + +def test_decoding_references_never_returns_a_short_list() -> None: + """Every scoring position is an index into `manifest.refs`. A skipped member would shorten the + list and shift each position after it onto a different reference, silently.""" + manifest, members, _images = _manifest_with_refs(3) + doc = cf.CharDoc(manifest=manifest, members=members) + assert len(encode.ref_images(doc)) == 3 + + members.pop(str(manifest.refs[1]["path"])) + with pytest.raises(cf.CharFileError, match="missing"): + encode.ref_images(doc) + + +def test_pruning_drops_the_harvested_reference_that_adds_least(monkeypatch) -> None: + """Coverage, not score: a pool of near-duplicates of the best-scoring angle is worth less to a + compile or a train than one spanning the angles the originals miss.""" + from inline_core.characters import scoring + + manifest, members, _images = _manifest_with_refs(2) + doc = cf.CharDoc(manifest=manifest, members=members) + # Two originals sitting on one axis; a near-duplicate of them, and a genuinely new angle. + frozen = {"refs/000.png": [1.0, 0.0, 0.0], "refs/001.png": [0.99, 0.1, 0.0]} + members["scoring/originals_dinov2-base.json"] = scoring.dump_keyed(frozen) + manifest.scoring["originals"] = {"refs": [{"path": p, "sha256": ""} for p in frozen]} + _harvest(doc, (10, 200, 30)) + _harvest(doc, (200, 10, 30)) + members["scoring/harvested_dinov2-base.json"] = scoring.dump_keyed( + {"harvested/000.png": [0.98, 0.0, 0.1], "harvested/001.png": [0.0, 0.0, 1.0]} + ) + # A cap of one, so exactly one of the two has to go. + monkeypatch.setattr(encode, "MAX_HARVESTED", 1) + + removed = encode.prune_harvested(doc) + + assert removed == ["harvested/000.png"], "the near-duplicate should have gone, not the new angle" + assert [r["path"] for r in encode.harvested(manifest)] == ["harvested/001.png"] + assert len(encode.originals(manifest)) == 2, "an original was pruned" diff --git a/core/tests/test_characters_rpc.py b/core/tests/test_characters_rpc.py index bb5e080..438ffd9 100644 --- a/core/tests/test_characters_rpc.py +++ b/core/tests/test_characters_rpc.py @@ -235,3 +235,54 @@ def test_a_character_written_before_the_current_encoders_rebuilds_when_applied( rebuilt = cf.read(path) assert not cf.centroid_valid(rebuilt.manifest, "dinov2-base", "0"), "stale version survived" assert "refFramings" in rebuilt.manifest.scoring, "framings were not recomputed" + + +def test_rescoring_a_stale_character_keeps_its_trained_adapter( + client: TestClient, project: dict +) -> None: + """Rebuilding scoring must not rebuild the character. + + `char_encode` builds a fresh manifest and a fresh members dict, so rescoring through it dropped + the trained adapter, every payload but flux2-klein and the apply override - and the write that + follows put that on disk, from a path a render reaches on every take it scores. + """ + from inline_core.characters import charfile as cf + from inline_core.characters import encode, library + from inline_core.studio.characters import Characters + + _make_character() + path = library.resolve("Ada.char") + assert path is not None + + doc = cf.read(path) + encode.set_lora_payload( + doc.manifest, + doc.members, + b"adapter-bytes", + arch=encode.FLUX2_KLEIN_ARCH, + base="flux2-klein-4b", + rank=16, + steps=500, + resolution=512, + ) + doc.manifest.payloads["minimax-h3"] = {"payload_version": 1, "type": "ref", "files": []} + doc.manifest.apply[encode.FLUX2_KLEIN_ARCH] = "lora" + doc.manifest.reserved = {"adapters": {}, "video_payloads": {}, "members": ["keep-me"]} + # The state a shipped encoder bump puts every character on disk into. + doc.manifest.scoring = { + **doc.manifest.scoring, + "encoders": [{"id": "dinov2-base", "version": "0", "dim": 768}], + } + cf.write(path, doc) + + Characters(object(), _NullEvents()).score_take(path, "Ada.char") + + rebuilt = cf.read(path) + key = encode.payload_key(encode.FLUX2_KLEIN_ARCH, encode.PAYLOAD_LORA) + assert key in rebuilt.manifest.payloads, "the trained adapter was destroyed" + assert rebuilt.members[f"payloads/{key}/adapter.safetensors"] == b"adapter-bytes" + assert "minimax-h3" in rebuilt.manifest.payloads, "another model's payload was destroyed" + assert rebuilt.manifest.apply.get(encode.FLUX2_KLEIN_ARCH) == "lora" + assert rebuilt.manifest.reserved.get("members") == ["keep-me"] + # And it did actually rescore, or the assertions above pass on a file nothing touched. + assert cf.centroid_valid(rebuilt.manifest, "dinov2-base", "2"), "scoring was not rebuilt" diff --git a/core/tests/test_characters_scoring.py b/core/tests/test_characters_scoring.py index f4b85dd..09209d2 100644 --- a/core/tests/test_characters_scoring.py +++ b/core/tests/test_characters_scoring.py @@ -242,6 +242,59 @@ def test_a_single_reference_agrees_with_itself() -> None: assert scoring.flagged_references([[1.0, 0.0]]) == [] +# --- references with no face --------------------------------------------------------------------- + + +def test_a_reference_with_no_face_does_not_drag_the_others_down() -> None: + """`cosine` against an empty vector is 0.0, so counting one as a score is a hard zero in the + mean. A four-reference set with two wide shots would put every genuine reference under the + floor, and in a mode that removes them it would delete the good ones.""" + refs = _same_person(4) + clean = scoring.reference_agreement(refs) + with_gaps = scoring.reference_agreement([refs[0], [], refs[1], [], refs[2], refs[3]]) + + assert with_gaps[1] is None and with_gaps[3] is None, "a missing face is not a low score" + assert [with_gaps[i] for i in (0, 2, 4, 5)] == clean + assert scoring.flagged_references([refs[0], [], refs[1], [], refs[2], refs[3]]) == [] + + +def test_flagged_positions_are_reference_positions_not_gallery_positions() -> None: + """The index is shown to the user as "reference N", so it has to survive a gap before it.""" + refs = _same_person(3) + [_different_person()] + assert scoring.flagged_references([[], refs[0], refs[1], refs[2], refs[3]]) == [4] + + +def test_empty_slots_do_not_count_toward_the_minimum_to_flag() -> None: + """Two real faces and a gap is still two, which cannot say which of the two is the odd one.""" + refs = _same_person(1) + [_different_person()] + assert scoring.flagged_references([refs[0], [], refs[1]]) == [] + + +# --- the harvest arm ----------------------------------------------------------------------------- + + +def test_a_candidate_is_measured_against_the_gallery_it_is_not_in() -> None: + gallery = _same_person(4) + assert (scoring.agreement_against(gallery[0], gallery) or 0) > 90.0 + assert (scoring.agreement_against(_different_person(), gallery) or 0) < 10.0 + + +def test_a_candidate_is_not_scored_against_too_small_a_gallery() -> None: + """The floor is a mean over a set; against one or two references it is a pairwise number, and + same-person pairs run as low as 23.0 - under the floor.""" + gallery = _same_person(4) + assert scoring.agreement_against(gallery[0], gallery[:2]) is None + assert scoring.agreement_against([], gallery) is None + + +def test_coverage_is_highest_for_the_reference_least_like_the_others() -> None: + """What decides which harvested reference survives the cap: a near-duplicate of an angle the + pool already holds is worth less than one that spans an angle it misses.""" + values = scoring.coverage_values(_same_person(3) + [_different_person()]) + assert values[3] is not None + assert all(v is not None and v < values[3] for v in values[:3]) + + # --- lookalike discrimination ------------------------------------------------------------------ diff --git a/core/tests/test_h3_characters.py b/core/tests/test_h3_characters.py index 5e059fb..8ea5305 100644 --- a/core/tests/test_h3_characters.py +++ b/core/tests/test_h3_characters.py @@ -2,6 +2,7 @@ from __future__ import annotations +import io from pathlib import Path from typing import Any @@ -133,3 +134,248 @@ def test_a_clip_that_cannot_be_read_scores_nothing(monkeypatch) -> None: monkeypatch.setattr(mod, "_sample_frames", lambda *_a, **_k: []) assert mod._score_video(Path("clip.mp4"), {}, [], [], []) is None + + +# --- which route a node gets ---------------------------------------------------------------------- + + +def test_the_reference_node_asks_for_references_over_an_adapter(tmp_path, monkeypatch) -> None: + """A character carrying both defaults to its adapter, which leaves H3's reference partition with + nothing to condition on: it refused the run saying no reference was wired, while one was.""" + from inline_core.characters import apply as characters + + seen: dict[str, object] = {} + + def fake(chosen: str, arch: str = "", prefer: str | None = None): + seen["arch"], seen["prefer"] = arch, prefer + return None + + monkeypatch.setattr(characters, "char_apply", fake) + from inline_core.models.minimaxh3.runner import VARIANTS, _apply_character + + ref = next(v for v in VARIANTS if v.references) + _apply_character({"character": [type("I", (), {"file": "x.char"})()]}, ref) + assert seen == {"arch": "minimax-h3", "prefer": "reference"} + + +def test_a_node_with_no_reference_channel_takes_whatever_the_character_prefers(monkeypatch) -> None: + from inline_core.characters import apply as characters + + seen: dict[str, object] = {} + + def fake(chosen: str, arch: str = "", prefer: str | None = None): + seen["prefer"] = prefer + return None + + monkeypatch.setattr(characters, "char_apply", fake) + from inline_core.models.minimaxh3.runner import VARIANTS, _apply_character + + fl2va = next(v for v in VARIANTS if not v.references) + _apply_character({"character": [type("I", (), {"file": "x.char"})()]}, fl2va) + assert seen["prefer"] is None + + +def test_prefer_overrides_the_adapter_default() -> None: + """`char_apply`'s own rule is adapter-wins; `prefer` is what a node uses to say it cannot.""" + import inspect + + from inline_core.characters.apply import char_apply + + assert "prefer" in inspect.signature(char_apply).parameters + + +def test_a_character_with_more_references_than_the_model_takes_is_trimmed(monkeypatch) -> None: + """H3 takes 9 images; a character built for another model may carry more. Refusing sent a user + to unwire images they had not wired, because every one of them came from the character.""" + from inline_core.characters import apply as characters + from inline_core.characters.apply import AppliedCharacter + from inline_core.models.minimaxh3.runner import VARIANTS, _apply_character + + monkeypatch.setattr( + characters, "char_apply", + lambda *_a, **_k: AppliedCharacter("Ada", [f"r{i}" for i in range(10)], "freckles"), + ) + ref = next(v for v in VARIANTS if v.references) + out = _apply_character({"character": [type("I", (), {"file": "x.char"})()]}, ref) + assert out is not None and len(out.refs) == 9 + + +def test_the_prefix_never_names_a_reference_that_was_trimmed(monkeypatch) -> None: + """The prefix is what the prompt resolves; naming when nine were sent addresses a + position the model cannot see.""" + from inline_core.characters import apply as characters + from inline_core.characters.apply import AppliedCharacter + from inline_core.models.minimaxh3.runner import VARIANTS, _apply_character + + monkeypatch.setattr( + characters, "char_apply", + lambda *_a, **_k: AppliedCharacter("Ada", [f"r{i}" for i in range(10)], "freckles"), + ) + ref = next(v for v in VARIANTS if v.references) + out = _apply_character({"character": [type("I", (), {"file": "x.char"})()]}, ref) + assert out is not None + assert "" in out.prefix and "" not in out.prefix + + +def test_wired_images_keep_priority_over_the_character(monkeypatch) -> None: + """What the user wired is explicit; the character fills whatever room is left.""" + from inline_core.characters import apply as characters + from inline_core.characters.apply import AppliedCharacter + from inline_core.models.minimaxh3.runner import VARIANTS, _apply_character + + monkeypatch.setattr( + characters, "char_apply", + lambda *_a, **_k: AppliedCharacter("Ada", [f"r{i}" for i in range(10)], "freckles"), + ) + ref = next(v for v in VARIANTS if v.references) + inputs = { + "character": [type("I", (), {"file": "x.char"})()], + "references": ["mine1", "mine2", "mine3"], + } + out = _apply_character(inputs, ref) + assert out is not None and len(out.refs) == 6, "3 wired + 6 from the character is the 9 cap" + assert out.prefix.startswith(""), "and it is numbered after the wired ones" + + + +def test_the_resolution_param_is_on_the_node_face_and_defaults_to_capping() -> None: + """Default 1024, not uncapped: H3's own policy is 2048, and a character compiled there is what + put 36,864 vision tokens on the card.""" + from inline_core.models.character.runner import COMPILE_REFS + + field = next(p for p in COMPILE_REFS.params if p.key == "ref_resolution") + assert field.label == "Resized Reference Resolution" + assert field.default == 1024 + assert field.on_face is True + assert field.min == encode.NO_REFERENCE_CAP + + +def test_the_cap_only_ever_lowers_a_model_policy() -> None: + """It is a ceiling, not a target: raising H3 past 2048 or FLUX.2 past its area cap would ask + each model for a size it does not accept.""" + h3, flux = encode.MINIMAX_H3_ARCH, encode.FLUX2_KLEIN_ARCH + assert encode.capped_policy(h3, 1024)["short_edge"] == 1024 + assert encode.capped_policy(h3, 4096)["short_edge"] == 2048 + assert encode.capped_policy(flux, 2048)["max_pixels"] == 1024 * 1024 + assert encode.capped_policy(flux, 512)["max_pixels"] == 512 * 512 + # -1 means the model's own policy, which is the only safe reading of "no resize": H3's packer + # requires the 32px grid, so a raw source size is not something it can be handed. + for arch in (h3, flux): + assert encode.capped_policy(arch, -1) == encode.reference_policy(arch) + assert encode.capped_policy(arch, None) == encode.reference_policy(arch) + + +def test_a_graph_saved_before_the_param_existed_still_gets_the_cap() -> None: + """Reading a missing param as uncapped would silently compile old graphs at 2048.""" + from inline_core.models.character.runner import _resolution + + assert _resolution(None) == 1024 + assert _resolution("") == 1024 + assert _resolution(2048) == 2048 + assert _resolution(-1) == encode.NO_REFERENCE_CAP + assert _resolution(0) == encode.NO_REFERENCE_CAP + + +def test_capping_h3_to_1024_quarters_the_vision_tokens() -> None: + """The arithmetic the whole param exists for.""" + from PIL import Image + + from inline_core.characters import charfile as cf + + manifest = cf.Manifest(char_id="c", name="c", created_at=0, modified_at=0) + members: dict[str, bytes] = {} + images = [Image.new("RGB", (3840, 2160)) for _ in range(2)] + for index, image in enumerate(images): + path = f"refs/ref_{index:03d}.png" + members[path] = encode._png_bytes(image) + manifest.refs.append({"path": path, "sha256": cf.sha256_bytes(members[path])}) + + def tokens(cap: int) -> int: + arch = encode.MINIMAX_H3_ARCH + encode.build_payload(manifest, members, images, arch, encode.capped_policy(arch, cap)) + total = 0 + for entry in manifest.payloads[arch]["files"]: + width, height = Image.open(io.BytesIO(members[entry["path"]])).size + total += (width // 32) * (height // 32) + return total + + assert tokens(1024) * 4 == tokens(2048) + # A 4K source is not special: the policy resizes onto its target either way. + assert tokens(-1) == tokens(2048) + + + +def test_an_encoder_oom_points_at_the_character_not_the_canvas(monkeypatch) -> None: + """The canvas hint sent a user to resize twice for nothing: references are encoded before any + frame exists, so a 1344x768 -> 544x768 drop left the failing allocation byte-identical. The + size that matters was fixed when the character was compiled, so that is what the error names. + """ + import tempfile + + from PIL import Image + + from inline_core.models import pipeline_runtime as rt + from inline_core.models.minimaxh3.runner import Request, _oom + from inline_core.models.references import ReferenceKind + + # Stubbed because it reads the live card otherwise, so this asserted on whatever else happened + # to be running: it passed on an idle box and failed beside a training run. + monkeypatch.setattr(rt, "foreign_vram_bytes", lambda *a, **k: 0) + + with tempfile.TemporaryDirectory() as tmp: + paths = [] + for index in range(9): + path = f"{tmp}/ref{index}.png" + Image.new("RGB", (2048, 2048)).save(path) + paths.append(path) + refs = tuple( + type("R", (), {"kind": ReferenceKind.IMAGE, "value": type("V", (), {"path": p})()})() + for p in paths + ) + request = Request( + prompt="", num_frames=144, width=544, height=768, num_inference_steps=50, + seed=1, partition="ref2va", references=refs, + ) + message = _oom(request) + + # Measured off the pixels, not off a setting this node no longer carries. + assert "36,864 vision tokens" in message + assert "Resized Reference Resolution" in message + assert "does not affect this step" in message + assert "960x544" not in message + + # With no references the canvas really is the lever, so that hint has to survive untouched. + plain = Request( + prompt="", num_frames=144, width=1344, height=768, + num_inference_steps=50, seed=1, partition="fl2va", + ) + assert "960x544" in _oom(plain) + + +def test_a_card_held_by_another_process_is_named_before_anything_on_this_node(monkeypatch) -> None: + """A run with 5 references and 5,120 vision tokens was told to lower its reference resolution, + while a training run held 29 of the card's 46 GB. Nothing on this node frees that, and every + other hint sends the user to change a setting that was never the cause.""" + from inline_core.models import pipeline_runtime as rt + from inline_core.models.minimaxh3.runner import Request, _oom + from inline_core.models.references import ReferenceKind + + refs = tuple(type("R", (), {"kind": ReferenceKind.IMAGE})() for _ in range(5)) + request = Request( + prompt="", num_frames=144, width=544, height=768, num_inference_steps=50, + seed=1, partition="ref2va", references=refs, + ) + + monkeypatch.setattr(rt, "foreign_vram_bytes", lambda *a, **k: 29 * 1024**3) + busy = _oom(request) + assert "29.0 GB" in busy and "another process" in busy + assert "Resized Reference Resolution" not in busy, "do not blame the character" + assert "960x544" not in busy, "do not blame the canvas" + + # Below the floor it is noise, and the reference hint is the useful one again. + monkeypatch.setattr(rt, "foreign_vram_bytes", lambda *a, **k: 200 * 1024**2) + assert "Resized Reference Resolution" in _oom(request) + + # A host-RAM exhaustion is never explained by another process's VRAM. + monkeypatch.setattr(rt, "foreign_vram_bytes", lambda *a, **k: 29 * 1024**3) + assert "another process" not in _oom(request, host=True) diff --git a/core/tests/test_minimaxh3_nodes.py b/core/tests/test_minimaxh3_nodes.py index fd4aa6d..8cac9a5 100644 --- a/core/tests/test_minimaxh3_nodes.py +++ b/core/tests/test_minimaxh3_nodes.py @@ -120,9 +120,11 @@ def test_fps_is_not_editable(node_type: str) -> None: assert "fps" not in {p.key for p in DESCRIPTORS[node_type].params} -def test_only_the_reference_node_offers_reference_detail() -> None: - assert "ref_image_size" in {p.key for p in DESCRIPTORS[REF].params} - assert "ref_image_size" not in {p.key for p in DESCRIPTORS[T2V].params} +def test_no_node_carries_a_reference_size_of_its_own() -> None: + """One number, one place. Reference size is fixed when the character is compiled, so a second + control here would let a node promise a size the stored payload does not have.""" + for descriptor in DESCRIPTORS.values(): + assert "ref_image_size" not in {p.key for p in descriptor.params} def test_the_default_duration_is_a_real_grid_point() -> None: @@ -407,10 +409,14 @@ def test_header_reads_are_cached_against_size_and_mtime(models_root: Path) -> No def test_the_reference_node_requires_the_other_partition(models_root: Path) -> None: + """Each partition asks only for its own transformer, and asks for the fp8 build by default.""" fl2va = {c.id: c for c in MiniMaxH3Provider("fl2va").components()} ref2va = {c.id: c for c in MiniMaxH3Provider("ref2va").components()} - assert fl2va["h3-ref2va"].optional and not fl2va["h3-fl2va"].optional - assert ref2va["h3-fl2va"].optional and not ref2va["h3-ref2va"].optional + assert not fl2va["h3-fl2va-fp8"].optional and fl2va["h3-ref2va-fp8"].optional + assert not ref2va["h3-ref2va-fp8"].optional and ref2va["h3-fl2va-fp8"].optional + # The bf16 builds are the training route, never a second thing to download to render. + for entries in (fl2va, ref2va): + assert entries["h3-fl2va"].optional and entries["h3-ref2va"].optional def test_the_folder_components_declare_a_repo_folder(models_root: Path) -> None: @@ -509,14 +515,19 @@ def set_footprint(self, *_a: object) -> None: ... def fit_estimate(self) -> None: return None -def test_the_fp8_build_is_offered_as_an_optional_download(models_root: Path) -> None: - """A third the download for the same model, so it belongs in the popup. Optional, because the - trainer cannot use it and the bf16 file stays the one a full install needs.""" +def test_generation_asks_for_the_fp8_build_and_training_for_bf16(models_root: Path) -> None: + """A third of the download for the same render, and it fits cards that cannot hold the 66.3 GB + bf16 at all - so a fresh install should not be told to fetch the big one first. Training still + names bf16: fp8 renders, it does not fine-tune.""" entries = {c.id: c for c in reqs.components("fl2va")} fp8 = entries["h3-fl2va-fp8"] - assert fp8.optional and fp8.filename == reqs.FL2VA_FP8_FILE + assert not fp8.optional and fp8.filename == reqs.FL2VA_FP8_FILE assert "generation only" in fp8.label - assert not entries["h3-fl2va"].optional + assert entries["h3-fl2va"].optional + assert "needed to train" in entries["h3-fl2va"].label + + training = {c.id: c for c in reqs.components("fl2va", fp8_substitutes=False)} + assert not training["h3-fl2va"].optional and training["h3-fl2va-fp8"].optional def test_training_refuses_a_pruned_build_by_name(models_root: Path) -> None: @@ -684,3 +695,29 @@ def cancelled(_done: int, _total: int) -> None: with loop.progress_bar(total=1) as bar: bar.update() # must not raise: the cancelled run's callback is no longer installed + + +def test_an_fp8_transformer_satisfies_the_partition_on_its_own(models_root: Path) -> None: + """A box holding only the fp8 build was told its 66.3 GB bf16 twin was missing: the ref2va fp8 + file was declared nowhere at all, so the only reference transformer on offer was one most + cards cannot hold. A partition needs *a* transformer, not a particular one.""" + fresh = {c.id: c for c in MiniMaxH3Provider("ref2va").components()} + assert not fresh["h3-ref2va-fp8"].optional, "with nothing on disk the partition still needs one" + + (models_root / "diffusion_models" / reqs.REF2VA_FILE).write_bytes(b"x") + held = {c.id: c for c in MiniMaxH3Provider("ref2va").components()} + assert held["h3-ref2va"].present + assert held["h3-ref2va-fp8"].optional, "holding bf16 is not missing the fp8 build" + # The other partition is unaffected: its own pair is still unsatisfied. + assert not {c.id: c for c in MiniMaxH3Provider("fl2va").components()}["h3-fl2va-fp8"].optional + + +def test_both_partitions_offer_an_fp8_build(models_root: Path) -> None: + """FL2VA had one and ref2va did not, which is the asymmetry that hid the smaller build.""" + by_id = {c.id: c for c in MiniMaxH3Provider().components()} + for component_id, filename in ( + ("h3-fl2va-fp8", reqs.FL2VA_FP8_FILE), + ("h3-ref2va-fp8", reqs.REF2VA_FP8_FILE), + ): + assert by_id[component_id].filename == filename + assert by_id[component_id].repo_file.endswith(filename) diff --git a/core/tests/test_minimaxh3_nvfp4.py b/core/tests/test_minimaxh3_nvfp4.py new file mode 100644 index 0000000..c869b34 --- /dev/null +++ b/core/tests/test_minimaxh3_nvfp4.py @@ -0,0 +1,124 @@ +"""NVFP4 unpacking, against ComfyUI's own quantiser rather than against our own assumptions.""" + +from __future__ import annotations + +import pytest + +torch = pytest.importorskip("torch") + +from inline_core.models.minimaxh3 import nvfp4 # noqa: E402 + + +def _weight(rows: int, cols: int, seed: int = 0) -> torch.Tensor: + generator = torch.Generator().manual_seed(seed) + return torch.randn(rows, cols, generator=generator, dtype=torch.float32) * 0.02 + + +def test_the_swizzle_round_trips() -> None: + """`from_blocked` has to invert `to_blocked` exactly; a near-miss still scores ~0.9 against a + real checkpoint, which reads as "close enough" and is not.""" + scales = torch.arange(128 * 320, dtype=torch.float32).reshape(128, 320) + assert torch.equal(nvfp4.from_blocked(nvfp4.to_blocked(scales), 128, 320), scales) + + +def test_the_swizzle_round_trips_across_several_tiles() -> None: + for rows, cols in ((256, 320), (384, 64), (128, 4)): + scales = torch.arange(rows * cols, dtype=torch.float32).reshape(rows, cols) + assert torch.equal(nvfp4.from_blocked(nvfp4.to_blocked(scales), rows, cols), scales) + + +def test_the_swizzle_is_not_the_identity() -> None: + """Guards the test above: if `to_blocked` were a no-op both would pass and prove nothing.""" + scales = torch.arange(128 * 320, dtype=torch.float32).reshape(128, 320) + assert not torch.equal(nvfp4.to_blocked(scales), scales) + + +def test_unpacking_is_high_nibble_first() -> None: + table = nvfp4.e2m1_table() + # 0x17 -> high 1 (0.5), low 7 (6.0). The reversed reading scores ~0 on a real checkpoint. + got = nvfp4.unpack_fp4(torch.tensor([[0x17]], dtype=torch.uint8), table) + assert got.tolist() == [[0.5, 6.0]] + # High bit is the sign, so 0x8F is -0.0 then -6.0. + assert nvfp4.unpack_fp4(torch.tensor([[0x8F]], dtype=torch.uint8), table).tolist() == [ + [-0.0, -6.0] + ] + + +def test_a_quantised_weight_dequantises_back() -> None: + """The whole contract: ComfyUI packs, we unpack, and the result is the same weight to within + 4-bit block quantisation error.""" + weight = _weight(256, 512) + packed, block_scale, global_scale = nvfp4.quantize_reference(weight) + assert packed.shape == (256, 256) + assert packed.dtype == torch.uint8 + assert block_scale.dtype == torch.float8_e4m3fn + + got = nvfp4.dequantize( + packed, block_scale, global_scale, + out_features=256, in_features=512, dtype=torch.float32, + ) + assert got.shape == weight.shape + cosine = torch.nn.functional.cosine_similarity(got.flatten(), weight.flatten(), dim=0) + assert cosine > 0.99, cosine + # Block-16 scaling keeps the magnitude, which is what a wrong scale convention destroys first. + assert 0.9 < got.std() / weight.std() < 1.1 + + +def test_skipping_the_swizzle_is_visibly_wrong() -> None: + """Records the bug that cost the most time: reading the scales in stored order still produces a + plausible-looking weight, so nothing short of comparing against a reference catches it.""" + weight = _weight(256, 512) + packed, block_scale, global_scale = nvfp4.quantize_reference(weight) + table = nvfp4.e2m1_table() + values = nvfp4.unpack_fp4(packed, table) + naive = ( + values.reshape(256, 32, 16) * block_scale.to(torch.float32).unsqueeze(-1) + ).reshape(256, 512) * float(global_scale) + cosine = torch.nn.functional.cosine_similarity(naive.flatten(), weight.flatten(), dim=0) + assert cosine < 0.95, "the un-swizzled read should be clearly wrong, not subtly wrong" + + +def test_the_linear_matches_an_ordinary_one() -> None: + weight = _weight(128, 256) + bias = torch.randn(128, dtype=torch.float32) * 0.01 + packed, block_scale, global_scale = nvfp4.quantize_reference(weight) + + layer = nvfp4.NVFP4Linear(256, 128, bias=True, dtype=torch.float32) + layer.weight.copy_(packed) + layer.weight_scale.copy_(block_scale) + layer.weight_scale_2.copy_(global_scale) + with torch.no_grad(): + layer.bias.copy_(bias) + + x = torch.randn(4, 256, dtype=torch.float32) + got = layer(x) + want = torch.nn.functional.linear(x, weight, bias) + assert got.shape == want.shape + assert torch.nn.functional.cosine_similarity(got.flatten(), want.flatten(), dim=0) > 0.99 + + +def test_the_weight_is_never_materialised_on_the_module() -> None: + """Holding the unpacked weight would defeat the point: the packed buffers are the reason a + 15.7 GB file stands in for a 63 GB folder.""" + layer = nvfp4.NVFP4Linear(256, 128, bias=False, dtype=torch.float32) + assert layer.weight.dtype == torch.uint8 + assert layer.weight.numel() == 128 * 128, "one byte per two weights" + + +def test_awq_smoothing_is_applied_to_the_input() -> None: + """Measured on the real file: `w * pre_quant_scale` recovers the original (1.03) where dividing + does not (0.59), so the activation is what carries the factor.""" + weight = _weight(64, 128) + smooth = torch.rand(128, dtype=torch.float32) + 0.5 + packed, block_scale, global_scale = nvfp4.quantize_reference(weight / smooth) + + layer = nvfp4.NVFP4Linear(128, 64, bias=False, dtype=torch.float32) + layer.weight.copy_(packed) + layer.weight_scale.copy_(block_scale) + layer.weight_scale_2.copy_(global_scale) + layer.pre_quant_scale = smooth + + x = torch.randn(4, 128, dtype=torch.float32) + got = layer(x) + want = torch.nn.functional.linear(x, weight) + assert torch.nn.functional.cosine_similarity(got.flatten(), want.flatten(), dim=0) > 0.99 diff --git a/core/tests/test_model_download_queue.py b/core/tests/test_model_download_queue.py index 1d51149..32fdc46 100644 --- a/core/tests/test_model_download_queue.py +++ b/core/tests/test_model_download_queue.py @@ -163,3 +163,43 @@ def fake_snapshot(repo: str, **kw: Any) -> str: assert landed.is_dir(), "the folder is what the node loads from" assert (landed / "model.safetensors").is_file() assert seen["allow_patterns"] == ["config.json", "model.safetensors"] + + +def test_xet_is_turned_back_on_only_for_the_call_that_needs_it() -> None: + """The server disables Xet at start so a download reports per-chunk progress, but plain HTTP + refuses anything over 50GB - every H3 transformer. The switch has to be per call, and it has to + be a real one: `is_xet_available` reads the constant on each call rather than at import.""" + from huggingface_hub import constants + from huggingface_hub.utils._runtime import is_xet_available + + from inline_core.studio.models import _with_xet + + previous = constants.HF_HUB_DISABLE_XET + constants.HF_HUB_DISABLE_XET = True + try: + assert is_xet_available() is False, "disabled outside the call" + seen: list[bool] = [] + _with_xet(lambda: (seen.append(is_xet_available()), "path")[1]) + assert seen == [True], "enabled inside it" + assert is_xet_available() is False, "and restored after, so the next download keeps its bar" + finally: + constants.HF_HUB_DISABLE_XET = previous + + +def test_the_flag_is_restored_even_when_the_download_raises() -> None: + from huggingface_hub import constants + + from inline_core.studio.models import _with_xet + + previous = constants.HF_HUB_DISABLE_XET + constants.HF_HUB_DISABLE_XET = True + try: + def boom() -> str: + raise RuntimeError("network died") + + with pytest.raises(RuntimeError): + _with_xet(boom) + assert constants.HF_HUB_DISABLE_XET is True + finally: + constants.HF_HUB_DISABLE_XET = previous + diff --git a/core/tests/test_studio_rpc.py b/core/tests/test_studio_rpc.py index d1da825..9af2199 100644 --- a/core/tests/test_studio_rpc.py +++ b/core/tests/test_studio_rpc.py @@ -83,6 +83,8 @@ def test_full_project_and_canvas_flow(client) -> None: "control/apply", "character/encode", "character/edit", + "character/verify-refs", + "character/ingest-approved", "character/load", "character/dataset", "character/references", diff --git a/core/tests/test_training_requirements.py b/core/tests/test_training_requirements.py index 873a1e2..600fc04 100644 --- a/core/tests/test_training_requirements.py +++ b/core/tests/test_training_requirements.py @@ -115,3 +115,22 @@ def test_the_recipe_puts_them_on_the_training_node() -> None: assert data["hyperparams"]["arch"] == "krea2" names = [m["name"] for m in data["models"]] assert "krea2_raw_bf16.safetensors" in names + + +def test_an_fp8_build_never_stands_in_for_training(tmp_path, monkeypatch) -> None: + """It substitutes for generation and not for training: the label says "generation only", and a + pruned fp8 checkpoint fine-tunes into nonsense. Relaxing the pair for both dropped the + transformer out of training's pre-flight entirely, because _required() keeps only the + non-optional ones.""" + from inline_core.models.minimaxh3 import requirements as reqs + + monkeypatch.setenv("INLINE_MODELS_DIR", str(tmp_path)) + (tmp_path / "diffusion_models").mkdir(parents=True) + (tmp_path / "diffusion_models" / reqs.FL2VA_FP8_FILE).write_bytes(b"x") + + assert ("diffusion_models", reqs.FL2VA_FILE) in _files("minimax-h3") + # Generation is happy with what is on disk; training still names the full-precision build. + generation = {c.id: c for c in reqs.components("fl2va")} + assert generation["h3-fl2va"].optional + training = {c.id: c for c in reqs.components("fl2va", fp8_substitutes=False)} + assert not training["h3-fl2va"].optional diff --git a/src/renderer/lib/starterGraph.character.test.ts b/src/renderer/lib/starterGraph.character.test.ts index 044b877..03164e8 100644 --- a/src/renderer/lib/starterGraph.character.test.ts +++ b/src/renderer/lib/starterGraph.character.test.ts @@ -28,15 +28,16 @@ beforeEach(() => { }) describe('buildCharacterStarter', () => { - it('drops the four nodes a reference character needs', async () => { + it('drops the five nodes a reference character needs', async () => { const ids = await buildCharacterStarter({ x: 0, y: 0 }) expect(added.map((a) => a.type)).toEqual([ 'loader', 'character/encode', + 'character/verify-refs', 'character/references', 'character/write', ]) - expect(ids).toHaveLength(4) + expect(ids).toHaveLength(5) }) it('drops below whatever already occupies the viewport centre', async () => { @@ -51,13 +52,15 @@ describe('buildCharacterStarter', () => { expect(placed[0]).toBeGreaterThanOrEqual(250) }) - it('wires the identity to Write directly, not through the payload', async () => { - // Payloads are compiled from the identity; Write needs both, so Encode fans out to each. + it('gives Write the verified identity, not the raw one', async () => { + // A payload node compiles from the doc Write hands it, not from its own input, so wiring + // Encode straight to Write would save the reference set nothing checked. await buildCharacterStarter({ x: 0, y: 0 }) expect(wires).toEqual([ 'loader:image -> character/encode:images', - 'character/encode:character -> character/references:character', - 'character/encode:character -> character/write:character', + 'character/encode:character -> character/verify-refs:character', + 'character/verify-refs:character -> character/references:character', + 'character/verify-refs:character -> character/write:character', 'character/references:payload -> character/write:payloads', ]) }) diff --git a/src/renderer/lib/starterGraph.ts b/src/renderer/lib/starterGraph.ts index 8b47af7..aaf5451 100644 --- a/src/renderer/lib/starterGraph.ts +++ b/src/renderer/lib/starterGraph.ts @@ -79,8 +79,8 @@ function clearRow(centre: Point, left: number, right: number, height: number): n } /** - * The character chain, pre-wired: Load Assets -> Encode Character -> Compile References -> Write. - * Wired for a reference model; a Krea 2 style character swaps the middle node for Train LoRA + + * The character chain, pre-wired: Load Assets -> Encode -> Verify References -> Compile -> Write. + * Wired for a reference model; a Krea 2 style character swaps the payload node for Train LoRA + * Attach Adapter, which is why the payload step is its own node rather than folded into Encode. */ export async function buildCharacterStarter( @@ -88,34 +88,39 @@ export async function buildCharacterStarter( assetIds: string[] = [], ): Promise { const store = useMoodboardStore.getState() - const y = clearRow(centre, centre.x - 700, centre.x + 700, 340) - const images = await store.addLoader(centre.x - 700, y) + const y = clearRow(centre, centre.x - 780, centre.x + 780, 340) + const images = await store.addLoader(centre.x - 780, y) if (images && assetIds.length > 0) await store.addLoaderAssets(images.id, assetIds) - const encode = images && (await store.addCoreNode('character/encode', centre.x - 340, y)) - const refs = encode && (await store.addCoreNode('character/references', centre.x + 20, y)) - const write = refs && (await store.addCoreNode('character/write', centre.x + 380, y)) - if (!images || !encode || !refs || !write) { + const encode = images && (await store.addCoreNode('character/encode', centre.x - 470, y)) + const verify = encode && (await store.addCoreNode('character/verify-refs', centre.x - 160, y)) + const refs = verify && (await store.addCoreNode('character/references', centre.x + 150, y)) + const write = refs && (await store.addCoreNode('character/write', centre.x + 460, y)) + if (!images || !encode || !verify || !refs || !write) { useGenerationStore .getState() .setError('Could not add the character nodes. Is Inline Core running?') return [] } await store.connect(images.id, encode.id, 'image', 'images') - await store.connect(encode.id, refs.id, 'character', 'character') - await store.connect(encode.id, write.id, 'character', 'character') + await store.connect(encode.id, verify.id, 'character', 'character') + await store.connect(verify.id, refs.id, 'character', 'character') + // Write takes the verified character, never the raw one: a payload node compiles from the doc + // Write hands it, so wiring Encode straight here would save the set nothing checked. + await store.connect(verify.id, write.id, 'character', 'character') await store.connect(refs.id, write.id, 'payload', 'payloads') useUiStore.getState().revealAt(centre.x, y + 170) - return [images.id, encode.id, refs.id, write.id] + return [images.id, encode.id, verify.id, refs.id, write.id] } -/** Load Character -> Edit Character -> Write .char, for changing one that is already saved. */ +/** Load -> Edit -> Verify References -> Write .char, for changing one that is already saved. */ export async function buildCharacterEditChain(file: string, centre: Point): Promise { const store = useMoodboardStore.getState() - const y = clearRow(centre, centre.x - 560, centre.x + 560, 340) - const load = await store.addCoreNode('character/load', centre.x - 560, y) - const edit = load && (await store.addCoreNode('character/edit', centre.x - 180, y)) - const write = edit && (await store.addCoreNode('character/write', centre.x + 200, y)) - if (!load || !edit || !write) { + const y = clearRow(centre, centre.x - 620, centre.x + 620, 340) + const load = await store.addCoreNode('character/load', centre.x - 620, y) + const edit = load && (await store.addCoreNode('character/edit', centre.x - 310, y)) + const verify = edit && (await store.addCoreNode('character/verify-refs', centre.x, y)) + const write = verify && (await store.addCoreNode('character/write', centre.x + 310, y)) + if (!load || !edit || !verify || !write) { useGenerationStore .getState() .setError('Could not add the character nodes. Is Inline Core running?') @@ -127,9 +132,10 @@ export async function buildCharacterEditChain(file: string, centre: Point): Prom false, ) await store.connect(load.id, edit.id, 'character', 'character') - await store.connect(edit.id, write.id, 'character', 'character') + await store.connect(edit.id, verify.id, 'character', 'character') + await store.connect(verify.id, write.id, 'character', 'character') useUiStore.getState().revealAt(centre.x - 180, y + 170) - return [load.id, edit.id, write.id] + return [load.id, edit.id, verify.id, write.id] } /** The training chain, pre-wired: Load Dataset -> Train LoRA -> Graph. Returns [] if any add failed. */ diff --git a/src/renderer/store/moodboardStore.ts b/src/renderer/store/moodboardStore.ts index 891f064..6178a6c 100644 --- a/src/renderer/store/moodboardStore.ts +++ b/src/renderer/store/moodboardStore.ts @@ -185,6 +185,21 @@ async function copyOne( case 'controlSpace': res = await m.addControlSpace(x, y) break + case 'train/dataset': + res = await m.addTrainDataset(x, y) + break + case 'train/caption': + res = await m.addCaption(x, y) + break + case 'train/lora': + res = await m.addTrainer(x, y) + break + case 'train/loss': + res = await m.addLossGraph(x, y) + break + case 'resource': + res = await m.addResource(x, y) + break default: return null } @@ -207,6 +222,19 @@ async function copyOne( patch.data = { ...res.value.data, promptText: data.promptText ?? '' } } else if (item.type === 'loader') { patch.data = { ...res.value.data, assetIds: data.assetIds ?? [] } + } else if (item.type === 'train/lora') { + // Settings only, never `runId`: the copy is a fresh slot, the same rule a core node's takes + // follow. Claiming the original's run would give two nodes one log and one Resume. + patch.data = { ...res.value.data, hyperparams: data.hyperparams ?? {} } + } else if (item.type === 'train/caption') { + patch.data = { + ...res.value.data, + overwrite: data.overwrite ?? false, + captioner: data.captioner ?? '', + } + } else if (item.type === 'train/dataset') { + // Kept, unlike an export: a duplicate lands in the same project, so the dataset row is there. + patch.data = { ...res.value.data, datasetId: data.datasetId ?? null } } else if (item.type === 'controlSpace') { patch.data = { ...res.value.data, diff --git a/src/renderer/views/Moodboard/AddNodeMenu.tsx b/src/renderer/views/Moodboard/AddNodeMenu.tsx index dac1675..e608820 100644 --- a/src/renderer/views/Moodboard/AddNodeMenu.tsx +++ b/src/renderer/views/Moodboard/AddNodeMenu.tsx @@ -19,20 +19,9 @@ import { isExtensionNode, extensionOf } from '@shared/extensions' import { listNodeDefs, groupByOwner } from '@shared/nodes/registry' import { CaptionGlyph, ChartIcon, CpuIcon, LayersIcon, WandIcon } from './nodes/NodeBadge' -/** The node kinds the Add menu can create (Text has its own toolbar tool, so it's not here). */ -export type AddNodeKind = - | 'load' - | 'layer' - | 'preview' - | 'director' - | 'trim' - | 'prompt' - | 'controlSpace' - | 'train/dataset' - | 'train/caption' - | 'train/lora' - | 'train/loss' - | 'resource' +import type { AddNodeKind } from './nodeKinds' + +export type { AddNodeKind } type Tab = 'core' | 'api' diff --git a/src/renderer/views/Moodboard/MoodboardPanel.tsx b/src/renderer/views/Moodboard/MoodboardPanel.tsx index dfb4506..1be16a3 100644 --- a/src/renderer/views/Moodboard/MoodboardPanel.tsx +++ b/src/renderer/views/Moodboard/MoodboardPanel.tsx @@ -80,6 +80,7 @@ import { MissingModelsDialog } from '../Models/MissingModelsDialog' import { checkGraphModels } from '../../lib/checkModels' import { CanvasToolbar } from './CanvasToolbar' import { AddNodeMenu, type AddNodeKind } from './AddNodeMenu' +import { BY_ID_TYPES } from './nodeKinds' import { FirstRunHints } from './GettingStarted/FirstRunHints' import { StarterCards } from './GettingStarted/StarterCards' import { useStarterGraph } from './GettingStarted/useStarterGraph' @@ -127,7 +128,6 @@ function writeViewport(id: string, v: { x: number; y: number; zoom: number }): v } /** Item types the Training category adds; they map straight through to their own components. */ -const TRAINING_TYPES = new Set(['train/dataset', 'train/caption', 'train/lora', 'train/loss']) const nodeTypes: NodeTypes = { image: ImageNode, @@ -1371,9 +1371,7 @@ function itemToNode( if (item.type === 'text') { return { ...common, type: 'text', data: { text: item.data.text ?? FALLBACK_TEXT } } } - // Training nodes read everything from the board context by id, like the Core node does. Without - // this they fall through to the asset branch below and render as a blank image. - if (TRAINING_TYPES.has(item.type)) { + if (BY_ID_TYPES.has(item.type)) { return { ...common, type: item.type, data: { itemId: item.id } } } const asset = item.assetId ? assetsById.get(item.assetId) : undefined diff --git a/src/renderer/views/Moodboard/nodeKinds.ts b/src/renderer/views/Moodboard/nodeKinds.ts new file mode 100644 index 0000000..bbf953a --- /dev/null +++ b/src/renderer/views/Moodboard/nodeKinds.ts @@ -0,0 +1,43 @@ +/** The Add-menu node kinds, and how each one reaches a renderer. */ + +export const ADD_NODE_KINDS = [ + 'load', + 'layer', + 'preview', + 'director', + 'trim', + 'prompt', + 'controlSpace', + 'train/dataset', + 'train/caption', + 'train/lora', + 'train/loss', + 'resource', +] as const + +/** The node kinds the Add menu can create (Text has its own toolbar tool, so it's not here). */ +export type AddNodeKind = (typeof ADD_NODE_KINDS)[number] + +/** Kinds `nodeFor` maps with a branch of their own. */ +export const EXPLICIT_RENDER_KINDS: ReadonlySet = new Set([ + 'load', + 'layer', + 'preview', + 'director', + 'trim', + 'prompt', + 'controlSpace', +]) + +/** + * Kinds whose node reads the board by item id rather than an asset. Named for the rule, not for the + * Training menu: `resource` sat outside a set called TRAINING_TYPES and so fell through to the + * asset branch, rendering an assetless blank that read as the node never being added at all. + */ +export const BY_ID_TYPES: ReadonlySet = new Set([ + 'train/dataset', + 'train/caption', + 'train/lora', + 'train/loss', + 'resource', +]) diff --git a/src/renderer/views/Moodboard/nodeRendering.test.ts b/src/renderer/views/Moodboard/nodeRendering.test.ts new file mode 100644 index 0000000..792c506 --- /dev/null +++ b/src/renderer/views/Moodboard/nodeRendering.test.ts @@ -0,0 +1,18 @@ +/** Every node the add menu offers must actually render; a missing mapping looks like a dead menu. */ +import { describe, expect, it } from 'vitest' +import { ADD_NODE_KINDS, BY_ID_TYPES, EXPLICIT_RENDER_KINDS } from './nodeKinds' + +describe('add-menu kinds all reach a renderer', () => { + it('classifies every kind the menu can create', () => { + const orphans = ADD_NODE_KINDS.filter( + (kind) => !EXPLICIT_RENDER_KINDS.has(kind) && !BY_ID_TYPES.has(kind), + ) + // An unclassified kind falls through nodeFor to the asset branch and renders an assetless + // blank, which on the canvas is indistinguishable from the add having done nothing. + expect(orphans).toEqual([]) + }) + + it('includes Resources, which shipped missing and read as an add that did nothing', () => { + expect(BY_ID_TYPES.has('resource')).toBe(true) + }) +}) diff --git a/src/shared/types.ts b/src/shared/types.ts index edb7f5f..0baf45c 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -300,6 +300,8 @@ export interface MoodboardItemData { datasetId?: string | null /** `caption` node: re-caption images that already have a caption. */ overwrite?: boolean + /** `caption` node: which captioner to run. Read by `graph_build`, so it belongs on the wire. */ + captioner?: string /** Training nodes: the training run they're bound to. Persisted so the node * rebinds after a reload and can still offer Resume on an interrupted run. */ runId?: string | null @@ -344,7 +346,12 @@ export interface CharacterSummary { file: string charId?: string name: string + /** The references the user curated. Harvested takes are counted separately on purpose. */ refs: number + /** Approved takes taken into the pool. Absent or 0 on a character that never harvested one. */ + harvested?: number + /** Reference positions whose face agrees least with the rest, by `character/verify-refs`. */ + flagged?: number[] createdAt?: number modifiedAt?: number description?: string From d5e2b9c6b0c123f9680ce3010fb68a7730de646f Mon Sep 17 00:00:00 2001 From: ashish-aesthisia Date: Tue, 25 Aug 2026 09:57:22 +0000 Subject: [PATCH 3/3] feat: minimax h3 char support --- core/pyproject.toml | 2 +- .../src/inline_core/models/minimaxh3/nvfp4.py | 71 ++++++++++++++----- .../inline_core/models/minimaxh3/pipeline.py | 54 +++++++++++--- .../inline_core/models/minimaxh3/runner.py | 28 ++++++-- .../inline_core/models/pipeline_runtime.py | 10 +++ core/tests/test_h3_characters.py | 43 +++++++++++ core/tests/test_minimaxh3_nvfp4.py | 66 +++++++++++++++++ core/uv.lock | 2 +- package-lock.json | 4 +- package.json | 2 +- packages/frontend/pyproject.toml | 2 +- 11 files changed, 247 insertions(+), 37 deletions(-) diff --git a/core/pyproject.toml b/core/pyproject.toml index dea7509..b77ab45 100644 --- a/core/pyproject.toml +++ b/core/pyproject.toml @@ -1,7 +1,7 @@ [project] # PyPI name; the import package is `inline_core` (src/inline_core). name = "inline-core" -version = "1.3.1" +version = "1.3.11" description = "The generation engine behind Inline Studio." readme = "README.md" license = "GPL-3.0-or-later" diff --git a/core/src/inline_core/models/minimaxh3/nvfp4.py b/core/src/inline_core/models/minimaxh3/nvfp4.py index 8982ff2..751fcf6 100644 --- a/core/src/inline_core/models/minimaxh3/nvfp4.py +++ b/core/src/inline_core/models/minimaxh3/nvfp4.py @@ -33,10 +33,16 @@ def unpack_fp4(packed: torch.Tensor, table: torch.Tensor) -> torch.Tensor: """``[..., n/2]`` uint8 to ``[..., n]`` values. High nibble first, then low, interleaved. The other three orderings score ~0 against the reference, so this is not a coin flip. + + Indexing needs int64, which is eight bytes per weight against the packed half-byte - so the + shift and the mask are taken one at a time and freed, never held together. """ - codes = packed.to(torch.int64) - pairs = torch.stack((table[codes >> 4], table[codes & 0xF]), dim=-1) - return pairs.reshape(*packed.shape[:-1], packed.shape[-1] * 2) + out = torch.empty( + (*packed.shape[:-1], packed.shape[-1] * 2), dtype=table.dtype, device=packed.device + ) + out[..., 0::2] = table[(packed >> 4).long()] + out[..., 1::2] = table[(packed & 0xF).long()] + return out def from_blocked(stored: torch.Tensor, rows: int, cols: int) -> torch.Tensor: @@ -77,14 +83,36 @@ def dequantize( dtype: torch.dtype = torch.bfloat16, ) -> torch.Tensor: """One NVFP4 weight back to ``dtype``: ``fp4 * block_scale * global_scale``.""" + rows, cols = packed.shape[0], packed.shape[1] * 2 + scales = from_blocked(block_scale.to(torch.float32), rows, cols // BLOCK) + out = torch.empty((rows, cols), dtype=dtype, device=packed.device) + for start, stop in _row_chunks(rows, cols): + out[start:stop] = _dequantize_rows( + packed[start:stop], scales[start:stop], float(global_scale), dtype + ) + # Padded up to a multiple of 16 on the way in, so trim back to the layer's real shape. + return out[:out_features, :in_features] + + +#: Roughly how much scratch one chunk may take. The intermediates are fp32 and int64 while the +#: weight is half a byte, so a whole 25600x5120 layer at once wants gigabytes to produce megabytes. +_CHUNK_BYTES = 32 * 1024**2 + + +def _row_chunks(rows: int, cols: int) -> list[tuple[int, int]]: + step = max(BLOCK, min(rows, _CHUNK_BYTES // max(1, cols * 4))) + return [(start, min(start + step, rows)) for start in range(0, rows, step)] + + +def _dequantize_rows( + packed: torch.Tensor, scales: torch.Tensor, global_scale: float, dtype: torch.dtype +) -> torch.Tensor: + """One row block, in fp32 so an fp8 scale times a 4-bit value keeps its bits until the cast.""" table = e2m1_table(packed.device, torch.float32) values = unpack_fp4(packed, table) - rows, blocks = values.shape[0], values.shape[1] // BLOCK - scales = from_blocked(block_scale.to(torch.float32), rows, blocks) - weight = (values.reshape(rows, blocks, BLOCK) * scales.unsqueeze(-1)).reshape(values.shape) - weight = weight * float(global_scale) - # Padded up to a multiple of 16 on the way in, so trim back to the layer's real shape. - return weight[:out_features, :in_features].to(dtype) + rows, cols = values.shape + values = values.reshape(rows, cols // BLOCK, BLOCK) * scales.unsqueeze(-1) + return (values.reshape(rows, cols) * global_scale).to(dtype) class NVFP4Linear(nn.Module): @@ -132,15 +160,22 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: smooth = self.pre_quant_scale if smooth is not None: x = x * smooth.to(x.dtype) - weight = dequantize( - self.weight, - self.weight_scale, - self.weight_scale_2, - out_features=self.out_features, - in_features=self.in_features, - dtype=x.dtype, - ) - return torch.nn.functional.linear(x, weight, self.bias) + rows, cols = self.weight.shape[0], self.weight.shape[1] * 2 + scales = from_blocked(self.weight_scale.to(torch.float32), rows, cols // BLOCK) + global_scale = float(self.weight_scale_2) + # Row-chunked so the whole weight never exists: the conditioner runs on one short prompt, so + # the output is a few kilobytes while the dequantised weight would be hundreds of megabytes. + parts: list[torch.Tensor] = [] + for start, stop in _row_chunks(rows, cols): + if start >= self.out_features: + break + block = _dequantize_rows( + self.weight[start:stop], scales[start:stop], global_scale, x.dtype + ) + block = block[: self.out_features - start, : self.in_features] + bias = None if self.bias is None else self.bias[start : start + block.shape[0]] + parts.append(torch.nn.functional.linear(x, block, bias)) + return torch.cat(parts, dim=-1) def extra_repr(self) -> str: return f"in_features={self.in_features}, out_features={self.out_features}, nvfp4" diff --git a/core/src/inline_core/models/minimaxh3/pipeline.py b/core/src/inline_core/models/minimaxh3/pipeline.py index e2dfe8e..9dd9929 100644 --- a/core/src/inline_core/models/minimaxh3/pipeline.py +++ b/core/src/inline_core/models/minimaxh3/pipeline.py @@ -388,7 +388,11 @@ def _build( # the card is full, and badly wrong when it is not: at 960x544 it turned a 6 minute render into # 32. Staged, the conditioner is on the CPU by decode time, so if the measured free VRAM covers # the VAE plus a working margin it stays resident instead. - vae_resident = staged and _vae_fits(video_vae, placement.device) + vae_resident = staged and _vae_fits( + video_vae, + placement.device, + _denoiser_card_bytes(transformer, recipe, resident_blocks), + ) if vae_resident: recipe = _replace(recipe, vae_offload=None) @@ -428,24 +432,51 @@ def _build( return pipe -#: Left free beside a resident VAE: the denoiser is already placed, so this only has to cover the -#: decode's own activations, which are the largest single allocation of the render. +#: Left free beside a resident VAE, to cover the decode's own activations - the largest single +#: allocation of the render. _VAE_RESIDENT_MARGIN_GB = 12.0 -def _vae_fits(path: Path, device: Any) -> bool: +def _denoiser_card_bytes(transformer: Any, recipe: Any, resident_blocks: int) -> int: + """What the denoiser will claim on the card once ``apply_offload`` places it. + + It is loaded but still on the CPU when the VAE decision is taken, so the card reads empty and + every later placement is invisible to anything measuring free VRAM at that moment. + """ + from ..offload import block_stack + + total = sum(t.numel() * t.element_size() for t in transformer.parameters()) + total += sum(t.numel() * t.element_size() for t in transformer.buffers()) + if recipe.denoiser_offload is None: + return total + blocks = len(list(block_stack(transformer))) + if not blocks: + return total + # Streaming leaves only the placed head blocks resident; the rest arrives and leaves per step. + return int(total * min(1.0, max(0, resident_blocks) / blocks)) + + +def _vae_fits(path: Path, device: Any, denoiser_bytes: int = 0) -> bool: """Whether the video VAE can stay on the card rather than streaming leaf by leaf. fp32 doubles what the file weighs, and that is what actually has to fit. + + ``denoiser_bytes`` is what the denoiser will take once it is placed, which happens *after* this + runs: measured free VRAM here is an empty card, and reserving against it put a 10.4 GB VAE and + a 33 GB denoiser onto 44 GB. The same correction `_plan_residency` already makes for host RAM. """ - free = rt.free_vram_bytes(device) - if not free: + free = rt.free_vram_bytes(device) - denoiser_bytes + if free <= 0: + logger.info( + "MiniMax H3 video VAE: streamed leaf by leaf (the denoiser claims the card first)" + ) return False needed = path.stat().st_size * 2 + int(_VAE_RESIDENT_MARGIN_GB * 1e9) fits = free > needed logger.info( - "MiniMax H3 video VAE: %s (%.1f GB free, needs %.1f GB with margin)", - "resident" if fits else "streamed leaf by leaf", free / 1e9, needed / 1e9, + "MiniMax H3 video VAE: %s (%.1f GB free after the denoiser's %.1f GB, needs %.1f GB)", + "resident" if fits else "streamed leaf by leaf", + free / 1e9, denoiser_bytes / 1e9, needed / 1e9, ) return fits @@ -640,6 +671,8 @@ def _load_nvfp4_encoder(path: Path, dtype: Any) -> Any: config = AutoConfig.from_pretrained(str(ensure_assets(ASSETS_ARCH) / "text_encoder")) text_config = getattr(config, "text_config", config) + full_depth = int(text_config.num_hidden_layers) + # Built at the file's depth so only the layers it carries are instantiated. text_config.num_hidden_layers = layers with init_empty_weights(): model = Qwen3VLForConditionalGeneration._from_config(config, dtype=dtype) @@ -648,6 +681,11 @@ def _load_nvfp4_encoder(path: Path, dtype: Any) -> Any: # index becomes the normed one, so keeping a real norm here would change the conditioning. if "model.norm.weight" not in keys: model.model.language_model.norm = torch.nn.Identity() + # Restored to the architecture this build was cut from. `encoders.py` refuses a conditioner + # whose config says 50 layers, because a naive truncation makes `hidden_states[50]` post-norm - + # true, and defeated above by the Identity. Transformers reads this only when constructing the + # stack, and iterates the real module list at inference, so the 50 layers still run. + text_config.num_hidden_layers = full_depth with safe_open(str(path), framework="pt") as handle: formats = _packed_formats(handle, keys) diff --git a/core/src/inline_core/models/minimaxh3/runner.py b/core/src/inline_core/models/minimaxh3/runner.py index b2314ea..9dfad0e 100644 --- a/core/src/inline_core/models/minimaxh3/runner.py +++ b/core/src/inline_core/models/minimaxh3/runner.py @@ -447,8 +447,18 @@ def on_step(done: int, total: int) -> None: rt.free_vram() raise except torch.cuda.OutOfMemoryError as error: - rt.free_vram() - raise ComponentError(_oom(request)) from error + # Evicted, not merely emptied: `free_vram` releases unused blocks and leaves the + # pipeline resident, so the failed run kept ~43 GB and every retry started from a full + # card. That reads as the same error forever, whatever the user changes. + held = rt.own_vram_bytes() + # Dropped before the clear, not after: `raise ... from error` keeps the traceback, the + # traceback keeps this frame, and this frame's locals still name the pipeline - so + # evicting the cache alone leaves it alive and the card still full. + pipe = None + call = {} + rt.PIPELINES.clear() + logger.info("MiniMax H3 released %.1f GB after an out-of-memory run", held / 1e9) + raise ComponentError(_oom(request, held=held)) from error except MemoryError as error: rt.free_vram() raise ComponentError(_oom(request, host=True)) from error @@ -524,12 +534,14 @@ def _reference_tokens(request: Request) -> tuple[int, int]: #: Below this, another process on the card is noise; above it, it is the whole story. _FOREIGN_VRAM_FLOOR = 2 * 1024**3 +#: What counts as "the last run was still holding the card" rather than a genuinely tight fit. +_HELD_VRAM_FLOOR = 8 * 1024**3 -def _oom(request: Request, *, host: bool = False) -> str: +def _oom(request: Request, *, host: bool = False, held: int = 0) -> str: where = "System RAM" if host else "VRAM" - # Asked first, because when it is true nothing on this node is the cause and every other hint - # below sends the user to change a setting that was never the problem. + # Asked first, because when either is true nothing on this node is the cause and every other + # hint below sends the user to change a setting that was never the problem. foreign = 0 if host else rt.foreign_vram_bytes() if foreign >= _FOREIGN_VRAM_FLOOR: return ( @@ -537,6 +549,12 @@ def _oom(request: Request, *, host: bool = False) -> str: "process - a training run, another render, or another app. Wait for it to finish or " "stop it, then run this again. Nothing on this node will free that memory." ) + if not host and held >= _HELD_VRAM_FLOOR: + return ( + f"{where} ran out with {held / 1024**3:.1f} GB already held by this render. That has " + "now been released, so run it again - the retry starts from an empty card. If it " + "fails the same way twice in a row, the model genuinely does not fit these settings." + ) canvas = ( f"{where} ran out at {request.width}x{request.height} for {request.seconds:.1f}s. " "Canvas is the biggest lever: 960x544 needs far less than 1344x768 and renders about " diff --git a/core/src/inline_core/models/pipeline_runtime.py b/core/src/inline_core/models/pipeline_runtime.py index 5a55053..98338d7 100644 --- a/core/src/inline_core/models/pipeline_runtime.py +++ b/core/src/inline_core/models/pipeline_runtime.py @@ -489,6 +489,16 @@ def free_vram() -> None: pass +def own_vram_bytes() -> int: + """What this process holds on the card, weights included - not just the allocator's cache.""" + try: + if not torch.cuda.is_available(): + return 0 + return int(torch.cuda.memory_reserved()) + except Exception: # noqa: BLE001 + return 0 + + def foreign_vram_bytes(device: Any = None) -> int: """VRAM held on this card by anyone but us - another render, a training run, another app. diff --git a/core/tests/test_h3_characters.py b/core/tests/test_h3_characters.py index 8ea5305..3eef5df 100644 --- a/core/tests/test_h3_characters.py +++ b/core/tests/test_h3_characters.py @@ -379,3 +379,46 @@ def test_a_card_held_by_another_process_is_named_before_anything_on_this_node(mo # A host-RAM exhaustion is never explained by another process's VRAM. monkeypatch.setattr(rt, "foreign_vram_bytes", lambda *a, **k: 29 * 1024**3) assert "another process" not in _oom(request, host=True) + + +def test_a_failed_run_releases_the_card_instead_of_poisoning_the_next(monkeypatch) -> None: + """`free_vram` drops unused blocks and leaves the pipeline resident, so one OOM left ~43 GB + held and every retry started from a full card - the same error forever, whatever was changed. + A VRAM failure has to evict, and say so, rather than blame the character again.""" + from inline_core.models import pipeline_runtime as rt + from inline_core.models.minimaxh3.runner import Request, _oom + from inline_core.models.references import ReferenceKind + + monkeypatch.setattr(rt, "foreign_vram_bytes", lambda *a, **k: 0) + refs = tuple(type("R", (), {"kind": ReferenceKind.IMAGE})() for _ in range(5)) + request = Request( + prompt="", num_frames=141, width=544, height=768, num_inference_steps=50, + seed=1, partition="ref2va", references=refs, + ) + + held = _oom(request, held=int(43.5 * 1024**3)) + assert "43.5 GB already held by this render" in held + assert "released" in held and "run it again" in held + assert "Resized Reference Resolution" not in held, "do not blame the character" + + # An empty card is the case where the reference hint is the useful one. + assert "Resized Reference Resolution" in _oom(request, held=0) + # And host exhaustion is never explained by VRAM the render was holding. + assert "already held" not in _oom(request, host=True, held=int(43.5 * 1024**3)) + + +def test_the_runner_clears_the_pipeline_cache_on_a_vram_failure() -> None: + """Emptying the allocator is not enough: the cached pipeline pins the weights themselves.""" + import inspect + + from inline_core.models.minimaxh3 import runner + + source = inspect.getsource(runner.MiniMaxH3Runner.run) + handler = source[source.index("except torch.cuda.OutOfMemoryError") :] + handler = handler.split("except MemoryError")[0] + assert "PIPELINES.clear()" in handler + # And the frame has to stop naming the pipeline first: `raise ... from error` keeps the + # traceback, which keeps these locals, so evicting the cache alone frees nothing. Measured: + # the card still held 43.5 GB after a failed run that did call clear(). + assert "pipe = None" in handler + assert handler.index("pipe = None") < handler.index("PIPELINES.clear()") diff --git a/core/tests/test_minimaxh3_nvfp4.py b/core/tests/test_minimaxh3_nvfp4.py index c869b34..9362ba4 100644 --- a/core/tests/test_minimaxh3_nvfp4.py +++ b/core/tests/test_minimaxh3_nvfp4.py @@ -122,3 +122,69 @@ def test_awq_smoothing_is_applied_to_the_input() -> None: got = layer(x) want = torch.nn.functional.linear(x, weight) assert torch.nn.functional.cosine_similarity(got.flatten(), want.flatten(), dim=0) > 0.99 + + +def test_the_build_satisfies_the_vendored_depth_guard() -> None: + """`encoders.py` refuses a conditioner whose config reports 50 layers, because a stack naively + truncated there makes `hidden_states[50]` post-norm. This build carries exactly 50 and defeats + that by swapping the trailing norm for Identity, so the state stays pre-norm - and the config + has to report the depth it was cut from or the guard rejects a correct encoder.""" + import pathlib + + from inline_core.models.minimaxh3.vendor.packing import MINIMAX_H3_TEXT_ENCODER_LAYER + + weights = pathlib.Path("models/text_encoders/qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors") + if not weights.is_file(): + pytest.skip("nvfp4 conditioner not present") + + from inline_core.models.minimaxh3.pipeline import _load_nvfp4_encoder + + model = _load_nvfp4_encoder(weights, torch.bfloat16) + depth = model.config.text_config.num_hidden_layers + assert depth > MINIMAX_H3_TEXT_ENCODER_LAYER, "the vendored guard would reject this" + # The stack really is the file's, whatever the config says: transformers reads the config only + # when constructing and iterates the module list at inference. + assert len(model.model.language_model.layers) == MINIMAX_H3_TEXT_ENCODER_LAYER + assert isinstance(model.model.language_model.norm, torch.nn.Identity) + assert isinstance(model.lm_head, torch.nn.Identity), "this build ships no head" + + +def test_the_vae_budget_counts_the_denoiser_that_lands_after_it(monkeypatch) -> None: + """`_vae_fits` runs three lines before `apply_offload`, so the card it measures is empty and + every later placement is invisible. Reserving against that put a 10.4 GB fp32 VAE beside a + 33 GB denoiser on a 44 GB card, and the render peaked at 43.4 GB and died.""" + import pathlib + + from inline_core.models import pipeline_runtime as rt + from inline_core.models.minimaxh3 import pipeline as pl + + vae = pathlib.Path("models/vae/minimax_h3_video_vae_fp16.safetensors") + if not vae.is_file(): + pytest.skip("video VAE not present") + + monkeypatch.setattr(rt, "free_vram_bytes", lambda *a, **k: int(46.6e9)) + assert pl._vae_fits(vae, None, 0), "an empty card looks like room, which is the old bug" + assert not pl._vae_fits(vae, None, int(33e9)), "counting the denoiser has to flip it" + + # The fast path survives where the card genuinely has room: leaf offload turned a 6 minute + # render into 32, so this must not become "always stream". + monkeypatch.setattr(rt, "free_vram_bytes", lambda *a, **k: int(80e9)) + assert pl._vae_fits(vae, None, int(33e9)) + + # A denoiser larger than the whole card is a decision, not a crash. + monkeypatch.setattr(rt, "free_vram_bytes", lambda *a, **k: int(40e9)) + assert not pl._vae_fits(vae, None, int(80e9)) + + +def test_a_resident_denoiser_is_sized_whole() -> None: + """With no offload every byte of it lands on the card, so that is what the VAE must budget.""" + from dataclasses import dataclass + + from inline_core.models.minimaxh3.pipeline import _denoiser_card_bytes + + @dataclass + class _Recipe: + denoiser_offload: object = None + + model = torch.nn.Linear(64, 32, bias=False, dtype=torch.float32) + assert _denoiser_card_bytes(model, _Recipe(), 0) == 64 * 32 * 4 diff --git a/core/uv.lock b/core/uv.lock index 484b3f8..ae44946 100644 --- a/core/uv.lock +++ b/core/uv.lock @@ -610,7 +610,7 @@ wheels = [ [[package]] name = "inline-core" -version = "1.3.0" +version = "1.3.11" source = { editable = "." } dependencies = [ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, diff --git a/package-lock.json b/package-lock.json index 3d9bda5..3748299 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "inline-studio", - "version": "1.3.1", + "version": "1.3.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "inline-studio", - "version": "1.3.1", + "version": "1.3.11", "license": "GPL-3.0-or-later", "dependencies": { "@react-three/drei": "^10.7.7", diff --git a/package.json b/package.json index f64dd52..7b47c9a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "inline-studio", - "version": "1.3.1", + "version": "1.3.11", "description": "AI filmmaking on a node canvas. Generate locally on your own GPU and train your own LoRAs on the same canvas, with the built-in Inline Core engine and hosted models. Every render is kept as a versioned take.", "keywords": [ "ai-filmmaking", diff --git a/packages/frontend/pyproject.toml b/packages/frontend/pyproject.toml index 667fc5a..4a6aab3 100644 --- a/packages/frontend/pyproject.toml +++ b/packages/frontend/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "inline-studio-frontend" -version = "1.3.1" +version = "1.3.11" description = "Prebuilt Inline Studio web UI (SPA), served by Inline Core. Mirrors comfyui-frontend-package." requires-python = ">=3.9" readme = "README.md"