feat(qualification): add MiniMax-H3 ACC and PERF

benchmarks/performance/README.md

@@ -20,7 +20,7 @@ The suite contains one row for every release-relevant single-process model
 profile marked `ready` in the benchmark catalog. Profiles whose names contain an
 `l0` segment are shorter PR-smoke duplicates and are deliberately excluded.
 Other temporary omissions must be named under `excluded_profiles` with a reason.
-The suite currently has 107 model-profile comparisons across 77 families and 78
+The suite currently has 111 model-profile comparisons across 79 families and 81
 `(family, operation)` contracts because some families expose multiple profiles
 and `eagle_vlm` exposes both `embed` and `rerank`. Catalog profiles marked
 `distributed` require their own multi-process launch and are not silently
@@ -247,10 +247,16 @@ Reference-specific upstream checkout paths remain process environment inputs:
 ```text
 TRTMC_ELF_REFERENCE_REPO
 TRTMC_LANCE_REFERENCE_REPO
+TRTMC_MINIMAX_H3_DIFFUSERS_REPO
+TRTMC_MINIMAX_H3_TRANSFORMERS_REPO
 TRTMC_SANA_WM_REFERENCE_REPO
 PERSONAPLEX_OFFICIAL_REPO
 ```
 
+Task-reference baselines declare the mapping from adapter option to environment
+variable with `baseline.adapter_environment`. This keeps model checkout names in
+the suite entry while the shared runner only resolves the declared mapping.
+
 CI should prebuild the selected Python profiles and set
 `TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=1`. Dependency installation is outside the
 measured campaign.

benchmarks/performance/baselines/task_reference.py

@@ -87,14 +87,15 @@
 class Session:
     """One loaded reference model and its repeatable timed operation."""
 
-    invoke: Callable[[], Mapping[str, Any]]
+    invoke: Callable[[], Any]
     resolved_revision: str
     framework: str
     timing_scope: str = "task-model-call-wall"
     input_preparation_included: bool = False
     asset_loading_included: bool = False
     reference_dependencies: Mapping[str, str] | None = None
     reference_source: Mapping[str, str] | None = None
+    summarize: Callable[[Any], Mapping[str, Any]] | None = None
 
 
 def build_parser() -> argparse.ArgumentParser:
@@ -1359,8 +1360,13 @@ def _diffusion_pipeline(
         "wan_t2v": ("WanPipeline", "DiffusionPipeline"),
         "wan2_2_ti2v": ("WanPipeline", "DiffusionPipeline"),
         "z_image": ("ZImagePipeline", "DiffusionPipeline"),
-    }[arguments.family]
+    }.get(arguments.family, ())
     configured_classes = options.get("pipeline_classes")
+    pipeline_load_mode = str(options.get("pipeline_load_mode", "from_pretrained"))
+    if pipeline_load_mode not in {"from_pretrained", "modular_components"}:
+        raise ValueError(
+            "pipeline_load_mode must be 'from_pretrained' or 'modular_components'"
+        )
     if configured_classes is None:
         classes = default_classes
     elif (
@@ -1380,6 +1386,37 @@ def _diffusion_pipeline(
             _cached_snapshot_path(model_id, requested_revision, "model_index.json")
             or model_source
         )
+    if pipeline_load_mode == "modular_components":
+        manager_class = getattr(diffusers, "ComponentsManager", None)
+        pipeline_class = getattr(diffusers, "ModularPipeline", None)
+        if manager_class is None or pipeline_class is None:
+            raise RuntimeError("Diffusers does not provide the modular pipeline API")
+        load_options = {
+            "trust_remote_code": bool(
+                options.get("trust_remote_code", arguments.trust_remote_code)
+            ),
+            "local_files_only": arguments.local_files_only,
+        }
+        if requested_revision and model_source == model_id:
+            load_options["revision"] = requested_revision
+        pipeline = pipeline_class.from_pretrained(
+            model_source,
+            components_manager=manager_class(),
+            **load_options,
+        )
+        component_options = {
+            "dtype": _torch_dtype(torch_module, arguments.precision),
+            "pretrained_model_name_or_path": model_source,
+            "local_files_only": arguments.local_files_only,
+        }
+        if requested_revision and model_source == model_id:
+            component_options["revision"] = requested_revision
+        pipeline.load_components(**component_options)
+        return pipeline
+    if not classes:
+        raise ValueError(
+            f"pipeline_classes must be configured for Diffusers family {arguments.family!r}"
+        )
     errors = []
     for name in classes:
         pipeline_class = getattr(diffusers, name, None)
@@ -1443,6 +1480,48 @@ def _load_diffusers(
     request: Mapping[str, Any],
     options: Mapping[str, Any],
 ) -> Session:
+    diffusers_revision = ""
+    transformers_revision = ""
+    transformers_repo = str(options.get("transformers_repo", "") or "")
+    if bool(options.get("require_pinned_transformers_source", False)):
+        expected_revision = str(options.get("transformers_compat_revision", "") or "")
+        transformers_revision = _pinned_checkout_revision(
+            transformers_repo,
+            expected_revision,
+            repository="pinned Transformers reference",
+        )
+        source_root = Path(transformers_repo).resolve() / "src"
+        entrypoint = source_root / "transformers" / "__init__.py"
+        if not entrypoint.is_file():
+            raise ValueError(f"pinned Transformers checkout is incomplete: {entrypoint}")
+        imported = sys.modules.get("transformers")
+        imported_path = Path(str(getattr(imported, "__file__", "") or ""))
+        if imported is not None and source_root not in imported_path.parents:
+            raise ValueError(
+                "Transformers was imported before the pinned source was activated"
+            )
+        if str(source_root) not in sys.path:
+            sys.path.insert(0, str(source_root))
+    diffusers_repo = str(options.get("diffusers_repo", "") or "")
+    if bool(options.get("require_pinned_diffusers_source", False)):
+        expected_revision = str(options.get("diffusers_revision", "") or "")
+        diffusers_revision = _pinned_checkout_revision(
+            diffusers_repo,
+            expected_revision,
+            repository="pinned Diffusers reference",
+        )
+        source_root = Path(diffusers_repo).resolve() / "src"
+        entrypoint = source_root / "diffusers" / "__init__.py"
+        if not entrypoint.is_file():
+            raise ValueError(f"pinned Diffusers checkout is incomplete: {entrypoint}")
+        imported = sys.modules.get("diffusers")
+        imported_path = Path(str(getattr(imported, "__file__", "") or ""))
+        if imported is not None and source_root not in imported_path.parents:
+            raise ValueError(
+                "Diffusers was imported before the pinned source was activated"
+            )
+        if str(source_root) not in sys.path:
+            sys.path.insert(0, str(source_root))
     import inspect
     import torch
     from PIL import Image
@@ -1517,6 +1596,15 @@ def _load_diffusers(
     if arguments.family == "qwen_image" and cfg_scale >= 0:
         values["true_cfg_scale"] = cfg_scale
     values["output_type"] = "np"
+    output_fields = options.get("output_fields")
+    if output_fields is not None:
+        if (
+            not isinstance(output_fields, list)
+            or not output_fields
+            or any(not isinstance(name, str) or not name for name in output_fields)
+        ):
+            raise ValueError("output_fields must be a non-empty list of names")
+        values["output"] = list(output_fields)
     image_path = str(request.get("image_path", "") or "")
     if image_path and ("image" in accepted or accepts_extra):
         values["image"] = Image.open(_asset_path(arguments, request, "image_path")).convert("RGB")
@@ -1541,14 +1629,19 @@ def _load_diffusers(
     else:
         seeds = seed
     if "generator" in accepted or accepts_extra:
+        generator_device = str(options.get("generator_device", "cuda"))
+        if generator_device not in {"cpu", "cuda"}:
+            raise ValueError("generator_device must be cpu or cuda")
         if isinstance(seeds, list):
             call_values["generator"] = [
-                torch.Generator("cuda").manual_seed(value) for value in seeds
+                torch.Generator(generator_device).manual_seed(value) for value in seeds
             ]
         else:
-            call_values["generator"] = torch.Generator("cuda").manual_seed(seeds)
+            call_values["generator"] = torch.Generator(generator_device).manual_seed(seeds)
 
-    def invoke() -> Mapping[str, Any]:
+    media_type = str(request.get("media_type", "image"))
+
+    def invoke() -> Any:
         if "generator" in call_values:
             generators = call_values["generator"]
             if isinstance(generators, list):
@@ -1560,8 +1653,12 @@ def invoke() -> Mapping[str, Any]:
         media = getattr(result, "images", None)
         if media is None:
             media = getattr(result, "frames", None)
-        media_type = str(request.get("media_type", "image"))
-        return _media_summary(media, media_type)
+        if media is None and isinstance(result, Mapping):
+            for name in output_fields or ("videos", "images", "frames"):
+                media = result.get(name)
+                if media is not None:
+                    break
+        return media
 
     requested_revision = str(
         options.get("model_revision", getattr(arguments, "revision", None) or "")
@@ -1572,6 +1669,13 @@ def invoke() -> Mapping[str, Any]:
         else _resolved_revision(arguments, getattr(pipeline, "transformer", pipeline))
     )
     reference_model = str(options.get("model_id", getattr(arguments, "model", "unresolved")))
+    dependencies = None
+    if diffusers_revision:
+        dependencies = {
+            "https://github.com/huggingface/diffusers.git": diffusers_revision,
+        }
+        if transformers_revision:
+            dependencies["https://github.com/huggingface/transformers.git"] = transformers_revision
     return Session(
         invoke,
         revision,
@@ -1582,6 +1686,8 @@ def invoke() -> Mapping[str, Any]:
             "repository": f"https://huggingface.co/{reference_model}",
             "revision": revision,
         },
+        reference_dependencies=dependencies,
+        summarize=lambda media: _media_summary(media, media_type),
     )
 
 
@@ -2305,7 +2411,7 @@ def _synchronize() -> None:
 
 
 def _measure(session: Session, warmup: int, iterations: int) -> tuple[list[float], dict[str, Any]]:
-    output: Mapping[str, Any] = {}
+    output: Any = {}
     for _ in range(warmup):
         output = session.invoke()
         _synchronize()
@@ -2316,7 +2422,8 @@ def _measure(session: Session, warmup: int, iterations: int) -> tuple[list[float
         output = session.invoke()
         _synchronize()
         samples.append((time.perf_counter() - started) * 1000.0)
-    return samples, dict(output)
+    summary = session.summarize(output) if session.summarize is not None else output
+    return samples, dict(summary)
 
 
 def _run_elf(

benchmarks/performance/release.yaml

@@ -30,11 +30,6 @@ excluded_profiles:
     reason: *lfm2_performance_exclusion
   - model: lfm2-700m
     reason: *lfm2_performance_exclusion
-  - model: minimax-h3-768p
-    reason: >-
-      The pinned Diffusers reference for MiniMax-H3 has not yet been integrated
-      into the release performance runner.
-
 entries:
   - id: albert.encode
     family: albert
@@ -524,6 +519,30 @@ entries:
       mode: torch-compile
       compile_scope: model.forward
       output_token_policy: strip-start
+  - id: minimax_h3.generate_image
+    family: minimax_h3
+    operation: generate_image
+    model: minimax-h3-768p
+    workload:
+      testcase: minimax-h3-768p
+    baseline:
+      runner: task-reference
+      adapter: hf-diffusers
+      mode: hf-eager
+      reference_backend: hf_diffusers
+      timing_scope: task-pipeline-call-wall
+      output_contract: media-shape
+      adapter_environment:
+        diffusers_repo: TRTMC_MINIMAX_H3_DIFFUSERS_REPO
+        transformers_repo: TRTMC_MINIMAX_H3_TRANSFORMERS_REPO
+      adapter_options:
+        diffusers_revision: abc5e9bf71fd38f53cd471bc3acaa84bc5ecbfdc
+        generator_device: cpu
+        output_fields: [videos]
+        pipeline_load_mode: modular_components
+        require_pinned_diffusers_source: true
+        require_pinned_transformers_source: true
+        transformers_compat_revision: bed02e1faee69e866e382f835b4f7b0a3c7b8431
   - id: mistral.generate
     family: mistral
     operation: generate

examples/trtmc_benchmark_worker.cpp

@@ -520,6 +520,7 @@ Json run_generate_image(trtmc::IPipeline& pipeline, const Json& request,
     for (int index = 0; index < timing.iterations; ++index) {
         const IterationTimer timer(timing.scope);
         last = generate();
+        const double measured_ms = timer.elapsed_ms();
         const std::size_t generated_pixels =
             std::accumulate(last.begin(), last.end(), std::size_t{0},
                             [](std::size_t count, const trtmc::ImageResult& image) {
@@ -530,7 +531,6 @@ Json run_generate_image(trtmc::IPipeline& pipeline, const Json& request,
             [](std::size_t count, const trtmc::ImageResult& image) {
                 return count + static_cast<std::size_t>(std::max<int32_t>(image.num_frames, 1));
             });
-        const double measured_ms = timer.elapsed_ms();
         observations.push_back({
             {"iteration", index},
             {"measured_wall_ms", measured_ms},

python/tensorrt_model_connect/benchmark/task_adapters.py

@@ -12,6 +12,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
+import json
 from pathlib import Path
 import sys
 from typing import Any, Callable, Mapping
@@ -276,6 +277,17 @@ def _prompt_from_file(testcase: Mapping[str, Any], model_root: Path) -> tuple[st
         raise BenchmarkError(f"cannot read generate_image prompt file {resolved}: {exc}") from exc
     if not value:
         raise BenchmarkError(f"generate_image prompt file is empty: {resolved}")
+    try:
+        structured = json.loads(value)
+    except json.JSONDecodeError:
+        structured = None
+    if isinstance(structured, Mapping):
+        prompt = structured.get("prompt")
+        if not isinstance(prompt, str) or not prompt.strip():
+            raise BenchmarkError(
+                f"generate_image JSON prompt file requires a non-empty prompt: {resolved}"
+            )
+        value = prompt.strip()
     return value, str(portable)
 
 

tests/e2e/models/minimax_h3/compare_video.py

@@ -118,7 +118,7 @@ def inventory_sha256(receipt: dict) -> str | None:
     receipt = {
         "source_revision": source_revision,
         **input_records,
-        "quality_contract": "human_visible_low_frequency_structure_and_motion",
+        "quality_contract": "aligned_low_frequency_structure_chroma_and_motion",
         "pixel_metrics_gating": False,
         "shape": list(decoded.shape),
         "expected_shape": expected_shape,

tests/e2e/models/minimax_h3/e2e_plugins/comparator.py

@@ -141,15 +141,15 @@ def compare(
             status=StageStatus.PASSED.value if passed else StageStatus.FAILED.value,
             metrics=metrics,
             composite_rule=(
-                "exact finite decoded RGB shape AND low-frequency frame structure AND "
-                "brightness profile AND temporal activity/profile AND non-degenerate "
-                "frame contrast; PSNR/MAE are diagnostic only"
+                "exact finite decoded RGB shape AND aligned chroma AND "
+                "low-frequency scene structure AND bounded motion/"
+                "contrast; Pearson profile correlations and PSNR/MAE are diagnostic only"
             ),
             message=(
-                f"{'PASS' if passed else 'FAIL'}: low_frequency_correlation="
+                f"{'PASS' if passed else 'FAIL'}: chroma MAE p95="
+                f"{decoded.chroma_absolute_error_p95:.4f}, low_frequency_correlation="
                 f"{decoded.frame_low_frequency_correlation_minimum:.4f}/"
                 f"{decoded.frame_low_frequency_correlation_mean:.4f} (min/mean), "
-                f"temporal_correlation={decoded.temporal_activity_correlation:.4f}, "
                 f"PSNR={decoded.psnr_db:.4f} dB (diagnostic), "
                 f"MAE={decoded.mean_absolute_error:.8f} (diagnostic)"
             ),

tests/e2e/models/minimax_h3/manifests/minimax-h3-768p.json

@@ -76,7 +76,7 @@
           "required": true
         }
       ],
-      "notes": "The production profile is a real single-device execution: it creates no TensorRT distributed collective and performs no NCCL initialization or communication. The HF backend is the pinned Diffusers modular pipeline. The acceptance gate compares every decoded frame for exact shape, finite pixels, low-frequency scene structure, brightness progression, temporal activity, and non-degenerate contrast; PSNR and pixel error remain diagnostic because harmless high-frequency texture drift is allowed."
+      "notes": "The production profile is a real single-device execution: it creates no TensorRT distributed collective and performs no NCCL initialization or communication. The HF backend is the pinned Diffusers modular pipeline. The acceptance gate requires exact decoded shape and finite pixels, then compares aligned chroma, low-frequency scene layout, bounded motion, and non-degenerate contrast. Brightness/activity Pearson correlations, PSNR, and pixel error remain diagnostic because low-amplitude profiles are correlation-unstable and harmless high-frequency texture drift is allowed."
     }
   ]
 }