diff --git a/benchmarks/performance/README.md b/benchmarks/performance/README.md index cd9e715671..4b9978f9ab 100644 --- a/benchmarks/performance/README.md +++ b/benchmarks/performance/README.md @@ -20,7 +20,7 @@ The suite contains one row for every release-relevant single-process model profile marked `ready` in the benchmark catalog. Profiles whose names contain an `l0` segment are shorter PR-smoke duplicates and are deliberately excluded. Other temporary omissions must be named under `excluded_profiles` with a reason. -The suite currently has 107 model-profile comparisons across 77 families and 78 +The suite currently has 111 model-profile comparisons across 79 families and 81 `(family, operation)` contracts because some families expose multiple profiles and `eagle_vlm` exposes both `embed` and `rerank`. Catalog profiles marked `distributed` require their own multi-process launch and are not silently @@ -247,10 +247,16 @@ Reference-specific upstream checkout paths remain process environment inputs: ```text TRTMC_ELF_REFERENCE_REPO TRTMC_LANCE_REFERENCE_REPO +TRTMC_MINIMAX_H3_DIFFUSERS_REPO +TRTMC_MINIMAX_H3_TRANSFORMERS_REPO TRTMC_SANA_WM_REFERENCE_REPO PERSONAPLEX_OFFICIAL_REPO ``` +Task-reference baselines declare the mapping from adapter option to environment +variable with `baseline.adapter_environment`. This keeps model checkout names in +the suite entry while the shared runner only resolves the declared mapping. + CI should prebuild the selected Python profiles and set `TRTMC_PYTHON_PROFILE_PREBUILT_ONLY=1`. Dependency installation is outside the measured campaign. diff --git a/benchmarks/performance/baselines/task_reference.py b/benchmarks/performance/baselines/task_reference.py index 43dfdc03a9..60556726e7 100644 --- a/benchmarks/performance/baselines/task_reference.py +++ b/benchmarks/performance/baselines/task_reference.py @@ -87,7 +87,7 @@ class Session: """One loaded reference model and its repeatable timed operation.""" - invoke: Callable[[], Mapping[str, Any]] + invoke: Callable[[], Any] resolved_revision: str framework: str timing_scope: str = "task-model-call-wall" @@ -95,6 +95,7 @@ class Session: asset_loading_included: bool = False reference_dependencies: Mapping[str, str] | None = None reference_source: Mapping[str, str] | None = None + summarize: Callable[[Any], Mapping[str, Any]] | None = None def build_parser() -> argparse.ArgumentParser: @@ -1359,8 +1360,13 @@ def _diffusion_pipeline( "wan_t2v": ("WanPipeline", "DiffusionPipeline"), "wan2_2_ti2v": ("WanPipeline", "DiffusionPipeline"), "z_image": ("ZImagePipeline", "DiffusionPipeline"), - }[arguments.family] + }.get(arguments.family, ()) configured_classes = options.get("pipeline_classes") + pipeline_load_mode = str(options.get("pipeline_load_mode", "from_pretrained")) + if pipeline_load_mode not in {"from_pretrained", "modular_components"}: + raise ValueError( + "pipeline_load_mode must be 'from_pretrained' or 'modular_components'" + ) if configured_classes is None: classes = default_classes elif ( @@ -1380,6 +1386,37 @@ def _diffusion_pipeline( _cached_snapshot_path(model_id, requested_revision, "model_index.json") or model_source ) + if pipeline_load_mode == "modular_components": + manager_class = getattr(diffusers, "ComponentsManager", None) + pipeline_class = getattr(diffusers, "ModularPipeline", None) + if manager_class is None or pipeline_class is None: + raise RuntimeError("Diffusers does not provide the modular pipeline API") + load_options = { + "trust_remote_code": bool( + options.get("trust_remote_code", arguments.trust_remote_code) + ), + "local_files_only": arguments.local_files_only, + } + if requested_revision and model_source == model_id: + load_options["revision"] = requested_revision + pipeline = pipeline_class.from_pretrained( + model_source, + components_manager=manager_class(), + **load_options, + ) + component_options = { + "dtype": _torch_dtype(torch_module, arguments.precision), + "pretrained_model_name_or_path": model_source, + "local_files_only": arguments.local_files_only, + } + if requested_revision and model_source == model_id: + component_options["revision"] = requested_revision + pipeline.load_components(**component_options) + return pipeline + if not classes: + raise ValueError( + f"pipeline_classes must be configured for Diffusers family {arguments.family!r}" + ) errors = [] for name in classes: pipeline_class = getattr(diffusers, name, None) @@ -1443,6 +1480,48 @@ def _load_diffusers( request: Mapping[str, Any], options: Mapping[str, Any], ) -> Session: + diffusers_revision = "" + transformers_revision = "" + transformers_repo = str(options.get("transformers_repo", "") or "") + if bool(options.get("require_pinned_transformers_source", False)): + expected_revision = str(options.get("transformers_compat_revision", "") or "") + transformers_revision = _pinned_checkout_revision( + transformers_repo, + expected_revision, + repository="pinned Transformers reference", + ) + source_root = Path(transformers_repo).resolve() / "src" + entrypoint = source_root / "transformers" / "__init__.py" + if not entrypoint.is_file(): + raise ValueError(f"pinned Transformers checkout is incomplete: {entrypoint}") + imported = sys.modules.get("transformers") + imported_path = Path(str(getattr(imported, "__file__", "") or "")) + if imported is not None and source_root not in imported_path.parents: + raise ValueError( + "Transformers was imported before the pinned source was activated" + ) + if str(source_root) not in sys.path: + sys.path.insert(0, str(source_root)) + diffusers_repo = str(options.get("diffusers_repo", "") or "") + if bool(options.get("require_pinned_diffusers_source", False)): + expected_revision = str(options.get("diffusers_revision", "") or "") + diffusers_revision = _pinned_checkout_revision( + diffusers_repo, + expected_revision, + repository="pinned Diffusers reference", + ) + source_root = Path(diffusers_repo).resolve() / "src" + entrypoint = source_root / "diffusers" / "__init__.py" + if not entrypoint.is_file(): + raise ValueError(f"pinned Diffusers checkout is incomplete: {entrypoint}") + imported = sys.modules.get("diffusers") + imported_path = Path(str(getattr(imported, "__file__", "") or "")) + if imported is not None and source_root not in imported_path.parents: + raise ValueError( + "Diffusers was imported before the pinned source was activated" + ) + if str(source_root) not in sys.path: + sys.path.insert(0, str(source_root)) import inspect import torch from PIL import Image @@ -1517,6 +1596,15 @@ def _load_diffusers( if arguments.family == "qwen_image" and cfg_scale >= 0: values["true_cfg_scale"] = cfg_scale values["output_type"] = "np" + output_fields = options.get("output_fields") + if output_fields is not None: + if ( + not isinstance(output_fields, list) + or not output_fields + or any(not isinstance(name, str) or not name for name in output_fields) + ): + raise ValueError("output_fields must be a non-empty list of names") + values["output"] = list(output_fields) image_path = str(request.get("image_path", "") or "") if image_path and ("image" in accepted or accepts_extra): values["image"] = Image.open(_asset_path(arguments, request, "image_path")).convert("RGB") @@ -1541,14 +1629,19 @@ def _load_diffusers( else: seeds = seed if "generator" in accepted or accepts_extra: + generator_device = str(options.get("generator_device", "cuda")) + if generator_device not in {"cpu", "cuda"}: + raise ValueError("generator_device must be cpu or cuda") if isinstance(seeds, list): call_values["generator"] = [ - torch.Generator("cuda").manual_seed(value) for value in seeds + torch.Generator(generator_device).manual_seed(value) for value in seeds ] else: - call_values["generator"] = torch.Generator("cuda").manual_seed(seeds) + call_values["generator"] = torch.Generator(generator_device).manual_seed(seeds) - def invoke() -> Mapping[str, Any]: + media_type = str(request.get("media_type", "image")) + + def invoke() -> Any: if "generator" in call_values: generators = call_values["generator"] if isinstance(generators, list): @@ -1560,8 +1653,12 @@ def invoke() -> Mapping[str, Any]: media = getattr(result, "images", None) if media is None: media = getattr(result, "frames", None) - media_type = str(request.get("media_type", "image")) - return _media_summary(media, media_type) + if media is None and isinstance(result, Mapping): + for name in output_fields or ("videos", "images", "frames"): + media = result.get(name) + if media is not None: + break + return media requested_revision = str( options.get("model_revision", getattr(arguments, "revision", None) or "") @@ -1572,6 +1669,13 @@ def invoke() -> Mapping[str, Any]: else _resolved_revision(arguments, getattr(pipeline, "transformer", pipeline)) ) reference_model = str(options.get("model_id", getattr(arguments, "model", "unresolved"))) + dependencies = None + if diffusers_revision: + dependencies = { + "https://github.com/huggingface/diffusers.git": diffusers_revision, + } + if transformers_revision: + dependencies["https://github.com/huggingface/transformers.git"] = transformers_revision return Session( invoke, revision, @@ -1582,6 +1686,8 @@ def invoke() -> Mapping[str, Any]: "repository": f"https://huggingface.co/{reference_model}", "revision": revision, }, + reference_dependencies=dependencies, + summarize=lambda media: _media_summary(media, media_type), ) @@ -2305,7 +2411,7 @@ def _synchronize() -> None: def _measure(session: Session, warmup: int, iterations: int) -> tuple[list[float], dict[str, Any]]: - output: Mapping[str, Any] = {} + output: Any = {} for _ in range(warmup): output = session.invoke() _synchronize() @@ -2316,7 +2422,8 @@ def _measure(session: Session, warmup: int, iterations: int) -> tuple[list[float output = session.invoke() _synchronize() samples.append((time.perf_counter() - started) * 1000.0) - return samples, dict(output) + summary = session.summarize(output) if session.summarize is not None else output + return samples, dict(summary) def _run_elf( diff --git a/benchmarks/performance/release.yaml b/benchmarks/performance/release.yaml index 0c5fdf3fb0..c2faa7b2d1 100644 --- a/benchmarks/performance/release.yaml +++ b/benchmarks/performance/release.yaml @@ -30,11 +30,6 @@ excluded_profiles: reason: *lfm2_performance_exclusion - model: lfm2-700m reason: *lfm2_performance_exclusion - - model: minimax-h3-768p - reason: >- - The pinned Diffusers reference for MiniMax-H3 has not yet been integrated - into the release performance runner. - entries: - id: albert.encode family: albert @@ -524,6 +519,30 @@ entries: mode: torch-compile compile_scope: model.forward output_token_policy: strip-start + - id: minimax_h3.generate_image + family: minimax_h3 + operation: generate_image + model: minimax-h3-768p + workload: + testcase: minimax-h3-768p + baseline: + runner: task-reference + adapter: hf-diffusers + mode: hf-eager + reference_backend: hf_diffusers + timing_scope: task-pipeline-call-wall + output_contract: media-shape + adapter_environment: + diffusers_repo: TRTMC_MINIMAX_H3_DIFFUSERS_REPO + transformers_repo: TRTMC_MINIMAX_H3_TRANSFORMERS_REPO + adapter_options: + diffusers_revision: abc5e9bf71fd38f53cd471bc3acaa84bc5ecbfdc + generator_device: cpu + output_fields: [videos] + pipeline_load_mode: modular_components + require_pinned_diffusers_source: true + require_pinned_transformers_source: true + transformers_compat_revision: bed02e1faee69e866e382f835b4f7b0a3c7b8431 - id: mistral.generate family: mistral operation: generate diff --git a/examples/trtmc_benchmark_worker.cpp b/examples/trtmc_benchmark_worker.cpp index 3c404fab01..6964c1faae 100644 --- a/examples/trtmc_benchmark_worker.cpp +++ b/examples/trtmc_benchmark_worker.cpp @@ -520,6 +520,7 @@ Json run_generate_image(trtmc::IPipeline& pipeline, const Json& request, for (int index = 0; index < timing.iterations; ++index) { const IterationTimer timer(timing.scope); last = generate(); + const double measured_ms = timer.elapsed_ms(); const std::size_t generated_pixels = std::accumulate(last.begin(), last.end(), std::size_t{0}, [](std::size_t count, const trtmc::ImageResult& image) { @@ -530,7 +531,6 @@ Json run_generate_image(trtmc::IPipeline& pipeline, const Json& request, [](std::size_t count, const trtmc::ImageResult& image) { return count + static_cast(std::max(image.num_frames, 1)); }); - const double measured_ms = timer.elapsed_ms(); observations.push_back({ {"iteration", index}, {"measured_wall_ms", measured_ms}, diff --git a/python/tensorrt_model_connect/benchmark/task_adapters.py b/python/tensorrt_model_connect/benchmark/task_adapters.py index badc2a4f86..ae0fc98401 100644 --- a/python/tensorrt_model_connect/benchmark/task_adapters.py +++ b/python/tensorrt_model_connect/benchmark/task_adapters.py @@ -12,6 +12,7 @@ from __future__ import annotations from dataclasses import dataclass, field +import json from pathlib import Path import sys from typing import Any, Callable, Mapping @@ -276,6 +277,17 @@ def _prompt_from_file(testcase: Mapping[str, Any], model_root: Path) -> tuple[st raise BenchmarkError(f"cannot read generate_image prompt file {resolved}: {exc}") from exc if not value: raise BenchmarkError(f"generate_image prompt file is empty: {resolved}") + try: + structured = json.loads(value) + except json.JSONDecodeError: + structured = None + if isinstance(structured, Mapping): + prompt = structured.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + raise BenchmarkError( + f"generate_image JSON prompt file requires a non-empty prompt: {resolved}" + ) + value = prompt.strip() return value, str(portable) diff --git a/tests/e2e/models/minimax_h3/compare_video.py b/tests/e2e/models/minimax_h3/compare_video.py index 053f0b1d11..d3d8d3f21b 100644 --- a/tests/e2e/models/minimax_h3/compare_video.py +++ b/tests/e2e/models/minimax_h3/compare_video.py @@ -118,7 +118,7 @@ def inventory_sha256(receipt: dict) -> str | None: receipt = { "source_revision": source_revision, **input_records, - "quality_contract": "human_visible_low_frequency_structure_and_motion", + "quality_contract": "aligned_low_frequency_structure_chroma_and_motion", "pixel_metrics_gating": False, "shape": list(decoded.shape), "expected_shape": expected_shape, diff --git a/tests/e2e/models/minimax_h3/e2e_plugins/comparator.py b/tests/e2e/models/minimax_h3/e2e_plugins/comparator.py index 5d1d369da9..985a059706 100644 --- a/tests/e2e/models/minimax_h3/e2e_plugins/comparator.py +++ b/tests/e2e/models/minimax_h3/e2e_plugins/comparator.py @@ -141,15 +141,15 @@ def compare( status=StageStatus.PASSED.value if passed else StageStatus.FAILED.value, metrics=metrics, composite_rule=( - "exact finite decoded RGB shape AND low-frequency frame structure AND " - "brightness profile AND temporal activity/profile AND non-degenerate " - "frame contrast; PSNR/MAE are diagnostic only" + "exact finite decoded RGB shape AND aligned chroma AND " + "low-frequency scene structure AND bounded motion/" + "contrast; Pearson profile correlations and PSNR/MAE are diagnostic only" ), message=( - f"{'PASS' if passed else 'FAIL'}: low_frequency_correlation=" + f"{'PASS' if passed else 'FAIL'}: chroma MAE p95=" + f"{decoded.chroma_absolute_error_p95:.4f}, low_frequency_correlation=" f"{decoded.frame_low_frequency_correlation_minimum:.4f}/" f"{decoded.frame_low_frequency_correlation_mean:.4f} (min/mean), " - f"temporal_correlation={decoded.temporal_activity_correlation:.4f}, " f"PSNR={decoded.psnr_db:.4f} dB (diagnostic), " f"MAE={decoded.mean_absolute_error:.8f} (diagnostic)" ), diff --git a/tests/e2e/models/minimax_h3/manifests/minimax-h3-768p.json b/tests/e2e/models/minimax_h3/manifests/minimax-h3-768p.json index 7471962e6d..83e5c06e38 100644 --- a/tests/e2e/models/minimax_h3/manifests/minimax-h3-768p.json +++ b/tests/e2e/models/minimax_h3/manifests/minimax-h3-768p.json @@ -76,7 +76,7 @@ "required": true } ], - "notes": "The production profile is a real single-device execution: it creates no TensorRT distributed collective and performs no NCCL initialization or communication. The HF backend is the pinned Diffusers modular pipeline. The acceptance gate compares every decoded frame for exact shape, finite pixels, low-frequency scene structure, brightness progression, temporal activity, and non-degenerate contrast; PSNR and pixel error remain diagnostic because harmless high-frequency texture drift is allowed." + "notes": "The production profile is a real single-device execution: it creates no TensorRT distributed collective and performs no NCCL initialization or communication. The HF backend is the pinned Diffusers modular pipeline. The acceptance gate requires exact decoded shape and finite pixels, then compares aligned chroma, low-frequency scene layout, bounded motion, and non-degenerate contrast. Brightness/activity Pearson correlations, PSNR, and pixel error remain diagnostic because low-amplitude profiles are correlation-unstable and harmless high-frequency texture drift is allowed." } ] } diff --git a/tests/e2e/models/minimax_h3/test_minimax_h3_e2e.py b/tests/e2e/models/minimax_h3/test_minimax_h3_e2e.py index 92f046506a..b2f0e5236b 100644 --- a/tests/e2e/models/minimax_h3/test_minimax_h3_e2e.py +++ b/tests/e2e/models/minimax_h3/test_minimax_h3_e2e.py @@ -80,6 +80,9 @@ def test_minimax_h3_manifest_is_truthful_single_device_contract() -> None: assert case.threshold_overrides["low_frequency_block_size"] == 16 assert case.threshold_overrides["minimum_frame_low_frequency_correlation"] == 0.8 assert case.threshold_overrides["minimum_mean_low_frequency_correlation"] == 0.9 + assert case.threshold_overrides["maximum_chroma_absolute_error_p95"] == 0.05 + assert "minimum_brightness_profile_correlation" not in case.threshold_overrides + assert "minimum_temporal_activity_correlation" not in case.threshold_overrides assert "minimum_psnr_db" not in case.threshold_overrides assert "maximum_mean_absolute_error" not in case.threshold_overrides @@ -208,14 +211,13 @@ def _visual_thresholds( "low_frequency_block_size": block_size, "minimum_frame_low_frequency_correlation": 0.8, "minimum_mean_low_frequency_correlation": 0.9, - "minimum_brightness_profile_correlation": 0.95, "maximum_frame_brightness_absolute_error": 0.08, - "minimum_temporal_activity_correlation": 0.9, "maximum_temporal_activity_absolute_error": 0.05, "minimum_temporal_activity_ratio": 0.5, "maximum_temporal_activity_ratio": 1.5, "minimum_frame_std_ratio": 0.7, "maximum_frame_std_ratio": 1.4, + "maximum_chroma_absolute_error_p95": 0.05, } @@ -315,6 +317,7 @@ def test_minimax_h3_comparator_accepts_high_frequency_texture_drift( assert result.metrics["mean_absolute_error"].operator == "diagnostic" assert result.metrics["frame_low_frequency_correlation_minimum"].value == pytest.approx(1.0) assert result.metrics["temporal_activity_correlation"].value == pytest.approx(1.0) + assert result.metrics["chroma_absolute_error_p95"].passed @pytest.mark.parametrize( @@ -322,7 +325,7 @@ def test_minimax_h3_comparator_accepts_high_frequency_texture_drift( [ ("collapse", "frame_std_ratio_minimum"), ("freeze", "temporal_activity_ratio_minimum"), - ("timing_shift", "temporal_activity_correlation"), + ("timing_shift", "frame_low_frequency_correlation_minimum"), ], ) def test_minimax_h3_comparator_rejects_visible_failure_modes( @@ -345,6 +348,30 @@ def test_minimax_h3_comparator_rejects_visible_failure_modes( assert not result.metrics[expected_failed_metric].passed +def test_minimax_h3_comparator_rejects_channel_swap_with_chroma_gate( + tmp_path: Path, +) -> None: + reference = _synthetic_video() + candidate = reference[..., [2, 1, 0]].copy() + + result = _compare_arrays(tmp_path, reference, candidate) + + assert result.status == "failed" + assert not result.metrics["chroma_absolute_error_p95"].passed + + +def test_minimax_h3_profile_correlations_are_diagnostic_only(tmp_path: Path) -> None: + reference = _synthetic_video() + candidate = np.roll(reference, shift=3, axis=0) + + result = _compare_arrays(tmp_path, reference, candidate) + + for name in ("brightness_profile_correlation", "temporal_activity_correlation"): + assert result.metrics[name].operator == "diagnostic" + assert result.metrics[name].threshold is None + assert result.metrics[name].passed + + def test_minimax_h3_comparator_requires_exact_shape_and_finite_pixels( tmp_path: Path, ) -> None: @@ -376,15 +403,15 @@ def test_minimax_h3_comparator_requires_exact_shape_and_finite_pixels( def test_compare_video_cli_binds_threshold_schema_and_run_receipts(tmp_path: Path) -> None: reference_path = tmp_path / "reference.npy" candidate_path = tmp_path / "candidate.npy" - frames = np.zeros((1, 16, 16, 3), dtype=np.float32) + frames = np.zeros((1, 64, 64, 3), dtype=np.float32) np.save(reference_path, frames) np.save(candidate_path, frames) revision = "1" * 40 workload = { "prompt": "test", "seed": 0, - "height": 16, - "width": 16, + "height": 64, + "width": 64, "num_frames": 1, "num_inference_steps": 1, } @@ -418,7 +445,7 @@ def test_compare_video_cli_binds_threshold_schema_and_run_receipts(tmp_path: Pat json.dumps( { "threshold_overrides": { - **_visual_thresholds(1, 16, 16), + **_visual_thresholds(1, 64, 64), } } ) @@ -442,7 +469,12 @@ def test_compare_video_cli_binds_threshold_schema_and_run_receipts(tmp_path: Pat ] environment = os.environ.copy() if environment.get("TRTMC_TEST_INSTALLED_WHEEL") != "1": - environment["PYTHONPATH"] = str(_PROJECT_DIR / "python") + environment["PYTHONPATH"] = os.pathsep.join( + filter( + None, + (str(_PROJECT_DIR / "python"), environment.get("PYTHONPATH", "")), + ) + ) result = subprocess.run( command, cwd=_PROJECT_DIR, env=environment, capture_output=True, text=True ) diff --git a/tests/e2e/models/minimax_h3/thresholds/minimax-h3-768p.json b/tests/e2e/models/minimax_h3/thresholds/minimax-h3-768p.json index 4256655037..b9fb1a67c4 100644 --- a/tests/e2e/models/minimax_h3/thresholds/minimax-h3-768p.json +++ b/tests/e2e/models/minimax_h3/thresholds/minimax-h3-768p.json @@ -6,13 +6,12 @@ "low_frequency_block_size": 16, "minimum_frame_low_frequency_correlation": 0.8, "minimum_mean_low_frequency_correlation": 0.9, - "minimum_brightness_profile_correlation": 0.95, "maximum_frame_brightness_absolute_error": 0.08, - "minimum_temporal_activity_correlation": 0.9, "maximum_temporal_activity_absolute_error": 0.05, "minimum_temporal_activity_ratio": 0.5, "maximum_temporal_activity_ratio": 1.5, "minimum_frame_std_ratio": 0.7, - "maximum_frame_std_ratio": 1.4 + "maximum_frame_std_ratio": 1.4, + "maximum_chroma_absolute_error_p95": 0.05 } } diff --git a/tests/e2e/models/minimax_h3/validation/minimax-h3-768p.json b/tests/e2e/models/minimax_h3/validation/minimax-h3-768p.json deleted file mode 100644 index 382bd4d816..0000000000 --- a/tests/e2e/models/minimax_h3/validation/minimax-h3-768p.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "schema_version": "trtmc.model-plugin-validation/v1", - "dataset": "MiniMax-H3 pinned 768p T2VA profile", - "version": "1", - "requests": [ - { - "sample_id": "minimax-h3-768p-official-profile", - "testcase": "minimax-h3-768p", - "stage": "end_to_end", - "category": "official-profile", - "inputs": {} - } - ] -} diff --git a/tests/e2e/models/minimax_h3/visual_metrics.py b/tests/e2e/models/minimax_h3/visual_metrics.py index 41f8bc20ba..be181bcfab 100644 --- a/tests/e2e/models/minimax_h3/visual_metrics.py +++ b/tests/e2e/models/minimax_h3/visual_metrics.py @@ -1,12 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Streaming decoded-video metrics for the MiniMax-H3 visual quality contract. +"""Decoded-video metrics for the MiniMax-H3 visual parity contract. -The acceptance contract deliberately compares low-frequency structure and motion -instead of requiring pixel identity. Diffusion implementations can differ in -high-frequency texture while producing the same coherent scene. Pixel-space -PSNR and MAE are still reported to aid debugging, but never gate acceptance. +The acceptance contract compares low-frequency scene layout, chroma, and motion +instead of requiring pixel identity. Diffusion +implementations can differ in high-frequency texture while producing the same +coherent scene. Pixel-space PSNR and MAE remain diagnostic only. """ from __future__ import annotations @@ -27,14 +27,13 @@ "low_frequency_block_size", "minimum_frame_low_frequency_correlation", "minimum_mean_low_frequency_correlation", - "minimum_brightness_profile_correlation", "maximum_frame_brightness_absolute_error", - "minimum_temporal_activity_correlation", "maximum_temporal_activity_absolute_error", "minimum_temporal_activity_ratio", "maximum_temporal_activity_ratio", "minimum_frame_std_ratio", "maximum_frame_std_ratio", + "maximum_chroma_absolute_error_p95", } ) @@ -57,6 +56,9 @@ class DecodedVisualMetrics: temporal_activity_ratio: float frame_std_ratio_minimum: float frame_std_ratio_maximum: float + chroma_absolute_error_mean: float + chroma_absolute_error_p95: float + chroma_absolute_error_maximum: float @dataclass(frozen=True) @@ -160,6 +162,7 @@ def compute_decoded_visual_metrics( reference_activity: list[float] = [] candidate_activity: list[float] = [] std_ratios: list[float] = [] + chroma_errors: list[float] = [] previous_reference_blocks: np.ndarray | None = None previous_candidate_blocks: np.ndarray | None = None @@ -187,6 +190,19 @@ def compute_decoded_visual_metrics( reference_brightness.append(float(reference_blocks.mean())) candidate_brightness.append(float(candidate_blocks.mean())) + luma_weights = np.asarray((0.2126, 0.7152, 0.0722), dtype=np.float32) + reference_luma = np.sum(reference_blocks * luma_weights, axis=-1) + candidate_luma = np.sum(candidate_blocks * luma_weights, axis=-1) + reference_chroma = np.stack( + (reference_blocks[..., 2] - reference_luma, reference_blocks[..., 0] - reference_luma), + axis=-1, + ) + candidate_chroma = np.stack( + (candidate_blocks[..., 2] - candidate_luma, candidate_blocks[..., 0] - candidate_luma), + axis=-1, + ) + chroma_errors.append(float(np.mean(np.abs(candidate_chroma - reference_chroma)))) + reference_std = float(reference_frame.std(dtype=np.float64)) candidate_std = float(candidate_frame.std(dtype=np.float64)) if reference_std <= np.finfo(np.float64).eps: @@ -243,6 +259,9 @@ def compute_decoded_visual_metrics( temporal_activity_ratio=activity_ratio, frame_std_ratio_minimum=float(min(std_ratios)), frame_std_ratio_maximum=float(max(std_ratios)), + chroma_absolute_error_mean=float(np.mean(chroma_errors)), + chroma_absolute_error_p95=float(np.quantile(chroma_errors, 0.95)), + chroma_absolute_error_maximum=float(np.max(chroma_errors)), ) @@ -268,6 +287,9 @@ def evaluate_visual_quality( raise ValueError("invalid MiniMax-H3 temporal activity ratio interval") if not (0.0 < minimum_std_ratio <= maximum_std_ratio): raise ValueError("invalid MiniMax-H3 frame standard-deviation ratio interval") + maximum_chroma_error = float(thresholds["maximum_chroma_absolute_error_p95"]) + if not math.isfinite(maximum_chroma_error) or maximum_chroma_error <= 0.0: + raise ValueError("maximum_chroma_absolute_error_p95 must be positive and finite") gates = { "num_frames": VisualGateResult( @@ -289,6 +311,14 @@ def evaluate_visual_quality( float(metrics.shape[3]), 3.0, "==", metrics.shape[3] == 3 ), "finite_pixels": VisualGateResult(1.0, 1.0, "==", True), + "chroma_absolute_error_p95": VisualGateResult( + metrics.chroma_absolute_error_p95, + float(thresholds["maximum_chroma_absolute_error_p95"]), + "<=", + metrics.chroma_absolute_error_p95 + <= float(thresholds["maximum_chroma_absolute_error_p95"]), + "Mean absolute error in aligned B-Y and R-Y channels.", + ), "frame_low_frequency_correlation_minimum": VisualGateResult( metrics.frame_low_frequency_correlation_minimum, float(thresholds["minimum_frame_low_frequency_correlation"]), @@ -305,10 +335,10 @@ def evaluate_visual_quality( ), "brightness_profile_correlation": VisualGateResult( metrics.brightness_profile_correlation, - float(thresholds["minimum_brightness_profile_correlation"]), - ">=", - metrics.brightness_profile_correlation - >= float(thresholds["minimum_brightness_profile_correlation"]), + None, + "diagnostic", + True, + "Pearson correlation is unstable for nearly constant brightness profiles.", ), "frame_brightness_absolute_error_maximum": VisualGateResult( metrics.frame_brightness_absolute_error_maximum, @@ -319,10 +349,10 @@ def evaluate_visual_quality( ), "temporal_activity_correlation": VisualGateResult( metrics.temporal_activity_correlation, - float(thresholds["minimum_temporal_activity_correlation"]), - ">=", - metrics.temporal_activity_correlation - >= float(thresholds["minimum_temporal_activity_correlation"]), + None, + "diagnostic", + True, + "Pearson correlation is unstable for low-amplitude activity profiles.", ), "temporal_activity_absolute_error_maximum": VisualGateResult( metrics.temporal_activity_absolute_error_maximum, @@ -372,6 +402,12 @@ def evaluate_visual_quality( "maximum_absolute_error": VisualGateResult( metrics.maximum_absolute_error, None, "diagnostic", True ), + "chroma_absolute_error_mean": VisualGateResult( + metrics.chroma_absolute_error_mean, None, "diagnostic", True + ), + "chroma_absolute_error_maximum": VisualGateResult( + metrics.chroma_absolute_error_maximum, None, "diagnostic", True + ), } return gates diff --git a/tests/tools/test_perf_matrix.py b/tests/tools/test_perf_matrix.py index 557d083aaf..74d5c97e71 100644 --- a/tests/tools/test_perf_matrix.py +++ b/tests/tools/test_perf_matrix.py @@ -37,12 +37,6 @@ def _suite_for_cases(cases, *, exclusions=None): cases=tuple(cases), excluded_profiles=dict(exclusions or {}), ) - - -MINIMAX_H3_EXCLUSION_REASON = ( - "The pinned Diffusers reference for MiniMax-H3 has not yet been integrated " - "into the release performance runner." -) LFM2_EXCLUSION_REASON = ( "Dense LFM2 functional and reference-parity qualification is present, but " "this change does not add a matching release-performance workload or receipt." @@ -62,6 +56,7 @@ def _suite_for_cases(cases, *, exclusions=None): "lance.generate": "upstream-lance", "locateanything.generate": "hf-transformers-vlm", "magpie_tts.generate_audio": "nemo-tts", + "minimax_h3.generate_image": "hf-diffusers", "nemotron_speech_streaming.transcribe": "nemo-asr", "patchtsmixer.solve": "pytorch-timeseries", "patchtst.solve": "pytorch-timeseries", @@ -279,7 +274,6 @@ def test_release_suite_covers_every_non_l0_ready_model_profile() -> None: "lfm2-350m-bf16-model-card": LFM2_EXCLUSION_REASON, "lfm2-350m-fp16": LFM2_EXCLUSION_REASON, "lfm2-700m": LFM2_EXCLUSION_REASON, - "minimax-h3-768p": MINIMAX_H3_EXCLUSION_REASON, } assert all( set(entry["workload"]) <= {"testcase", "request", "runtime"} for entry in raw_entries @@ -2197,7 +2191,9 @@ def preflight_after_pending_report(cases, options): ] assert not scratch_root.exists() results = json.loads((output / "results.json").read_text(encoding="utf-8")) - rows = {row["id"]: row for row in results["cases"]} + result_cases = results["cases"] + rows = {row["id"]: row for row in result_cases} + assert len(rows) == len(result_cases) assert set(rows) == { case["id"] for case in performance_catalog.load_suite(SUITE).cases } @@ -2213,8 +2209,8 @@ def preflight_after_pending_report(cases, options): expected_catalog_coverage = { "total_profiles": len(catalog_entries), "ready_profiles": catalog_counts["ready"], - "release_profiles": catalog_counts["ready"] - excluded_l0_profiles - 6, - "explicitly_excluded_profiles": 6, + "release_profiles": catalog_counts["ready"] - excluded_l0_profiles - 5, + "explicitly_excluded_profiles": 5, "explicit_exclusions": [ { "model": "lfm2-1.2b", @@ -2236,10 +2232,6 @@ def preflight_after_pending_report(cases, options): "model": "lfm2-700m", "reason": LFM2_EXCLUSION_REASON, }, - { - "model": "minimax-h3-768p", - "reason": MINIMAX_H3_EXCLUSION_REASON, - }, ], "excluded_l0_profiles": excluded_l0_profiles, "distributed_profiles": catalog_counts["distributed"], @@ -2333,7 +2325,6 @@ def preflight_after_pending_report(cases, options): for command in public_row["commands"].values() ) assert "minimax-h3-768p" not in json.dumps(public_report) - assert MINIMAX_H3_EXCLUSION_REASON not in json.dumps(public_report) log_records = public_row["debug"]["logs"] assert {record["label"] for record in log_records} == { "TRTMC stdout", @@ -2608,6 +2599,8 @@ def test_task_reference_commands_record_external_checkout_paths( ) -> None: monkeypatch.setenv("TRTMC_ELF_REFERENCE_REPO", "/references/ELF") monkeypatch.setenv("TRTMC_LANCE_REFERENCE_REPO", "/references/Lance") + monkeypatch.setenv("EXAMPLE_DIFFUSERS_REPO", "/references/Diffusers") + monkeypatch.setenv("EXAMPLE_TRANSFORMERS_REPO", "/references/Transformers") monkeypatch.setenv("TRTMC_SANA_WM_REFERENCE_REPO", "/references/Sana") monkeypatch.setenv("PERSONAPLEX_OFFICIAL_REPO", "/references/PersonaPlex") @@ -2620,6 +2613,18 @@ def test_task_reference_commands_record_external_checkout_paths( assert perf_matrix._resolved_adapter_options({"adapter": "upstream-sana-wm"}) == { "reference_repo": "/references/Sana" } + assert perf_matrix._resolved_adapter_options( + { + "adapter": "hf-diffusers", + "adapter_environment": { + "diffusers_repo": "EXAMPLE_DIFFUSERS_REPO", + "transformers_repo": "EXAMPLE_TRANSFORMERS_REPO", + }, + } + ) == { + "diffusers_repo": "/references/Diffusers", + "transformers_repo": "/references/Transformers", + } assert perf_matrix._resolved_adapter_options({"adapter": "pytorch-personaplex"}) == { "official_repo": "/references/PersonaPlex" } @@ -2643,6 +2648,30 @@ def test_external_reference_adapter_rejects_a_missing_checkout( perf_matrix._resolved_adapter_options({"adapter": "upstream-elf"}) +def test_declared_adapter_environment_rejects_a_missing_checkout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("EXAMPLE_DIFFUSERS_REPO", "/references/Diffusers") + monkeypatch.delenv("EXAMPLE_TRANSFORMERS_REPO", raising=False) + + with pytest.raises( + perf_matrix.PerfMatrixError, + match=( + "requires adapter_options.transformers_repo or " + "EXAMPLE_TRANSFORMERS_REPO" + ), + ): + perf_matrix._resolved_adapter_options( + { + "adapter": "hf-diffusers", + "adapter_environment": { + "diffusers_repo": "EXAMPLE_DIFFUSERS_REPO", + "transformers_repo": "EXAMPLE_TRANSFORMERS_REPO", + }, + } + ) + + def test_suite_has_explicit_eager_and_task_reference_rows() -> None: raw = yaml.safe_load(SUITE.read_text(encoding="utf-8")) rows = {row["id"]: row for row in raw["entries"]} @@ -3716,6 +3745,61 @@ def from_pretrained(cls, model, **kwargs): assert captured["kwargs"]["local_files_only"] is True +def test_diffusers_adapter_loads_declared_modular_components( + monkeypatch: pytest.MonkeyPatch, +) -> None: + runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/task_reference.py")) + captured: dict[str, object] = {} + + class FakeManager: + pass + + class FakePipeline: + @classmethod + def from_pretrained(cls, model, **kwargs): + captured.update(model=model, from_pretrained=kwargs) + return cls() + + def load_components(self, **kwargs): + captured["load_components"] = kwargs + + monkeypatch.setitem( + sys.modules, + "diffusers", + Namespace(ComponentsManager=FakeManager, ModularPipeline=FakePipeline), + ) + arguments = Namespace( + family="example_video", + local_files_only=False, + model="MiniMaxAI/MiniMax-H3", + precision="bf16", + revision="model-revision", + trust_remote_code=False, + ) + torch_module = Namespace(float16="fp16", float32="fp32", bfloat16="bf16") + + runner["_diffusion_pipeline"]( + arguments, torch_module, {"pipeline_load_mode": "modular_components"} + ) + + assert captured["model"] == arguments.model + from_pretrained = captured["from_pretrained"] + assert isinstance(from_pretrained["components_manager"], FakeManager) + assert { + key: value for key, value in from_pretrained.items() if key != "components_manager" + } == { + "local_files_only": False, + "revision": "model-revision", + "trust_remote_code": False, + } + assert captured["load_components"] == { + "dtype": "bf16", + "local_files_only": False, + "pretrained_model_name_or_path": arguments.model, + "revision": "model-revision", + } + + def test_diffusers_adapter_uses_configured_pipeline_classes(monkeypatch) -> None: runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/task_reference.py")) selected = [] @@ -4027,7 +4111,7 @@ def fake_pipeline(_arguments, _torch, options): ], }, ) - summary = session.invoke() + _, summary = runner["_measure"](session, 0, 1) assert captured["action"] == "w-80,jw-40" assert captured["intrinsics"] == "1,2,3,4" @@ -4076,6 +4160,7 @@ def __call__(self, *, prompt, generator): globals_ = runner["_load_diffusers"].__globals__ globals_["_diffusion_pipeline"] = lambda *_args: FakePipeline() globals_["_resolved_revision"] = lambda *_args: "snapshot" + globals_["_synchronize"] = lambda: None monkeypatch.setitem(sys.modules, "torch", Namespace(Generator=FakeGenerator)) arguments = Namespace( family="flux", @@ -4097,8 +4182,8 @@ def __call__(self, *, prompt, generator): {}, ) - assert session.invoke()["media_count"] == 2 - assert session.invoke()["media_count"] == 2 + _, summary = runner["_measure"](session, 1, 1) + assert summary["media_count"] == 2 assert captured == [ {"prompt": ["red cube", "blue sphere"], "seeds": [41, 42]}, {"prompt": ["red cube", "blue sphere"], "seeds": [41, 42]}, @@ -4153,10 +4238,113 @@ def __call__(self, *, prompt, output_type): {}, ) - assert session.invoke()["finite"] is True + _, summary = runner["_measure"](session, 0, 1) + assert summary["finite"] is True assert captured == {"prompt": "cat", "output_type": "np"} +def test_minimax_h3_diffusers_adapter_times_video_output_only_with_cpu_generator( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/task_reference.py")) + captured: dict[str, object] = {} + + class FakeGenerator: + def __init__(self, device): + captured["generator_device"] = device + self.seed = None + + def manual_seed(self, seed): + self.seed = seed + return self + + class FakePipeline: + transformer = Namespace() + + def to(self, device): + assert device == "cuda" + return self + + def __call__(self, **kwargs): + captured["call"] = kwargs + return {"videos": np.zeros((1, 2, 4, 6, 3), dtype=np.float32)} + + globals_ = runner["_load_diffusers"].__globals__ + globals_["_diffusion_pipeline"] = lambda *_args: FakePipeline() + globals_["_resolved_revision"] = lambda *_args: "snapshot" + globals_["_pinned_checkout_revision"] = lambda _repo, revision, **_kwargs: revision + globals_["_synchronize"] = lambda: None + monkeypatch.setitem(sys.modules, "torch", Namespace(Generator=FakeGenerator)) + diffusers_repo = tmp_path / "diffusers" + diffusers_package = diffusers_repo / "src/diffusers" + diffusers_package.mkdir(parents=True) + diffusers_entrypoint = diffusers_package / "__init__.py" + diffusers_entrypoint.write_text("", encoding="utf-8") + transformers_repo = tmp_path / "transformers" + transformers_package = transformers_repo / "src/transformers" + transformers_package.mkdir(parents=True) + transformers_entrypoint = transformers_package / "__init__.py" + transformers_entrypoint.write_text("", encoding="utf-8") + monkeypatch.setitem( + sys.modules, + "diffusers", + Namespace(__file__=str(diffusers_entrypoint)), + ) + monkeypatch.setitem( + sys.modules, + "transformers", + Namespace(__file__=str(transformers_entrypoint)), + ) + arguments = Namespace( + family="minimax_h3", + precision="bf16", + model="MiniMaxAI/MiniMax-H3", + revision="model-revision", + ) + + session = runner["_load_diffusers"]( + arguments, + { + "prompt": "A moving subject", + "seed": 0, + "media_type": "video", + "video_height": 4, + "video_width": 6, + "video_num_frames": 2, + "num_inference_steps": 2, + }, + { + "diffusers_repo": str(diffusers_repo), + "diffusers_revision": "diffusers-revision", + "generator_device": "cpu", + "output_fields": ["videos"], + "require_pinned_diffusers_source": True, + "require_pinned_transformers_source": True, + "transformers_repo": str(transformers_repo), + "transformers_compat_revision": "transformers-revision", + }, + ) + + _, summary = runner["_measure"](session, 0, 1) + assert summary == { + "media_type": "video", + "media_count": 2, + "height": 4, + "width": 6, + "channels": 3, + "finite": True, + } + assert captured["generator_device"] == "cpu" + assert captured["call"]["output"] == ["videos"] + assert captured["call"]["output_type"] == "np" + assert captured["call"]["generator"].seed == 0 + assert session.reference_dependencies == { + "https://github.com/huggingface/diffusers.git": "diffusers-revision", + "https://github.com/huggingface/transformers.git": "transformers-revision", + } + + def test_diffusers_media_summary_rejects_non_finite_pixels() -> None: runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/task_reference.py")) finite = np.zeros((1, 4, 6, 3), dtype=np.float32) @@ -4176,6 +4364,31 @@ def test_diffusers_media_summary_rejects_non_finite_pixels() -> None: runner["_media_summary"](invalid, "image") +def test_task_reference_summarizes_only_after_all_timed_invocations() -> None: + runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/task_reference.py")) + events: list[str] = [] + + def invoke() -> dict[str, str]: + events.append("invoke") + return {"text": "ok"} + + def summarize(output: dict[str, str]) -> dict[str, str]: + events.append("summarize") + return output + + session = runner["Session"]( + invoke, + "revision", + "framework", + summarize=summarize, + ) + + _, summary = runner["_measure"](session, 1, 2) + + assert events == ["invoke", "invoke", "invoke", "summarize"] + assert summary == {"text": "ok"} + + def test_personaplex_loader_adds_vendored_moshi_package_root() -> None: source = (REPOSITORY / "benchmarks/performance/baselines/task_reference.py").read_text() diff --git a/tests/tools/test_performance_catalog.py b/tests/tools/test_performance_catalog.py index baf6d46e14..69f8ce984c 100644 --- a/tests/tools/test_performance_catalog.py +++ b/tests/tools/test_performance_catalog.py @@ -28,6 +28,30 @@ def test_release_suite_includes_fast_foundation_stereo() -> None: assert case["id"] == "fast_foundation_stereo.disparity" +def test_release_suite_includes_minimax_h3_video_only_performance() -> None: + suite = performance_catalog.load_suite(SUITE) + + assert "minimax-h3-768p" not in suite.excluded_profiles + case = next(case for case in suite.cases if case["model"] == "minimax-h3-768p") + assert case["id"] == "minimax_h3.generate_image" + assert case["operation"] == "generate_image" + assert case["measurement"] == {"warmup": 3, "iterations": 10} + assert case["baseline"]["adapter"] == "hf-diffusers" + assert case["baseline"]["adapter_environment"] == { + "diffusers_repo": "TRTMC_MINIMAX_H3_DIFFUSERS_REPO", + "transformers_repo": "TRTMC_MINIMAX_H3_TRANSFORMERS_REPO", + } + assert case["baseline"]["adapter_options"] == { + "diffusers_revision": "abc5e9bf71fd38f53cd471bc3acaa84bc5ecbfdc", + "generator_device": "cpu", + "output_fields": ["videos"], + "pipeline_load_mode": "modular_components", + "require_pinned_diffusers_source": True, + "require_pinned_transformers_source": True, + "transformers_compat_revision": "bed02e1faee69e866e382f835b4f7b0a3c7b8431", + } + + def test_selection_rejects_multiple_modes() -> None: suite = performance_catalog.load_suite(SUITE) diff --git a/tests/tools/test_trtmc_bench.py b/tests/tools/test_trtmc_bench.py index 2c60788f49..96a0e1d24b 100644 --- a/tests/tools/test_trtmc_bench.py +++ b/tests/tools/test_trtmc_bench.py @@ -294,6 +294,11 @@ def test_native_worker_has_a_runner_for_every_advertised_operation() -> None: ): assert replay_input in worker_source + image_runner = worker_source.split("Json run_generate_image", 1)[1].split( + "std::size_t audio_sample_count", 1 + )[0] + assert image_runner.index("timer.elapsed_ms()") < image_runner.index("generated_pixels") + def test_default_catalog_falls_back_to_installed_package_data( tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -497,6 +502,24 @@ def test_future_family_reuses_existing_task_adapter_without_benchmark_changes( assert command[command.index("--video-num-frames") + 1] == "17" +def test_minimax_h3_benchmark_extracts_prompt_from_structured_prompt_file( + tmp_path: Path, +) -> None: + model = ManifestCatalog().resolve("minimax-h3-768p") + + case = resolve_case(model, tmp_path / "pending.bundle") + + prompt_record = json.loads( + (REPOSITORY_ROOT / "tests/e2e/models/minimax_h3/prompts/t2va-example-1.json").read_text( + encoding="utf-8" + ) + ) + assert case.operation == "generate_image" + assert case.request["prompt"] == prompt_record["prompt"] + assert not case.request["prompt"].lstrip().startswith("{") + assert case.request["seed"] == prompt_record["seed"] == 0 + + def test_future_object_detection_family_uses_existing_public_capability(tmp_path: Path) -> None: family = tmp_path / "yolox" manifest = family / "manifests/yolox-tiny.json" diff --git a/tests/tools/test_trtmc_validate.py b/tests/tools/test_trtmc_validate.py index 925980e9fc..9f43d46b72 100644 --- a/tests/tools/test_trtmc_validate.py +++ b/tests/tools/test_trtmc_validate.py @@ -114,10 +114,12 @@ def test_lerobot_act_catalog_binds_recorded_control_parity() -> None: } -def test_minimax_h3_catalog_uses_model_owned_official_profile() -> None: +def test_minimax_h3_catalog_uses_vbench_profile() -> None: catalog = trtmc_validate.load_catalog() suites = validation_catalog.load_suites() - suite = next(value for value in suites if value["id"] == "minimax_h3_official_profile_parity") + suites_by_id = {value["id"]: value for value in suites} + vbench_suite = suites_by_id["minimax_h3_vbench_reference_parity"] + assert "minimax_h3_official_profile_parity" not in suites_by_id model = next( value for value in validation_catalog.load_manifest_records(trtmc_validate.DEFAULT_MODELS) @@ -125,37 +127,20 @@ def test_minimax_h3_catalog_uses_model_owned_official_profile() -> None: ) assert catalog["models"]["minimax-h3-768p"] == { - "workloads": ["minimax_h3_official_profile_parity"], + "workloads": ["minimax_h3_vbench_reference_parity"], } - assert validation_catalog.suite_match_reason(suite, model) == ( + assert catalog["sample_limits"]["minimax_h3_vbench_reference_parity"] == 10 + assert validation_catalog.suite_match_reason(vbench_suite, model) == ( True, "selected", ) - assert suite["dataset"] == { + assert vbench_suite["dataset"] == { "kind": "model_plugin_json", - "default_path": ("tests/e2e/models/minimax_h3/validation/minimax-h3-768p.json"), - } - assert suite["scoring"] == {"scorer": "model_plugin_parity"} - assert suite["gates"] == {"min_sample_pass_rate": 1.0} - - dataset_path = trtmc_validate.REPO_ROOT / suite["dataset"]["default_path"] - dataset = json.loads(dataset_path.read_text(encoding="utf-8")) - assert dataset["requests"] == [ - { - "sample_id": "minimax-h3-768p-official-profile", - "testcase": "minimax-h3-768p", - "stage": "end_to_end", - "category": "official-profile", - "inputs": {}, - } - ] - resolved = validation_catalog.resolve_suite_for_model(suite, model) - assert resolved["generation"] == { - "video_num_frames": 124, - "video_height": 768, - "video_width": 1344, - "num_inference_steps": 50, + "default_path": "/mnt/data/VBench-fd18b3d-model-plugin-v1/dataset.json", + "input_asset_fields": ["prompt_file"], } + assert vbench_suite["scoring"] == {"scorer": "model_plugin_parity"} + assert vbench_suite["gates"] == {"min_sample_pass_rate": 1.0} def test_dataset_path_keeps_repository_owned_default_with_dataset_root( @@ -227,7 +212,6 @@ def test_catalog_defines_sample_limit_for_every_dataset_workload(): "fast_foundation_stereo_synthetic_parity", "lfm2_model_card_sampling_parity", "lerobot_act_recorded_control_fp32_parity", - "minimax_h3_official_profile_parity", "moge_monocular_geometry_fp32_parity", "nemotron_voicechat_model_card_general_conversation", "seedtts_en_omni_audio_parity", diff --git a/tests/tools/test_validation_engine.py b/tests/tools/test_validation_engine.py index 6ca1af1641..12bb2101b5 100644 --- a/tests/tools/test_validation_engine.py +++ b/tests/tools/test_validation_engine.py @@ -7433,7 +7433,7 @@ def test_eval_resolves_reference_source_revision_before_preparing_cache_inputs( revision = "a" * 40 suite = validation_engine.suite_by_id( validation_engine.load_suites(), - "minimax_h3_official_profile_parity", + "minimax_h3_vbench_reference_parity", ) model = { "name": "minimax-h3-768p", @@ -7476,6 +7476,15 @@ def fake_prepare(**kwargs): assert captured["model_manifest"] == model["manifest"] +def test_minimax_h3_reference_parity_accepts_eight_of_ten_samples() -> None: + suite = validation_engine.suite_by_id( + validation_engine.load_suites(), + "minimax_h3_vbench_reference_parity", + ) + + assert suite["gates"]["min_sample_pass_rate"] == 0.8 + + def test_flux_validation_build_command_preserves_diffusion_shape(tmp_path: Path) -> None: model = next( model @@ -9259,7 +9268,10 @@ def test_public_ci_artifacts_omit_private_runner_paths(tmp_path: Path) -> None: assert "/private" not in numeric_public.read_text(encoding="utf-8") -def test_prepare_vbench_selects_ten_unique_review_dimensions(tmp_path: Path) -> None: +def test_prepare_vbench_selects_ten_unique_review_dimensions( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: source = tmp_path / "VBench_full_info.json" source.write_text( json.dumps( @@ -9273,6 +9285,11 @@ def test_prepare_vbench_selects_ten_unique_review_dimensions(tmp_path: Path) -> ), encoding="utf-8", ) + monkeypatch.setattr( + prepare_media, + "VBENCH_INFO_SHA256", + prepare_media._sha256(source), + ) output = prepare_media.prepare_vbench(source, tmp_path / "out") payload = json.loads(output.read_text(encoding="utf-8")) @@ -9284,6 +9301,68 @@ def test_prepare_vbench_selects_ten_unique_review_dimensions(tmp_path: Path) -> assert len({row["prompt"] for row in payload["requests"]}) == 10 assert payload["source_info_sha256"] assert payload["license"] == "Apache-2.0" + assert payload["source_revision"] == prepare_media.VBENCH_REVISION + + +def test_prepare_vbench_rejects_source_from_another_revision(tmp_path: Path) -> None: + source = tmp_path / "VBench_full_info.json" + source.write_text("[]\n", encoding="utf-8") + + with pytest.raises(ValueError, match="does not match the pinned revision"): + prepare_media.prepare_vbench(source, tmp_path / "out") + + +def test_prepare_vbench_model_plugin_dataset_is_portable_and_pinned( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source = tmp_path / "VBench_full_info.json" + source.write_text( + json.dumps( + [ + { + "prompt_en": f"official prompt {index}", + "dimension": [dimension], + } + for index, dimension in enumerate(prepare_media.VBENCH_DIMENSIONS) + ] + ), + encoding="utf-8", + ) + monkeypatch.setattr( + prepare_media, + "VBENCH_INFO_SHA256", + prepare_media._sha256(source), + ) + + outputs = prepare_media.prepare_media_datasets( + output_root=tmp_path / "out", + vbench_info=source, + vbench_model_plugin=True, + ) + assert len(outputs) == 1 + dataset = outputs[0] + assert not (tmp_path / "out" / "VBench").exists() + payload = json.loads(dataset.read_text(encoding="utf-8")) + manifest = json.loads( + (dataset.parent / "DATASET_MANIFEST.json").read_text(encoding="utf-8") + ) + + assert payload["request_count"] == 10 + assert payload["license"] == "Apache-2.0" + assert payload["source_revision"] == prepare_media.VBENCH_REVISION + assert [row["category"] for row in payload["requests"]] == list( + prepare_media.VBENCH_DIMENSIONS + ) + for row in payload["requests"]: + prompt_file = dataset.parent / row["inputs"]["prompt_file"] + prompt = json.loads(prompt_file.read_text(encoding="utf-8")) + assert prompt == {"prompt": row["prompt"], "seed": 0} + assert manifest["request_count"] == 10 + assert manifest["source"]["license"] == "Apache-2.0" + assert {record["path"] for record in manifest["files"]} >= { + "dataset.json", + } def test_prepare_gedit_writes_task_diverse_static_condition_images(tmp_path: Path) -> None: diff --git a/tests/validation/README.md b/tests/validation/README.md index 7b11521f00..da8274d5fc 100644 --- a/tests/validation/README.md +++ b/tests/validation/README.md @@ -253,6 +253,23 @@ reference, TRTMC runner, and comparator are invoked directly without calling the E2E orchestrator. Array-valued outputs are persisted as artifacts so a cached reference can be compared in later runs. +MiniMax-H3 reference consistency uses a ten-prompt, task-diverse slice of the +Apache-2.0 VBench prompt suite. Prepare the versioned prompt-file asset from +the pinned upstream files before publishing it to NAS: + +```bash +python tools/validation/engine.py prepare-media \ + --vbench-info /path/to/VBench/vbench/VBench_full_info.json \ + --vbench-model-plugin \ + --output-root /mnt/data \ + --limit 10 +``` + +Publish `VBench-fd18b3d-model-plugin-v1` without changing its relative layout. +A validation machine mounts the same directory at `/mnt/data`. This asset +contains prompts and source provenance only; it contains no generated model +output or external evaluator. + Prepare the fixed task datasets from public benchmark sources already staged on the validation machine: diff --git a/tests/validation/model_workloads.yaml b/tests/validation/model_workloads.yaml index a7bb4e49fa..747d219f7a 100644 --- a/tests/validation/model_workloads.yaml +++ b/tests/validation/model_workloads.yaml @@ -34,7 +34,7 @@ sample_limits: nemotron_voicechat_model_card_general_conversation: 1 mmmu_pro_vision_plugin_parity: 5 mmmu_pro_vision_square_plugin_parity: 5 - minimax_h3_official_profile_parity: 1 + minimax_h3_vbench_reference_parity: 10 moge_monocular_geometry_fp32_parity: 1 newstest2019_en_ru_marian_translation_parity: 10 ocrbench_v2_unified: 5 @@ -168,7 +168,7 @@ models: marian-en-ru: workloads: [newstest2019_en_ru_marian_translation_parity] minimax-h3-768p: - workloads: [minimax_h3_official_profile_parity] + workloads: [minimax_h3_vbench_reference_parity] minitron-4b-depth: workloads: [mmlu_continuation_parity] minitron-4b-width: diff --git a/tests/validation/workloads.yaml b/tests/validation/workloads.yaml index 4cb0f481a1..69146252eb 100644 --- a/tests/validation/workloads.yaml +++ b/tests/validation/workloads.yaml @@ -1902,17 +1902,19 @@ suites: lane: local_only notes: GB300-only native-reference consistency. - - id: minimax_h3_official_profile_parity + - id: minimax_h3_vbench_reference_parity description: > - MiniMax-H3 consistency at the pinned native 1344x768, 124-frame, - 50-step T2VA profile. The single repo-owned request deliberately selects - the model-owned testcase without overriding its prompt, seed, geometry, - or comparison thresholds. + HF-to-TRTMC reference consistency at the pinned MiniMax-H3 1344x768, + 124-frame, 50-step profile over ten task-diverse prompts from VBench. + The versioned NAS asset carries the pinned Apache-2.0 source, license, + prompt files, and checksums. This is not an official VBench aggregate + score; the model-owned visual comparator gates every generated sample. user_contract: diffusion_video default_model_names: [minimax-h3-768p] dataset: kind: model_plugin_json - default_path: tests/e2e/models/minimax_h3/validation/minimax-h3-768p.json + default_path: /mnt/data/VBench-fd18b3d-model-plugin-v1/dataset.json + input_asset_fields: [prompt_file] selectors: model_names: [minimax-h3-768p] task_strategies: [diffusion_media_generation] @@ -1926,13 +1928,13 @@ suites: scoring: scorer: model_plugin_parity gates: - min_sample_pass_rate: 1.0 + min_sample_pass_rate: 0.8 ci: eligible: false lane: local_only notes: > - GB300-only full-profile validation. The model-owned comparator keeps - the checked-in visual thresholds; this suite adds no threshold override. + GB300-only sampled reference consistency. Use the model catalog limit + of ten; VBench benchmark scoring is outside this workload. - id: lfm2_model_card_sampling_parity description: > diff --git a/tools/perf_matrix.py b/tools/perf_matrix.py index 23f736e78c..efe5933b7e 100644 --- a/tools/perf_matrix.py +++ b/tools/perf_matrix.py @@ -1237,17 +1237,21 @@ def _resolved_adapter_options(baseline: Mapping[str, Any]) -> dict[str, Any]: configured = baseline.get("adapter_options", {}) options = dict(configured) if isinstance(configured, Mapping) else {} adapter = str(baseline.get("adapter", "")) - external_checkout = { - "upstream-elf": ("reference_repo", "TRTMC_ELF_REFERENCE_REPO"), - "upstream-lance": ("reference_repo", "TRTMC_LANCE_REFERENCE_REPO"), - "upstream-sana-wm": ( - "reference_repo", - "TRTMC_SANA_WM_REFERENCE_REPO", - ), - "pytorch-personaplex": ("official_repo", "PERSONAPLEX_OFFICIAL_REPO"), - }.get(adapter) - if external_checkout is not None: + declared_environment = baseline.get("adapter_environment", {}) + if not isinstance(declared_environment, Mapping): + raise PerfMatrixError("baseline adapter_environment must be an object") + external_checkouts = { + "upstream-elf": (("reference_repo", "TRTMC_ELF_REFERENCE_REPO"),), + "upstream-lance": (("reference_repo", "TRTMC_LANCE_REFERENCE_REPO"),), + "upstream-sana-wm": (("reference_repo", "TRTMC_SANA_WM_REFERENCE_REPO"),), + "pytorch-personaplex": (("official_repo", "PERSONAPLEX_OFFICIAL_REPO"),), + }.get(adapter, ()) + tuple(declared_environment.items()) + for external_checkout in external_checkouts: option_name, environment_name = external_checkout + if not isinstance(option_name, str) or not isinstance(environment_name, str): + raise PerfMatrixError( + "baseline adapter_environment must map option names to environment variable names" + ) environment_value = os.environ.get(environment_name, "").strip() if option_name not in options and environment_value: options[option_name] = environment_value diff --git a/tools/performance/catalog.py b/tools/performance/catalog.py index 7137b02d57..0195a5a762 100644 --- a/tools/performance/catalog.py +++ b/tools/performance/catalog.py @@ -302,6 +302,18 @@ def _validate_baseline(case: Mapping[str, Any]) -> None: raise PerformanceSuiteError( f"case {case['id']} task-reference adapter_options must be an object" ) + adapter_environment = baseline.get("adapter_environment", {}) + if not isinstance(adapter_environment, Mapping) or any( + not isinstance(option_name, str) + or not option_name + or not isinstance(environment_name, str) + or not environment_name + for option_name, environment_name in adapter_environment.items() + ): + raise PerformanceSuiteError( + f"case {case['id']} task-reference adapter_environment must map " + "option names to environment variable names" + ) expected_mode = ( "pytorch-eager" if adapter diff --git a/tools/prepare_media_validation_datasets.py b/tools/prepare_media_validation_datasets.py index b56bbc1efb..1e83d873e3 100644 --- a/tools/prepare_media_validation_datasets.py +++ b/tools/prepare_media_validation_datasets.py @@ -24,9 +24,15 @@ from PIL import Image, ImageOps +VBENCH_REPOSITORY = "https://github.com/Vchitect/VBench.git" +VBENCH_REVISION = "fd18b3d055cb0fc6f066ca90fe2c3c8cbb698490" VBENCH_SOURCE = ( - "https://github.com/Vchitect/VBench/blob/master/vbench/VBench_full_info.json" + f"https://github.com/Vchitect/VBench/blob/{VBENCH_REVISION}/" + "vbench/VBench_full_info.json" ) +VBENCH_INFO_SHA256 = "5dd2de80ee43cda750b2b72ea7023657c0b90d3702041c7e4608c65dbe50dccd" +VBENCH_LICENSE = "Apache-2.0" +VBENCH_MODEL_PLUGIN_DIR = "VBench-fd18b3d-model-plugin-v1" GEDIT_SOURCE = "https://huggingface.co/datasets/stepfun-ai/GEdit-Bench" GEDIT_REVISION = "50766778e2a737474c7e9bdf84cdce82c3ea3f4f" SANA_WM_SOURCE = "https://huggingface.co/datasets/Efficient-Large-Model/SANA-WM-Bench" @@ -83,7 +89,7 @@ def _safe_name(value: str) -> str: return cleaned or "sample" -def prepare_vbench(source_info: Path, output_root: Path, limit: int = 10) -> Path: +def _select_vbench_requests(source_info: Path, limit: int) -> list[dict[str, Any]]: """Select one unique official prompt from each review dimension.""" raw = json.loads(source_info.read_text(encoding="utf-8")) if not isinstance(raw, list): @@ -118,14 +124,23 @@ def prepare_vbench(source_info: Path, output_root: Path, limit: int = 10) -> Pat ) if limit < 1 or limit > len(selected): raise ValueError(f"VBench validation limit must be in [1, {len(selected)}]") - selected = selected[:limit] + return selected[:limit] + + +def prepare_vbench(source_info: Path, output_root: Path, limit: int = 10) -> Path: + """Write the shared diffusion-runner view of the VBench prompt slice.""" + source_info = source_info.resolve(strict=True) + if _sha256(source_info) != VBENCH_INFO_SHA256: + raise ValueError("VBench_full_info.json does not match the pinned revision") + selected = _select_vbench_requests(source_info, limit) return _write_json( output_root / "VBench" / "vbench_t2v_task_eval.json", { "dataset": "VBench text-to-video prompt suite (validation slice)", "source": VBENCH_SOURCE, "source_info_sha256": _sha256(source_info), - "license": "Apache-2.0", + "source_revision": VBENCH_REVISION, + "license": VBENCH_LICENSE, "sampling": "first unique prompt in each fixed review dimension", "request_count": len(selected), "requests": selected, @@ -133,6 +148,85 @@ def prepare_vbench(source_info: Path, output_root: Path, limit: int = 10) -> Pat ) +def prepare_vbench_model_plugin_dataset( + source_info: Path, + output_root: Path, + limit: int = 10, +) -> Path: + """Package the pinned VBench slice for prompt-file model plugins. + + The output is a versioned, portable dataset asset intended for NAS + publication. It contains no model outputs and runs no external evaluator. + """ + source_info = source_info.resolve(strict=True) + if _sha256(source_info) != VBENCH_INFO_SHA256: + raise ValueError("VBench_full_info.json does not match the pinned revision") + + output_dir = output_root / VBENCH_MODEL_PLUGIN_DIR + if output_dir.exists(): + raise FileExistsError(f"refusing to overwrite existing dataset: {output_dir}") + selected = _select_vbench_requests(source_info, limit) + output_dir.mkdir(parents=True) + + requests: list[dict[str, Any]] = [] + for row in selected: + sample_id = str(row["sample_id"]) + prompt_relative = Path("prompts") / f"{sample_id}.json" + _write_json( + output_dir / prompt_relative, + {"prompt": str(row["prompt"]), "seed": 0}, + ) + requests.append( + { + **row, + "inputs": {"prompt_file": prompt_relative.as_posix()}, + } + ) + + dataset_name = "VBench text-to-video prompt suite (TRTMC model-plugin slice)" + dataset_path = _write_json( + output_dir / "dataset.json", + { + "schema_version": "trtmc.model-plugin-validation/v1", + "dataset": dataset_name, + "version": f"{VBENCH_REVISION}-model-plugin-v1", + "source": VBENCH_REPOSITORY, + "source_revision": VBENCH_REVISION, + "source_info_sha256": VBENCH_INFO_SHA256, + "license": VBENCH_LICENSE, + "sampling": "first unique prompt in each fixed review dimension", + "request_count": len(requests), + "requests": requests, + }, + ) + + files = sorted(path for path in output_dir.rglob("*") if path.is_file()) + _write_json( + output_dir / "DATASET_MANIFEST.json", + { + "schema_version": "trtmc.dataset-manifest/v1", + "dataset": dataset_name, + "source": { + "repository": VBENCH_REPOSITORY, + "revision": VBENCH_REVISION, + "info_sha256": VBENCH_INFO_SHA256, + "license": VBENCH_LICENSE, + }, + "request_count": len(requests), + "path_policy": "manifest_relative", + "files": [ + { + "path": path.relative_to(output_dir).as_posix(), + "sha256": _sha256(path), + "bytes": path.stat().st_size, + } + for path in files + ], + }, + ) + return dataset_path + + def _english_gedit_rows(rows: Iterable[Mapping[str, Any]]) -> Iterator[Mapping[str, Any]]: for row in rows: language = str(row.get("instruction_language", "")).strip().lower() @@ -422,13 +516,25 @@ def prepare_media_datasets( *, output_root: Path, vbench_info: Path | None = None, + vbench_model_plugin: bool = False, gedit_source: str = "", sana_wm_root: Path | None = None, limit: int = 10, ) -> list[Path]: outputs: list[Path] = [] if vbench_info: - outputs.append(prepare_vbench(vbench_info, output_root, limit)) + if vbench_model_plugin: + outputs.append( + prepare_vbench_model_plugin_dataset( + vbench_info, + output_root, + limit, + ) + ) + else: + outputs.append(prepare_vbench(vbench_info, output_root, limit)) + elif vbench_model_plugin: + raise ValueError("VBench model-plugin preparation requires --vbench-info") if gedit_source: outputs.append(prepare_gedit(gedit_source, output_root, limit)) if sana_wm_root: diff --git a/tools/validation/engine.py b/tools/validation/engine.py index eb01578be8..def8a09743 100644 --- a/tools/validation/engine.py +++ b/tools/validation/engine.py @@ -12714,6 +12714,7 @@ def build_arg_parser() -> argparse.ArgumentParser: p = sub.add_parser("prepare-media") p.add_argument("--output-root", type=Path, required=True) p.add_argument("--vbench-info", type=Path) + p.add_argument("--vbench-model-plugin", action="store_true") p.add_argument("--gedit-source", default="") p.add_argument("--sana-wm-root", type=Path) p.add_argument("--limit", type=int, default=10) @@ -13202,6 +13203,7 @@ def cmd_prepare_media(args: argparse.Namespace) -> int: outputs = prepare_media_datasets( output_root=args.output_root, vbench_info=args.vbench_info, + vbench_model_plugin=args.vbench_model_plugin, gedit_source=args.gedit_source, sana_wm_root=args.sana_wm_root, limit=args.limit,