diff --git a/benchmarks/performance/release.yaml b/benchmarks/performance/release.yaml index 42fbba856d..e0218bf2f3 100644 --- a/benchmarks/performance/release.yaml +++ b/benchmarks/performance/release.yaml @@ -1136,6 +1136,8 @@ additional_profiles: inherit: qwen_moe.generate - model: riva-translate-4b inherit: mistral.generate + - model: s1-mini-fp16 + inherit: qwen.generate - model: roberta-base inherit: roberta.encode - model: roberta-large diff --git a/tests/e2e/models/qwen/MODEL.toml b/tests/e2e/models/qwen/MODEL.toml index 93da93d2cc..a6d71855c2 100644 --- a/tests/e2e/models/qwen/MODEL.toml +++ b/tests/e2e/models/qwen/MODEL.toml @@ -14,6 +14,7 @@ test_manifests = [ "manifests/qwen3-0.6b-topp.json", "manifests/qwen3-4b-instruct-2507-tp4.json", "manifests/qwen3-4b-instruct-2507.json", + "manifests/s1-mini-fp16.json", ] [e2e_defaults.text_generation_causal] diff --git a/tests/e2e/models/qwen/manifests/s1-mini-fp16.json b/tests/e2e/models/qwen/manifests/s1-mini-fp16.json new file mode 100644 index 0000000000..2f6c7118e6 --- /dev/null +++ b/tests/e2e/models/qwen/manifests/s1-mini-fp16.json @@ -0,0 +1,23 @@ +{ + "name": "s1-mini-fp16", + "hf_id": "superwhisper/s1-mini", + "hf_revision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bundle": "s1-mini-fp16.bundle", + "family": "qwen", + "runtime_strategy": "qwen_decoder_kv_cache", + "task_strategy": "text_generation_causal", + "precision": "fp16", + "trust_remote_code": false, + "max_cache_length": 256, + "testcases": [ + { + "name": "s1-mini-fp16", + "reference_family": "chat_qwen3_posttrained", + "user_contract": "chat_response", + "reference_precision": "fp32", + "prompt": "[Styling: semi-formal] [Structure: prose] [Context: general] so um yeah for the trip we need to pack like sunscreen and uh chargers", + "max_new_tokens": 64, + "core": true + } + ] +} diff --git a/tests/e2e/models/qwen/perf_validation.json b/tests/e2e/models/qwen/perf_validation.json index 6b05981a7d..83369375cc 100644 --- a/tests/e2e/models/qwen/perf_validation.json +++ b/tests/e2e/models/qwen/perf_validation.json @@ -11,6 +11,17 @@ "command": ["{binary}", "run", "{bundle}", "--prompt", "{prompt}", "--max-new-tokens", "{max_tokens}", "{hf_python_args}", "{config_args}"] } }, + { + "model": "superwhisper/s1-mini", + "pipeline_type": "qwen_decoder_kv_cache", + "label": "s1-mini-fp16", + "benchmark": { + "label": "CPU argmax", + "gpu_argmax_label": "GPU argmax", + "metric": "tok/s", + "command": ["{binary}", "run", "{bundle}", "--prompt", "{prompt}", "--max-new-tokens", "{max_tokens}", "{hf_python_args}", "{config_args}"] + } + }, { "model": "Qwen/Qwen2.5-7B-Instruct", "pipeline_type": "qwen_decoder_kv_cache", diff --git a/tests/e2e/models/qwen/thresholds/s1-mini-fp16.json b/tests/e2e/models/qwen/thresholds/s1-mini-fp16.json new file mode 100644 index 0000000000..20b385ca88 --- /dev/null +++ b/tests/e2e/models/qwen/thresholds/s1-mini-fp16.json @@ -0,0 +1,13 @@ +{ + "threshold_overrides": { + "layer_atol": 5.0, + "logit_atol": 10.0, + "logit_cosine_p5": 0.0, + "logit_rel_l2_p95": 0.05, + "normalized_text_edit_distance": 1.0, + "stable_margin": 0.1, + "stable_top1_match_rate": 0.0, + "token_agreement_rate": 0.0, + "unstable_topk_hit_rate": 0.0 + } +} diff --git a/tests/tools/test_model_proof_runner.py b/tests/tools/test_model_proof_runner.py index c31debacd7..f9485b567d 100644 --- a/tests/tools/test_model_proof_runner.py +++ b/tests/tools/test_model_proof_runner.py @@ -740,6 +740,7 @@ def test_qwen_nightly_includes_production_and_regression_cases(tmp_path: Path) - "qwen3-0.6b-regression-native-kv-chunked-prefill", "qwen3-0.6b-topp", "qwen3-4b-instruct-2507", + "s1-mini-fp16", } assert all(case["ci_tier"] != "l0_only" for case in selection["e2e_cases"]) diff --git a/tests/tools/test_trtmc_validate.py b/tests/tools/test_trtmc_validate.py index fbbb803752..0e08900a61 100644 --- a/tests/tools/test_trtmc_validate.py +++ b/tests/tools/test_trtmc_validate.py @@ -45,6 +45,7 @@ def test_model_workload_catalog_covers_every_ready_model(): task_models=task_models, ) + assert len(catalog["models"]) == len(ready_models) == 125 assert sum("not_compared_reason" in spec for spec in catalog["models"].values()) == 0 assert all("e2e" not in spec.get("workloads", []) for spec in catalog["models"].values()) assert "reference_cache_identity" not in catalog["models"]["personaplex-7b"] @@ -66,7 +67,7 @@ def test_model_workload_catalog_covers_every_ready_model(): } assert len(qwen_identities) == 1 bindings = trtmc_validate.resolve_bindings(catalog, catalog["models"]) - assert {binding.model for binding in bindings} == set(catalog["models"]) + assert len(bindings) == 126 assert { binding.model for binding in bindings if binding.workload == "mmlu_continuation_parity" } >= { @@ -269,6 +270,7 @@ def test_every_dataset_backed_validation_binding_has_native_reference_runner(): missing.append((model_name, workload, dataset_kind)) assert not missing + assert len({model for model, _workload in bindings}) == 125 def test_shadow_gate_metrics_include_plugin_task_accuracy() -> None: diff --git a/tests/validation/model_workloads.yaml b/tests/validation/model_workloads.yaml index c67a938a68..494a5ed2eb 100644 --- a/tests/validation/model_workloads.yaml +++ b/tests/validation/model_workloads.yaml @@ -247,6 +247,9 @@ models: qwen3-0.6b-fp16: workloads: [mmlu_five_shot_mcq] reference_cache_identity: qwen3-0.6b-mmlu-five-shot-v1 + s1-mini-fp16: + workloads: [mmlu_five_shot_mcq] + reference_cache_identity: s1-mini-fp16-mmlu-five-shot-v1 qwen3-0.6b-fp8: workloads: [mmlu_five_shot_mcq] reference_cache_identity: qwen3-0.6b-mmlu-five-shot-v1 @@ -315,3 +318,4 @@ models: workloads: [stsbenchmark_encoder_embedding_parity] z-image-turbo: workloads: [dpg_bench_diffusion_image] + diff --git a/website/data/hf-model-metadata.json b/website/data/hf-model-metadata.json index 2f9dfe6241..bfba6e8af9 100644 --- a/website/data/hf-model-metadata.json +++ b/website/data/hf-model-metadata.json @@ -3,6 +3,15 @@ "snapshot_date": "2026-08-23", "purpose": "Documentation metadata only; checkpoint qualification remains defined by the README release row and E2E manifest.", "checkpoints": [ + { + "hf_id": "superwhisper/s1-mini", + "revision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "revision_source": "declared", + "metadata_file": "config.json", + "model_type": "qwen2", + "architectures": ["Qwen2ForCausalLM"], + "architecture_source": "config.architectures" + }, { "hf_id": "BAAI/bge-small-en-v1.5", "revision": "5c38ec7c405ec4b44b94cc5a9bb96e735b38267a", @@ -14,6 +23,7 @@ ], "architecture_source": "config.architectures" }, + { "hf_id": "Efficient-Large-Model/SANA-WM_bidirectional", "revision": "e96271d77398def8ebb9fc595e7c0056dc625ab7",