diff --git a/tests/e2e/waives.txt b/tests/e2e/waives.txt index 640250d432..e0af22e582 100644 --- a/tests/e2e/waives.txt +++ b/tests/e2e/waives.txt @@ -1,6 +1,3 @@ # Waived E2E tests — centralized skip/xfail tracking. # Format: model-name SKIP|XFAIL (reason) # Platform-specific: GB300/model-name XFAIL (reason) - -eagle-embed-vl-1b-v2 SKIP (HF repo 404) -eagle-rerank-vl-1b-v2 SKIP (HF repo 404) diff --git a/tests/tools/test_e2e_waives.py b/tests/tools/test_e2e_waives.py index d9bc1e54cd..dcc3f33225 100644 --- a/tests/tools/test_e2e_waives.py +++ b/tests/tools/test_e2e_waives.py @@ -5,7 +5,10 @@ from __future__ import annotations +from pathlib import Path + from tests import test_e2e +from tests.e2e_harness.manifest_loader import get_case_names def test_load_waives_filters_with_explicit_platform(monkeypatch, tmp_path) -> None: @@ -45,3 +48,46 @@ def test_load_waives_without_platform_ignores_prefixed_entries(monkeypatch, tmp_ waives = test_e2e._load_waives() assert waives == {"model-shared": ("XFAIL", "shared waive")} + + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_WAIVES_FILE = _REPO_ROOT / "tests" / "e2e" / "waives.txt" + + +def _declared_case_names() -> set[str]: + return set(get_case_names()) + + +def test_every_waive_names_a_declared_case() -> None: + """A waive for a name no E2E testcase declares is silently inert. + + The runtime resolves a waive against ``case.name``, so the namespace to + check against is the child testcase names, not the top-level manifest + names -- 50 of the former are not the latter. ``_load_waives`` skips any + line it cannot parse and reports nothing when a name stops matching, so + such an entry quietly stops doing anything. If the name is ever reused, + the waive springs back and skips a case no one intended to waive. + """ + declared = _declared_case_names() + assert declared, "no E2E testcases were discovered" + + # Platform-prefixed waives are only visible when that platform is + # selected, so collect across every platform the file mentions. + platforms = {""} + for line in _WAIVES_FILE.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + name_part = line.split(None, 1)[0] + if "/" in name_part: + platforms.add(name_part.split("/", 1)[0]) + + waived: set[str] = set() + for platform in platforms: + waived.update(test_e2e._load_waives(platform)) + + orphans = sorted(waived - declared) + assert not orphans, ( + "waives.txt names cases that no E2E testcase declares, so these waives " + f"are silently inert: {orphans}" + )