Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions tests/e2e/waives.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
# Waived E2E tests — centralized skip/xfail tracking.
# Format: model-name SKIP|XFAIL (reason)
# Platform-specific: GB300/model-name XFAIL (reason)

eagle-embed-vl-1b-v2 SKIP (HF repo 404)
eagle-rerank-vl-1b-v2 SKIP (HF repo 404)
46 changes: 46 additions & 0 deletions tests/tools/test_e2e_waives.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@

from __future__ import annotations

from pathlib import Path

from tests import test_e2e
from tests.e2e_harness.manifest_loader import get_case_names


def test_load_waives_filters_with_explicit_platform(monkeypatch, tmp_path) -> None:
Expand Down Expand Up @@ -45,3 +48,46 @@ def test_load_waives_without_platform_ignores_prefixed_entries(monkeypatch, tmp_
waives = test_e2e._load_waives()

assert waives == {"model-shared": ("XFAIL", "shared waive")}


_REPO_ROOT = Path(__file__).resolve().parents[2]
_WAIVES_FILE = _REPO_ROOT / "tests" / "e2e" / "waives.txt"


def _declared_case_names() -> set[str]:
return set(get_case_names())


def test_every_waive_names_a_declared_case() -> None:
"""A waive for a name no E2E testcase declares is silently inert.

The runtime resolves a waive against ``case.name``, so the namespace to
check against is the child testcase names, not the top-level manifest
names -- 50 of the former are not the latter. ``_load_waives`` skips any
line it cannot parse and reports nothing when a name stops matching, so
such an entry quietly stops doing anything. If the name is ever reused,
the waive springs back and skips a case no one intended to waive.
"""
declared = _declared_case_names()
assert declared, "no E2E testcases were discovered"

# Platform-prefixed waives are only visible when that platform is
# selected, so collect across every platform the file mentions.
platforms = {""}
for line in _WAIVES_FILE.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
name_part = line.split(None, 1)[0]
if "/" in name_part:
platforms.add(name_part.split("/", 1)[0])

waived: set[str] = set()
for platform in platforms:
waived.update(test_e2e._load_waives(platform))

orphans = sorted(waived - declared)
assert not orphans, (
"waives.txt names cases that no E2E testcase declares, so these waives "
f"are silently inert: {orphans}"
)
Loading