diff --git a/.github/workflows/audit-selfcheck.yml b/.github/workflows/audit-selfcheck.yml index 3954e6de..358585e9 100644 --- a/.github/workflows/audit-selfcheck.yml +++ b/.github/workflows/audit-selfcheck.yml @@ -21,7 +21,14 @@ jobs: run: pip install -e '.[dev,env]' - name: Run audit self-check run: labtrust audit-selfcheck --out ./audit_self_check + - name: Show audit diagnostics on failure + if: failure() + run: | + if test -f audit_self_check/AUDIT_SELF_CHECK.json; then + python -m json.tool audit_self_check/AUDIT_SELF_CHECK.json + fi - name: Upload audit self-check artifact + if: always() uses: actions/upload-artifact@v4 with: name: audit_self_check diff --git a/.github/workflows/pcs.yml b/.github/workflows/pcs.yml index 5932357d..1c7e5c57 100644 --- a/.github/workflows/pcs.yml +++ b/.github/workflows/pcs.yml @@ -65,6 +65,8 @@ on: jobs: pcs: runs-on: ubuntu-latest + env: + PCS_CORE_PATH: ${{ github.workspace }}/pcs-core defaults: run: shell: bash @@ -73,16 +75,47 @@ jobs: - uses: actions/checkout@v4 with: repository: SentinelOps-CI/pcs-core + ref: 9c971f5f9da8a424924dd8f48d6a3b71a1009e1b path: pcs-core - uses: actions/checkout@v4 with: - repository: SentinelOps-CI/pcs-bench + repository: fraware/pcs-bench + ref: 6092cccefee7841dfde1393e6881d433838a2252 path: pcs-bench - name: Apply LabTrust pcs-core schema profiles run: python scripts/apply_pcs_core_labtrust_schema_profiles.py --pcs-core pcs-core - uses: actions/setup-python@v5 with: python-version: "3.11" + - name: Prepare pinned pcs-core Python packaging metadata + run: | + python - <<'PY' + import shutil + import subprocess + from pathlib import Path + + root = Path("pcs-core") + revision = subprocess.check_output( + ["git", "-C", str(root), "rev-parse", "HEAD"], text=True + ).strip() + expected = "9c971f5f9da8a424924dd8f48d6a3b71a1009e1b" + if revision != expected: + raise RuntimeError(f"Unexpected pcs-core revision: {revision}") + metadata = root / "python" / "pyproject.toml" + original = metadata.read_text(encoding="utf-8") + previous = 'readme = "../README.md"' + if original.count(previous) != 1: + raise RuntimeError("Unknown upstream packaging metadata; review needed") + source_readme = root / "README.md" + destination = root / "python" / "README.md" + if not source_readme.is_file() or destination.exists(): + raise RuntimeError("Unexpected upstream README layout") + metadata.write_text( + original.replace(previous, 'readme = "README.md"'), encoding="utf-8" + ) + shutil.copyfile(source_readme, destination) + print(f"Applied README-only packaging repair to pinned pcs-core {revision}") + PY - name: Install pcs-core, pcs-bench, and LabTrust-Gym run: | pip install -e "pcs-core/python" diff --git a/.github/workflows/wheel-smoke.yml b/.github/workflows/wheel-smoke.yml index fed7e4f0..ba25c3f9 100644 --- a/.github/workflows/wheel-smoke.yml +++ b/.github/workflows/wheel-smoke.yml @@ -26,9 +26,22 @@ jobs: python -m venv .venv-wheel .venv-wheel/bin/pip install dist/*.whl .venv-wheel/bin/pip install '.[env]' # quick-eval uses env - .venv-wheel/bin/pip install pytest # audit-selfcheck doctor (extras_dev) and risk_register_contract_gate + .venv-wheel/bin/pip install 'pytest>=8,<10' 'pytest-timeout>=2.3,<3' # repo addopts includes --timeout - name: Audit selfcheck (doctor) run: .venv-wheel/bin/labtrust audit-selfcheck --out ./audit_out + - name: Show audit diagnostics on failure + if: failure() + run: | + if test -f audit_out/AUDIT_SELF_CHECK.json; then + python -m json.tool audit_out/AUDIT_SELF_CHECK.json + fi + - name: Upload audit diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: wheel-audit-diagnostics + path: audit_out/AUDIT_SELF_CHECK.json + if-no-files-found: warn - name: Validate policy run: .venv-wheel/bin/labtrust validate-policy - name: Quick eval (smoke) diff --git a/policy/schemas/risk_register_bundle.v0.1.schema.json b/policy/schemas/risk_register_bundle.v0.1.schema.json index 456b17fe..a2059764 100644 --- a/policy/schemas/risk_register_bundle.v0.1.schema.json +++ b/policy/schemas/risk_register_bundle.v0.1.schema.json @@ -137,7 +137,7 @@ "evidence_id": { "type": "string" }, "type": { "type": "string", - "enum": ["security_suite", "coordination_study", "coordination_pack", "official_pack", "safety_case", "bundle_verification", "other"] + "enum": ["security_suite", "coordination_study", "coordination_pack", "coord_risk_run", "official_pack", "safety_case", "bundle_verification", "other"] }, "path": { "type": "string", "description": "Repo-relative or run-relative path; empty or omitted for evidence gaps (status=missing)." }, "label": { "type": "string" }, diff --git a/src/labtrust_gym/cli/main.py b/src/labtrust_gym/cli/main.py index d1fa7f56..3abd111f 100644 --- a/src/labtrust_gym/cli/main.py +++ b/src/labtrust_gym/cli/main.py @@ -3081,6 +3081,8 @@ def _run_audit_selfcheck(args: argparse.Namespace) -> int: # 5. risk register contract gate (pytest) t0 = time.perf_counter() + contract_stdout = "" + contract_stderr = "" try: r = subprocess.run( [sys.executable, "-m", "pytest", "tests/test_risk_register_contract_gate.py", "-v", "--tb=short"], @@ -3090,11 +3092,21 @@ def _run_audit_selfcheck(args: argparse.Namespace) -> int: timeout=120, ) passed = r.returncode == 0 - except Exception: + if not passed: + contract_stdout = r.stdout[-8000:] + contract_stderr = r.stderr[-4000:] + except Exception as exc: passed = False - steps.append( - {"name": "risk_register_contract_gate", "pass": passed, "duration_s": round(time.perf_counter() - t0, 3)} - ) + contract_stderr = str(exc) + contract_step: dict[str, Any] = { + "name": "risk_register_contract_gate", + "pass": passed, + "duration_s": round(time.perf_counter() - t0, 3), + } + if not passed: + contract_step["output"] = contract_stdout + contract_step["error"] = contract_stderr + steps.append(contract_step) if not passed: all_pass = False diff --git a/src/labtrust_gym/cli/pcs.py b/src/labtrust_gym/cli/pcs.py index 33ab3311..976e3d57 100644 --- a/src/labtrust_gym/cli/pcs.py +++ b/src/labtrust_gym/cli/pcs.py @@ -10,25 +10,7 @@ from labtrust_gym.cli.console import get_console from labtrust_gym.config import get_repo_root -from labtrust_gym.pcs.attach_certificate import attach_certificate_files from labtrust_gym.pcs.demo import DEMO_SCENARIOS -from labtrust_gym.pcs.demo import run_demo as execute_pcs_demo -from labtrust_gym.pcs.export import export_pcs_bundle, export_runtime_receipt, export_trace -from labtrust_gym.pcs.handoff import export_handoff_bundle -from labtrust_gym.pcs.handoff_manifest import ( - emit_handoff_manifest, - emit_handoff_to_certifyedge, - emit_handoff_to_pf, -) -from labtrust_gym.pcs.benchmark_cases import generate_benchmark_cases, verify_benchmark_cases -from labtrust_gym.pcs.benchmark_reproducibility import benchmark_reproducibility -from labtrust_gym.pcs.failure_gallery import generate_failure_gallery -from labtrust_gym.pcs.regenerate_release_chain import regenerate_release_chain -from labtrust_gym.pcs.regenerate_release_protocol import regenerate_release_protocol -from labtrust_gym.pcs.release_protocol_producer import LABTRUST_PROTOCOL_PACKAGE_ARTIFACTS -from labtrust_gym.pcs.release_fragment import emit_labtrust_release_fragment -from labtrust_gym.pcs.status_policy import check_release_status_policy -from labtrust_gym.pcs.verify_release_protocol import verify_release_protocol from labtrust_gym.pcs.validate import PcsValidationError, validate_artifact_file, validate_run_dir @@ -46,6 +28,8 @@ def _print_validation_errors(errors: list[str]) -> None: def _run_demo(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.demo import run_demo as execute_pcs_demo + name = args.demo out = _resolve_path(args.out) if args.out else None try: @@ -73,6 +57,8 @@ def _run_demo(args: argparse.Namespace) -> int: def _run_export_trace(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.export import export_trace + run_dir = _resolve_path(args.run) out = _resolve_path(args.out) if not run_dir.is_dir(): @@ -91,6 +77,8 @@ def _run_export_trace(args: argparse.Namespace) -> int: def _run_export_runtime_receipt(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.export import export_runtime_receipt + run_dir = _resolve_path(args.run) out = _resolve_path(args.out) if not run_dir.is_dir(): @@ -109,6 +97,8 @@ def _run_export_runtime_receipt(args: argparse.Namespace) -> int: def _run_export_pcs(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.export import export_pcs_bundle + run_dir = _resolve_path(args.run) out = _resolve_path(args.out) if not run_dir.is_dir(): @@ -127,6 +117,8 @@ def _run_export_pcs(args: argparse.Namespace) -> int: def _run_attach_certificate(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.attach_certificate import attach_certificate_files + bundle_path = _resolve_path(args.bundle) cert_path = _resolve_path(args.certificate) out_path = _resolve_path(args.out) @@ -167,6 +159,8 @@ def _run_validate_pcs(args: argparse.Namespace) -> int: def _run_emit_release_fragment(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.release_fragment import emit_labtrust_release_fragment + release_dir = _resolve_path(args.release_dir) out = _resolve_path(args.out) if args.out else release_dir / "labtrust_release_fragment.json" try: @@ -179,6 +173,8 @@ def _run_emit_release_fragment(args: argparse.Namespace) -> int: def _run_emit_handoff(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.handoff_manifest import emit_handoff_manifest + out = _resolve_path(args.out) try: emit_handoff_manifest( @@ -198,6 +194,8 @@ def _run_emit_handoff(args: argparse.Namespace) -> int: def _run_verify_release_protocol(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.verify_release_protocol import verify_release_protocol + release_dir = _resolve_path(args.release_dir) pcs_core = _resolve_path(args.pcs_core) if args.pcs_core else None try: @@ -216,6 +214,8 @@ def _run_verify_release_fixtures(args: argparse.Namespace) -> int: def _run_emit_handoff_to_certifyedge(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.handoff_manifest import emit_handoff_to_certifyedge + out = _resolve_path(args.out) try: emit_handoff_to_certifyedge( @@ -233,6 +233,8 @@ def _run_emit_handoff_to_certifyedge(args: argparse.Namespace) -> int: def _run_emit_handoff_to_pf(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.handoff_manifest import emit_handoff_to_pf + out = _resolve_path(args.out) try: emit_handoff_to_pf( @@ -248,6 +250,10 @@ def _run_emit_handoff_to_pf(args: argparse.Namespace) -> int: def _run_regenerate_release_protocol(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.regenerate_release_protocol import regenerate_release_protocol + from labtrust_gym.pcs.verify_release_protocol import verify_release_protocol + from labtrust_gym.pcs.release_protocol_producer import LABTRUST_PROTOCOL_PACKAGE_ARTIFACTS + out = _resolve_path(args.out) pcs_core = _resolve_path(args.pcs_core) if args.pcs_core else None ce_spec = _resolve_path(args.certifyedge_spec) if args.certifyedge_spec else None @@ -296,6 +302,8 @@ def _run_regenerate_release_protocol(args: argparse.Namespace) -> int: def _run_check_status_policy(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.status_policy import check_release_status_policy + release_dir = _resolve_path(args.release_dir) try: profile_path = _resolve_path(args.workflow_profile) if getattr(args, "workflow_profile", None) else None @@ -314,6 +322,8 @@ def _run_check_status_policy(args: argparse.Namespace) -> int: def _run_generate_benchmark_cases(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.benchmark_cases import generate_benchmark_cases + out = _resolve_path(args.out) release_dir = _resolve_path(args.release_dir) if args.release_dir else None validate_root = ( @@ -361,6 +371,8 @@ def _run_generate_benchmark_cases(args: argparse.Namespace) -> int: def _run_benchmark_reproducibility(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.benchmark_reproducibility import benchmark_reproducibility + out = _resolve_path(args.out) pcs_core = _resolve_path(args.pcs_core) if args.pcs_core else None release_dir = _resolve_path(args.release_dir) if args.release_dir else None @@ -442,6 +454,8 @@ def _resolve_pcs_core_root(explicit: Path | None) -> Path | None: def _run_verify_benchmark_cases(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.benchmark_cases import verify_benchmark_cases + root = _resolve_path(args.benchmark_dir) pcs_core = _resolve_pcs_core_root( _resolve_path(args.pcs_core) if getattr(args, "pcs_core", None) else None @@ -495,6 +509,8 @@ def _run_verify_benchmark_cases(args: argparse.Namespace) -> int: def _run_generate_failure_gallery(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.failure_gallery import generate_failure_gallery + out = _resolve_path(args.out) release_dir = _resolve_path(args.release_dir) if args.release_dir else None try: @@ -518,6 +534,8 @@ def _run_generate_failure_gallery(args: argparse.Namespace) -> int: def _run_regenerate_release_chain(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.regenerate_release_chain import regenerate_release_chain + out = _resolve_path(args.out) pcs_core = _resolve_path(args.pcs_core) if args.pcs_core else None ce_spec = _resolve_path(args.certifyedge_spec) if args.certifyedge_spec else None @@ -538,6 +556,8 @@ def _run_regenerate_release_chain(args: argparse.Namespace) -> int: def _run_export_handoff(args: argparse.Namespace) -> int: + from labtrust_gym.pcs.handoff import export_handoff_bundle + out = _resolve_path(args.out) try: manifest = export_handoff_bundle( diff --git a/src/labtrust_gym/pcs/demo.py b/src/labtrust_gym/pcs/demo.py index 1e973a0b..00e1fc3a 100644 --- a/src/labtrust_gym/pcs/demo.py +++ b/src/labtrust_gym/pcs/demo.py @@ -5,8 +5,6 @@ from pathlib import Path from labtrust_gym.config import get_repo_root -from labtrust_gym.pcs.workflows.qc_release import FAILURE_CASES, run_failure_demo -from labtrust_gym.pcs.workflows.registry import get_workflow DEMO_SCENARIOS: dict[str, tuple[str, str]] = { "qc-release": ("valid_workflow.yaml", "runs/qc-release"), @@ -33,6 +31,10 @@ def run_demo( policy_root: Path | None = None, deterministic: bool | None = None, ) -> Path: + # Import workflow engines only when a PCS demo is actually requested. + from labtrust_gym.pcs.workflows.qc_release import FAILURE_CASES, run_failure_demo + from labtrust_gym.pcs.workflows.registry import get_workflow + if demo_name not in DEMO_SCENARIOS: raise ValueError(f"unknown demo {demo_name!r}; choose from {list(DEMO_SCENARIOS)}") root = policy_root or get_repo_root() diff --git a/src/labtrust_gym/verifier_assurance/reward/composition.py b/src/labtrust_gym/verifier_assurance/reward/composition.py index 98058d5b..b1d119ee 100644 --- a/src/labtrust_gym/verifier_assurance/reward/composition.py +++ b/src/labtrust_gym/verifier_assurance/reward/composition.py @@ -6,6 +6,7 @@ import hashlib from typing import Any, Mapping +from labtrust_gym.config import get_repo_root from labtrust_gym.errors import PolicyLoadError from labtrust_gym.policy.loader import load_json, validate_against_schema from labtrust_gym.util.json_utils import canonical_json @@ -26,20 +27,9 @@ "safety_violation_penalty", ) -_SCHEMA_PATH = ( - Path(__file__).resolve().parents[4] - / "policy" - / "schemas" - / "verifier_assurance" - / "RewardCompositionPolicy.v1.schema.json" -) -_ENVELOPE_SCHEMA_PATH = ( - Path(__file__).resolve().parents[4] - / "policy" - / "schemas" - / "pcs" - / "RewardEvidenceEnvelope.v1.schema.json" -) +def _reward_schema_path(*parts: str) -> Path: + """Resolve a policy schema using package, checkout, or explicit configuration.""" + return get_repo_root() / "policy" / "schemas" / Path(*parts) class RewardCompositionError(ValueError): @@ -52,8 +42,9 @@ def compute_policy_digest(policy: dict[str, Any]) -> str: def validate_composition_policy(policy: dict[str, Any]) -> dict[str, Any]: + schema_path = _reward_schema_path("verifier_assurance", "RewardCompositionPolicy.v1.schema.json") try: - validate_against_schema(policy, load_json(_SCHEMA_PATH), path=_SCHEMA_PATH) + validate_against_schema(policy, load_json(schema_path), path=schema_path) except PolicyLoadError as exc: raise RewardCompositionError(str(exc)) from exc weights = policy.get("weights") or {} @@ -209,8 +200,9 @@ def build_reward_evidence_envelope( "public_decision": public_decision, "claim_boundary": CLAIM_BOUNDARY, } + schema_path = _reward_schema_path("pcs", "RewardEvidenceEnvelope.v1.schema.json") try: - validate_against_schema(envelope, load_json(_ENVELOPE_SCHEMA_PATH), path=_ENVELOPE_SCHEMA_PATH) + validate_against_schema(envelope, load_json(schema_path), path=schema_path) except PolicyLoadError as exc: raise RewardCompositionError(str(exc)) from exc return envelope diff --git a/tests/fixtures/risk_register_bundle_ui_fixtures.v0.1.json b/tests/fixtures/risk_register_bundle_ui_fixtures.v0.1.json index 14f64f19..0208a52b 100644 --- a/tests/fixtures/risk_register_bundle_ui_fixtures.v0.1.json +++ b/tests/fixtures/risk_register_bundle_ui_fixtures.v0.1.json @@ -619,7 +619,7 @@ "method_risk_matrix": "8a44089fce4ac7e95cdc1040987fc5bddfd7d4b4fc6348dbc7fc851a753fd195", "risk_registry": "74c6a81361dd50e0d291e8d21cdb21ce0d2996bbf5f56ae93501d8dc51280b45", "safety_case_claims": "8de856fa7fb6b2b3a1aed725971cb2fc9cfd20b2210dc9a1565e6445a998d749", - "security_attack_suite": "b994010c6d13810b66621a8c3169d13aed8cc144584af50ef66976d7bc18c1e0" + "security_attack_suite": "bada4e173a03dff42b598203cf09df2705699a16a7c81cd73b700726e5fdc829" }, "reproduce": [ { @@ -1283,4 +1283,4 @@ "typical_failure_mode": "Malicious input to tool; abuse of optional parameters; bypass of checks." } ] -} \ No newline at end of file +} diff --git a/tests/test_cli_optional_pcs_import_boundary.py b/tests/test_cli_optional_pcs_import_boundary.py new file mode 100644 index 00000000..12f21a2f --- /dev/null +++ b/tests/test_cli_optional_pcs_import_boundary.py @@ -0,0 +1,27 @@ +"""Regression: core CLI initialization must not import the optional PCS package.""" + +from __future__ import annotations + +import subprocess +import sys + + +def test_cli_help_works_without_pcs_core() -> None: + """The default CLI command registry is available with only base dependencies.""" + script = """ +import sys +# Block importing pcs_core even if a developer installed the PCS optional extra. +sys.modules["pcs_core"] = None +from labtrust_gym.cli.main import main +sys.argv = ["labtrust", "--help"] +main() +""" + result = subprocess.run( + [sys.executable, "-c", script], + text=True, + capture_output=True, + timeout=40, + check=False, + ) + assert result.returncode == 0, result.stderr + assert "usage:" in result.stdout.lower() diff --git a/tests/test_no_placeholders.py b/tests/test_no_placeholders.py index 77c0a9be..cd0c64a1 100644 --- a/tests/test_no_placeholders.py +++ b/tests/test_no_placeholders.py @@ -162,3 +162,56 @@ def test_clean_tree_no_violations(tmp_path: Path) -> None: ) violations = scan_root(tmp_path) assert not violations + + +def test_exception_types_are_distinct_from_unimplemented_functions(tmp_path: Path) -> None: + """A handler and a custom exception class are legal; an unfinished function fails.""" + _write_tree( + tmp_path, + { + "src/types.py": ( + "class Unavailable(NotImplementedError):\n pass\n" + "def handle():\n" + " try:\n return 1\n" + " except (ValueError, NotImplementedError):\n return 0\n" + ), + "src/unimplemented.py": "def unfinished():\n raise NotImplementedError('TODO')\n", + }, + ) + problems = [ + _norm_path(p, tmp_path) + for p, _, token in scan_root(tmp_path) + if token == "NotImplementedError" + ] + assert problems == ["src/unimplemented.py"] + + +def test_template_scaffold_is_excluded_without_excluding_production(tmp_path: Path) -> None: + _write_tree( + tmp_path, + { + "templates/pcs_workflow_template/workflow.py": "raise NotImplementedError('template')\n", + "src/actual.py": "raise NotImplementedError('production')\n", + }, + ) + problems = [_norm_path(p, tmp_path) for p, _, _ in scan_root(tmp_path)] + assert problems == ["src/actual.py"] + + +def test_schema_fields_are_precisely_exempt_and_other_policy_markers_fail( + tmp_path: Path, +) -> None: + _write_tree( + tmp_path, + { + "policy/schemas/pcs/LabtrustBenchmarkExtension.v0.schema.json": ( + '"placeholder_fields": {\n' + '"description": "Documents placeholder benchmark_case fields when pcs-core expects null on valid cases.",\n' + '"description": "Unexpected placeholder claim.",\n' + ), + }, + ) + failures = [ + (line, token) for _, line, token in scan_root(tmp_path) if token == "placeholder" + ] + assert failures == [(3, "placeholder")] diff --git a/tests/test_risk_register_bundle.py b/tests/test_risk_register_bundle.py index 82e523bc..f5a5112e 100644 --- a/tests/test_risk_register_bundle.py +++ b/tests/test_risk_register_bundle.py @@ -271,3 +271,26 @@ def test_coord_summary_includes_metrics(tmp_path: Path) -> None: assert len(rows) >= 1 row = rows[0] assert "sec.attack_success_rate" in row or "robustness.resilience_score" in row + +def test_coord_risk_evidence_type_is_supported_without_accepting_unknown_types() -> None: + """Preserve the evidence type emitted by the strict coordination-risk producer.""" + root = _repo_root() + bundle = build_risk_register_bundle( + root, + run_dirs=[], + include_generated_at=False, + include_git_hash=False, + ) + entry = { + "evidence_id": "EVID-COORD-RISK-SYNTHETIC", + "type": "coord_risk_run", + "status": "present", + "path": "runs/coord_risk_fixture/coord_risk_results.json", + "risk_ids": ["R-SYS-001"], + } + bundle["evidence"].append(entry) + assert validate_bundle_against_schema(bundle, root) == [] + + entry["type"] = "unrecognized_evidence_class" + assert validate_bundle_against_schema(bundle, root) + diff --git a/tests/verifier_assurance/test_va03_reward_composition.py b/tests/verifier_assurance/test_va03_reward_composition.py index f84fb0e2..3641150f 100644 --- a/tests/verifier_assurance/test_va03_reward_composition.py +++ b/tests/verifier_assurance/test_va03_reward_composition.py @@ -3,11 +3,13 @@ from __future__ import annotations import copy +from pathlib import Path import pytest from labtrust_gym.verifier_assurance.reward.composition import ( COMPONENT_VOCAB, + _reward_schema_path, RewardComposer, RewardCompositionError, apply_legacy_rewards, @@ -116,3 +118,17 @@ def test_emit_component_breakdown_when_enabled() -> None: assert rewards["ops_0"] == 1.0 assert breakdown is not None assert breakdown["ops_0"]["operational_success"] == 1.0 + +def test_reward_schema_paths_respect_explicit_policy_root(monkeypatch: pytest.MonkeyPatch) -> None: + """An installed module must resolve schemas through its configured policy root.""" + repo = Path(__file__).resolve().parents[2] + monkeypatch.setenv("LABTRUST_POLICY_DIR", str(repo / "policy")) + composition_schema = _reward_schema_path( + "verifier_assurance", "RewardCompositionPolicy.v1.schema.json" + ) + envelope_schema = _reward_schema_path("pcs", "RewardEvidenceEnvelope.v1.schema.json") + assert composition_schema.is_file() + assert envelope_schema.is_file() + assert composition_schema.is_relative_to(repo / "policy") + assert envelope_schema.is_relative_to(repo / "policy") + diff --git a/tools/no_placeholders.py b/tools/no_placeholders.py index 73fe2622..8c642e55 100644 --- a/tools/no_placeholders.py +++ b/tools/no_placeholders.py @@ -8,6 +8,7 @@ from __future__ import annotations +import ast import re import subprocess import sys @@ -85,7 +86,12 @@ def _is_excluded_path(rel_path: str) -> bool: SECRET_SCRUBBER_NAME = "secret_scrubber.py" # Files that define or test the gate itself: skip to avoid self-fail -EXCLUDE_FILES = {"tools/no_placeholders.py", "tests/test_no_placeholders.py"} +EXCLUDE_FILES = { + "tools/no_placeholders.py", + "tests/test_no_placeholders.py", + # This published integration scaffold intentionally contains abstract method bodies. + "templates/pcs_workflow_template/workflow.py", +} # (rel_path, pattern): allow these global-fail patterns in config (e.g. coverage exclude_lines) # agent_api.py: base class default act() raises NotImplementedError; subclasses must implement. @@ -135,15 +141,53 @@ def _is_under_tests_for_skip(rel_path: str) -> bool: return rel_path.startswith("tests/") +def _non_stub_not_implemented_lines(path: Path, lines: list[str]) -> set[int]: + """Recognize exception *types* in handlers and class bases, never stub raises.""" + if path.suffix.lower() != ".py": + return set() + try: + tree = ast.parse("\n".join(lines) + "\n") + except SyntaxError: + return set() + + def exception_names(node: ast.expr | None) -> list[ast.Name]: + if isinstance(node, ast.Name): + return [node] + if isinstance(node, ast.Tuple): + return [name for item in node.elts for name in exception_names(item)] + return [] + + allowed: set[int] = set() + for node in ast.walk(tree): + if isinstance(node, ast.ExceptHandler): + refs = exception_names(node.type) + elif isinstance(node, ast.ClassDef): + refs = [name for base in node.bases for name in exception_names(base)] + else: + continue + allowed.update( + name.lineno for name in refs if name.id == "NotImplementedError" + ) + return allowed + + def _check_global_fails(path: Path, root: Path, lines: list[str]) -> list[tuple[int, str]]: violations: list[tuple[int, str]] = [] rel_path = _norm_path(path, root) if _is_under_tests_for_skip(rel_path): return [] + allowed_exception_lines = _non_stub_not_implemented_lines(path, lines) for i, line in enumerate(lines, start=1): for pat in GLOBAL_FAIL_PATTERNS: - if pat in line and (rel_path, pat) not in ALLOWED_GLOBAL_VIOLATIONS: - violations.append((i, pat)) + if pat not in line or (rel_path, pat) in ALLOWED_GLOBAL_VIOLATIONS: + continue + if ( + pat == "NotImplementedError" + and i in allowed_exception_lines + and not re.search(r"\braise\s+NotImplementedError\b", line) + ): + continue + violations.append((i, pat)) return violations @@ -174,7 +218,15 @@ def _check_word_placeholder_stub(path: Path, root: Path, lines: list[str]) -> li if _is_secret_scrubber(rel_path) and "placeholder" in line.lower(): continue if "placeholder" in line.lower() and not allow_placeholder: - violations.append((i, "placeholder")) + # Exact schema declarations describe compatibility metadata; + # unrelated unfinished policy text still fails the scan. + canonical_schema = "policy/schemas/pcs/LabtrustBenchmarkExtension.v0.schema.json" + recognized_fields = { + '"placeholder_fields": {', + '"description": "Documents placeholder benchmark_case fields when pcs-core expects null on valid cases.",', + } + if rel_path != canonical_schema or line.strip() not in recognized_fields: + violations.append((i, "placeholder")) # In tests/, allow capital-S "Stub" (e.g. StubTask); fail only on lowercase "stub" if "stub" in line.lower() and not allow_stub: if rel_path.startswith("tests/") and re.search(r"\bstub\b", line):