Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
cc30d02
fix(cli): load optional PCS implementations only for PCS subcommands
fraware Oct 11, 2026
c93840e
fix(pcs): defer workflow imports until demo execution
fraware Oct 11, 2026
cabf162
test(cli): guard base installation against optional PCS imports
fraware Oct 11, 2026
cfafd44
diagnostic(ci): retain failing contract-test output in audit report
fraware Oct 11, 2026
5cafdc9
diagnostic(ci): expose failing wheel audit without changing gate
fraware Oct 11, 2026
6d5bc52
diagnostic(ci): preserve and expose failed audit evidence
fraware Oct 11, 2026
91965b6
fix(ci): use actual public fraware/pcs-bench repository
fraware Oct 11, 2026
7580cc2
fix(ci): pin PCS components and repair incompatible upstream packagin…
fraware Oct 11, 2026
8f91189
test(snapshot): align security-suite policy fingerprint with current …
fraware Oct 11, 2026
3a785e6
fix(audit): classify real unimplemented code separately from exceptio…
fraware Oct 11, 2026
2c0d741
fix(audit): exact canonical schema terms and validate scanner syntax
fraware Oct 11, 2026
fec5409
test(audit): reject real stubs while permitting handlers, declared sc…
fraware Oct 11, 2026
0825656
test(audit): correct generated fixture newline escapes
fraware Oct 11, 2026
96c55c4
fix(wheel): install pytest-timeout required by strict root pytest con…
fraware Oct 11, 2026
073d773
fix(ci): bind PCS tests to the pinned checked-out schema repository
fraware Oct 11, 2026
c742a01
fix(audit): remove duplicated malformed trailing scanner content
fraware Oct 11, 2026
46e4e77
fix(schema): admit typed coordination-risk run evidence produced by s…
fraware Oct 11, 2026
191eb91
test(schema): accept coordination-risk evidence and reject unknown ev…
fraware Oct 11, 2026
910d699
fix(wheel): resolve reward policy schemas through configured policy root
fraware Oct 11, 2026
909eaa4
test(wheel): verify both reward schemas honor configured policy root
fraware Oct 11, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .github/workflows/audit-selfcheck.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,14 @@ jobs:
run: pip install -e '.[dev,env]'
- name: Run audit self-check
run: labtrust audit-selfcheck --out ./audit_self_check
- name: Show audit diagnostics on failure
if: failure()
run: |
if test -f audit_self_check/AUDIT_SELF_CHECK.json; then
python -m json.tool audit_self_check/AUDIT_SELF_CHECK.json
fi
- name: Upload audit self-check artifact
if: always()
uses: actions/upload-artifact@v4
with:
name: audit_self_check
Expand Down
35 changes: 34 additions & 1 deletion .github/workflows/pcs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ on:
jobs:
pcs:
runs-on: ubuntu-latest
env:
PCS_CORE_PATH: ${{ github.workspace }}/pcs-core
defaults:
run:
shell: bash
Expand All @@ -73,16 +75,47 @@ jobs:
- uses: actions/checkout@v4
with:
repository: SentinelOps-CI/pcs-core
ref: 9c971f5f9da8a424924dd8f48d6a3b71a1009e1b
path: pcs-core
- uses: actions/checkout@v4
with:
repository: SentinelOps-CI/pcs-bench
repository: fraware/pcs-bench
ref: 6092cccefee7841dfde1393e6881d433838a2252
path: pcs-bench
- name: Apply LabTrust pcs-core schema profiles
run: python scripts/apply_pcs_core_labtrust_schema_profiles.py --pcs-core pcs-core
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Prepare pinned pcs-core Python packaging metadata
run: |
python - <<'PY'
import shutil
import subprocess
from pathlib import Path

root = Path("pcs-core")
revision = subprocess.check_output(
["git", "-C", str(root), "rev-parse", "HEAD"], text=True
).strip()
expected = "9c971f5f9da8a424924dd8f48d6a3b71a1009e1b"
if revision != expected:
raise RuntimeError(f"Unexpected pcs-core revision: {revision}")
metadata = root / "python" / "pyproject.toml"
original = metadata.read_text(encoding="utf-8")
previous = 'readme = "../README.md"'
if original.count(previous) != 1:
raise RuntimeError("Unknown upstream packaging metadata; review needed")
source_readme = root / "README.md"
destination = root / "python" / "README.md"
if not source_readme.is_file() or destination.exists():
raise RuntimeError("Unexpected upstream README layout")
metadata.write_text(
original.replace(previous, 'readme = "README.md"'), encoding="utf-8"
)
shutil.copyfile(source_readme, destination)
print(f"Applied README-only packaging repair to pinned pcs-core {revision}")
PY
- name: Install pcs-core, pcs-bench, and LabTrust-Gym
run: |
pip install -e "pcs-core/python"
Expand Down
15 changes: 14 additions & 1 deletion .github/workflows/wheel-smoke.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,22 @@ jobs:
python -m venv .venv-wheel
.venv-wheel/bin/pip install dist/*.whl
.venv-wheel/bin/pip install '.[env]' # quick-eval uses env
.venv-wheel/bin/pip install pytest # audit-selfcheck doctor (extras_dev) and risk_register_contract_gate
.venv-wheel/bin/pip install 'pytest>=8,<10' 'pytest-timeout>=2.3,<3' # repo addopts includes --timeout
- name: Audit selfcheck (doctor)
run: .venv-wheel/bin/labtrust audit-selfcheck --out ./audit_out
- name: Show audit diagnostics on failure
if: failure()
run: |
if test -f audit_out/AUDIT_SELF_CHECK.json; then
python -m json.tool audit_out/AUDIT_SELF_CHECK.json
fi
- name: Upload audit diagnostics
if: always()
uses: actions/upload-artifact@v4
with:
name: wheel-audit-diagnostics
path: audit_out/AUDIT_SELF_CHECK.json
if-no-files-found: warn
- name: Validate policy
run: .venv-wheel/bin/labtrust validate-policy
- name: Quick eval (smoke)
Expand Down
2 changes: 1 addition & 1 deletion policy/schemas/risk_register_bundle.v0.1.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@
"evidence_id": { "type": "string" },
"type": {
"type": "string",
"enum": ["security_suite", "coordination_study", "coordination_pack", "official_pack", "safety_case", "bundle_verification", "other"]
"enum": ["security_suite", "coordination_study", "coordination_pack", "coord_risk_run", "official_pack", "safety_case", "bundle_verification", "other"]
},
"path": { "type": "string", "description": "Repo-relative or run-relative path; empty or omitted for evidence gaps (status=missing)." },
"label": { "type": "string" },
Expand Down
20 changes: 16 additions & 4 deletions src/labtrust_gym/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -3081,6 +3081,8 @@ def _run_audit_selfcheck(args: argparse.Namespace) -> int:

# 5. risk register contract gate (pytest)
t0 = time.perf_counter()
contract_stdout = ""
contract_stderr = ""
try:
r = subprocess.run(
[sys.executable, "-m", "pytest", "tests/test_risk_register_contract_gate.py", "-v", "--tb=short"],
Expand All @@ -3090,11 +3092,21 @@ def _run_audit_selfcheck(args: argparse.Namespace) -> int:
timeout=120,
)
passed = r.returncode == 0
except Exception:
if not passed:
contract_stdout = r.stdout[-8000:]
contract_stderr = r.stderr[-4000:]
except Exception as exc:
passed = False
steps.append(
{"name": "risk_register_contract_gate", "pass": passed, "duration_s": round(time.perf_counter() - t0, 3)}
)
contract_stderr = str(exc)
contract_step: dict[str, Any] = {
"name": "risk_register_contract_gate",
"pass": passed,
"duration_s": round(time.perf_counter() - t0, 3),
}
if not passed:
contract_step["output"] = contract_stdout
contract_step["error"] = contract_stderr
steps.append(contract_step)
if not passed:
all_pass = False

Expand Down
56 changes: 38 additions & 18 deletions src/labtrust_gym/cli/pcs.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,25 +10,7 @@

from labtrust_gym.cli.console import get_console
from labtrust_gym.config import get_repo_root
from labtrust_gym.pcs.attach_certificate import attach_certificate_files
from labtrust_gym.pcs.demo import DEMO_SCENARIOS
from labtrust_gym.pcs.demo import run_demo as execute_pcs_demo
from labtrust_gym.pcs.export import export_pcs_bundle, export_runtime_receipt, export_trace
from labtrust_gym.pcs.handoff import export_handoff_bundle
from labtrust_gym.pcs.handoff_manifest import (
emit_handoff_manifest,
emit_handoff_to_certifyedge,
emit_handoff_to_pf,
)
from labtrust_gym.pcs.benchmark_cases import generate_benchmark_cases, verify_benchmark_cases
from labtrust_gym.pcs.benchmark_reproducibility import benchmark_reproducibility
from labtrust_gym.pcs.failure_gallery import generate_failure_gallery
from labtrust_gym.pcs.regenerate_release_chain import regenerate_release_chain
from labtrust_gym.pcs.regenerate_release_protocol import regenerate_release_protocol
from labtrust_gym.pcs.release_protocol_producer import LABTRUST_PROTOCOL_PACKAGE_ARTIFACTS
from labtrust_gym.pcs.release_fragment import emit_labtrust_release_fragment
from labtrust_gym.pcs.status_policy import check_release_status_policy
from labtrust_gym.pcs.verify_release_protocol import verify_release_protocol
from labtrust_gym.pcs.validate import PcsValidationError, validate_artifact_file, validate_run_dir


Expand All @@ -46,6 +28,8 @@ def _print_validation_errors(errors: list[str]) -> None:


def _run_demo(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.demo import run_demo as execute_pcs_demo

name = args.demo
out = _resolve_path(args.out) if args.out else None
try:
Expand Down Expand Up @@ -73,6 +57,8 @@ def _run_demo(args: argparse.Namespace) -> int:


def _run_export_trace(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.export import export_trace

run_dir = _resolve_path(args.run)
out = _resolve_path(args.out)
if not run_dir.is_dir():
Expand All @@ -91,6 +77,8 @@ def _run_export_trace(args: argparse.Namespace) -> int:


def _run_export_runtime_receipt(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.export import export_runtime_receipt

run_dir = _resolve_path(args.run)
out = _resolve_path(args.out)
if not run_dir.is_dir():
Expand All @@ -109,6 +97,8 @@ def _run_export_runtime_receipt(args: argparse.Namespace) -> int:


def _run_export_pcs(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.export import export_pcs_bundle

run_dir = _resolve_path(args.run)
out = _resolve_path(args.out)
if not run_dir.is_dir():
Expand All @@ -127,6 +117,8 @@ def _run_export_pcs(args: argparse.Namespace) -> int:


def _run_attach_certificate(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.attach_certificate import attach_certificate_files

bundle_path = _resolve_path(args.bundle)
cert_path = _resolve_path(args.certificate)
out_path = _resolve_path(args.out)
Expand Down Expand Up @@ -167,6 +159,8 @@ def _run_validate_pcs(args: argparse.Namespace) -> int:


def _run_emit_release_fragment(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.release_fragment import emit_labtrust_release_fragment

release_dir = _resolve_path(args.release_dir)
out = _resolve_path(args.out) if args.out else release_dir / "labtrust_release_fragment.json"
try:
Expand All @@ -179,6 +173,8 @@ def _run_emit_release_fragment(args: argparse.Namespace) -> int:


def _run_emit_handoff(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.handoff_manifest import emit_handoff_manifest

out = _resolve_path(args.out)
try:
emit_handoff_manifest(
Expand All @@ -198,6 +194,8 @@ def _run_emit_handoff(args: argparse.Namespace) -> int:


def _run_verify_release_protocol(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.verify_release_protocol import verify_release_protocol

release_dir = _resolve_path(args.release_dir)
pcs_core = _resolve_path(args.pcs_core) if args.pcs_core else None
try:
Expand All @@ -216,6 +214,8 @@ def _run_verify_release_fixtures(args: argparse.Namespace) -> int:


def _run_emit_handoff_to_certifyedge(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.handoff_manifest import emit_handoff_to_certifyedge

out = _resolve_path(args.out)
try:
emit_handoff_to_certifyedge(
Expand All @@ -233,6 +233,8 @@ def _run_emit_handoff_to_certifyedge(args: argparse.Namespace) -> int:


def _run_emit_handoff_to_pf(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.handoff_manifest import emit_handoff_to_pf

out = _resolve_path(args.out)
try:
emit_handoff_to_pf(
Expand All @@ -248,6 +250,10 @@ def _run_emit_handoff_to_pf(args: argparse.Namespace) -> int:


def _run_regenerate_release_protocol(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.regenerate_release_protocol import regenerate_release_protocol
from labtrust_gym.pcs.verify_release_protocol import verify_release_protocol
from labtrust_gym.pcs.release_protocol_producer import LABTRUST_PROTOCOL_PACKAGE_ARTIFACTS

out = _resolve_path(args.out)
pcs_core = _resolve_path(args.pcs_core) if args.pcs_core else None
ce_spec = _resolve_path(args.certifyedge_spec) if args.certifyedge_spec else None
Expand Down Expand Up @@ -296,6 +302,8 @@ def _run_regenerate_release_protocol(args: argparse.Namespace) -> int:


def _run_check_status_policy(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.status_policy import check_release_status_policy

release_dir = _resolve_path(args.release_dir)
try:
profile_path = _resolve_path(args.workflow_profile) if getattr(args, "workflow_profile", None) else None
Expand All @@ -314,6 +322,8 @@ def _run_check_status_policy(args: argparse.Namespace) -> int:


def _run_generate_benchmark_cases(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.benchmark_cases import generate_benchmark_cases

out = _resolve_path(args.out)
release_dir = _resolve_path(args.release_dir) if args.release_dir else None
validate_root = (
Expand Down Expand Up @@ -361,6 +371,8 @@ def _run_generate_benchmark_cases(args: argparse.Namespace) -> int:


def _run_benchmark_reproducibility(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.benchmark_reproducibility import benchmark_reproducibility

out = _resolve_path(args.out)
pcs_core = _resolve_path(args.pcs_core) if args.pcs_core else None
release_dir = _resolve_path(args.release_dir) if args.release_dir else None
Expand Down Expand Up @@ -442,6 +454,8 @@ def _resolve_pcs_core_root(explicit: Path | None) -> Path | None:


def _run_verify_benchmark_cases(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.benchmark_cases import verify_benchmark_cases

root = _resolve_path(args.benchmark_dir)
pcs_core = _resolve_pcs_core_root(
_resolve_path(args.pcs_core) if getattr(args, "pcs_core", None) else None
Expand Down Expand Up @@ -495,6 +509,8 @@ def _run_verify_benchmark_cases(args: argparse.Namespace) -> int:


def _run_generate_failure_gallery(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.failure_gallery import generate_failure_gallery

out = _resolve_path(args.out)
release_dir = _resolve_path(args.release_dir) if args.release_dir else None
try:
Expand All @@ -518,6 +534,8 @@ def _run_generate_failure_gallery(args: argparse.Namespace) -> int:


def _run_regenerate_release_chain(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.regenerate_release_chain import regenerate_release_chain

out = _resolve_path(args.out)
pcs_core = _resolve_path(args.pcs_core) if args.pcs_core else None
ce_spec = _resolve_path(args.certifyedge_spec) if args.certifyedge_spec else None
Expand All @@ -538,6 +556,8 @@ def _run_regenerate_release_chain(args: argparse.Namespace) -> int:


def _run_export_handoff(args: argparse.Namespace) -> int:
from labtrust_gym.pcs.handoff import export_handoff_bundle

out = _resolve_path(args.out)
try:
manifest = export_handoff_bundle(
Expand Down
6 changes: 4 additions & 2 deletions src/labtrust_gym/pcs/demo.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@
from pathlib import Path

from labtrust_gym.config import get_repo_root
from labtrust_gym.pcs.workflows.qc_release import FAILURE_CASES, run_failure_demo
from labtrust_gym.pcs.workflows.registry import get_workflow

DEMO_SCENARIOS: dict[str, tuple[str, str]] = {
"qc-release": ("valid_workflow.yaml", "runs/qc-release"),
Expand All @@ -33,6 +31,10 @@ def run_demo(
policy_root: Path | None = None,
deterministic: bool | None = None,
) -> Path:
# Import workflow engines only when a PCS demo is actually requested.
from labtrust_gym.pcs.workflows.qc_release import FAILURE_CASES, run_failure_demo
from labtrust_gym.pcs.workflows.registry import get_workflow

if demo_name not in DEMO_SCENARIOS:
raise ValueError(f"unknown demo {demo_name!r}; choose from {list(DEMO_SCENARIOS)}")
root = policy_root or get_repo_root()
Expand Down
Loading
Loading