Integrate Layev model runs into Overwatch

.github/workflows/ci.yml

@@ -0,0 +1,72 @@
+name: Overwatch acceptance gates
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: overwatch-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  backend:
+    name: backend-lint-and-locked-tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    # Configure required reviewers and the authorized short-lived index token
+    # in this environment before using this as an acceptance gate.
+    environment: overwatch-ci
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: '3.13.12'
+      - name: Install pinned uv
+        run: python -m pip install uv==0.10.0
+      - name: Require supported runtime and authorized CI credential route
+        env:
+          OVERWATCH_CI_HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+          UV_INDEX_TSCORE_PASSWORD: ${{ secrets.TSCORE_CODEARTIFACT_TOKEN }}
+        run: python scripts/ci_prerequisites.py --ci
+      - name: Install the unchanged dependency lock
+        env:
+          UV_INDEX_TSCORE_USERNAME: aws
+          UV_INDEX_TSCORE_PASSWORD: ${{ secrets.TSCORE_CODEARTIFACT_TOKEN }}
+          UV_HTTP_TIMEOUT: '30'
+          UV_HTTP_RETRIES: '2'
+        run: uv sync --locked --group dev --python 3.13.12
+      - name: Backend lint
+        run: uv run --locked --no-sync ruff check src tests scripts
+      - name: Backend suite with explicit skip reasons
+        run: uv run --locked --no-sync pytest -q -ra --junitxml=backend-junit.xml
+      - name: Preserve lock and dependency constraints
+        run: git diff --exit-code -- pyproject.toml uv.lock
+
+  frontend:
+    name: frontend-typecheck-vitest-build
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version: '22.16.0'
+      - name: Install frontend lock without lifecycle scripts
+        run: npm ci --ignore-scripts --no-audit --no-fund
+      - name: Typecheck
+        run: npm run typecheck
+      - name: Vitest
+        run: npm test
+      - name: Production build
+        run: npm run build
+      - name: Preserve frontend dependencies
+        run: git diff --exit-code -- package.json package-lock.json

docs/CI_ACCEPTANCE.md

@@ -0,0 +1,76 @@
+# Locked Overwatch CI and acceptance boundaries
+
+These source changes target Overwatch issue #2 and the existing draft PR #1.
+They do not establish successful private authentication, backend tests or the
+producer-to-visible-UI acceptance gate. Keep PR #1 draft until its live gates pass.
+
+## Supported runtime and unchanged dependency sources
+
+Use Python `>=3.13.12,<3.14`. The workflow selects `3.13.12`; neither
+`pyproject.toml` nor `uv.lock` is changed. `typesafe` and `tspath` remain bound to
+the named `tscore` index and the existing `subprocess` keyring configuration.
+No public package substitutes, unlocked install or version relaxation is allowed.
+
+For a local checkout, retain the **existing authorized** CodeArtifact/keyring
+route. Check that its credential is current without printing it. uv also supports
+`UV_INDEX_TSCORE_USERNAME=aws` and `UV_INDEX_TSCORE_PASSWORD` for this exact named
+index. Set the password only through the existing secret manager, not a shell
+command containing its value, a checked-in dotenv file, a URL or a log.
+The preflight deliberately does not dump variables or query a credential store.
+A present value is not proof of authorization; only the actual locked install is.
+
+```sh
+python scripts/ci_prerequisites.py
+uv sync --locked --group dev --python 3.13.12
+uv run --locked --no-sync ruff check src tests scripts
+uv run --locked --no-sync pytest -q -ra --junitxml=backend-junit.xml
+npm ci --ignore-scripts --no-audit --no-fund
+npm run typecheck
+npm test
+npm run build
+```
+
+## Hosted credential boundary
+
+A maintainer must configure the `overwatch-ci` GitHub environment with required
+reviewers and bind the already authorized, short-lived index credential as
+`TSCORE_CODEARTIFACT_TOKEN`. This patch neither obtains that credential nor changes
+AWS access policies. Rotate/refresh expired credentials using the existing approved
+route. Check the environment and secret configuration before running credentialed
+jobs; this source file cannot enforce GitHub environment protection by itself.
+
+The token is scoped only to the prerequisite/install steps, never frontend or
+test steps. No `pull_request_target` trigger, OIDC/cloud deployment, cache upload,
+secret artifact upload or automatic remote telemetry is added. Review the exact
+PR commit before permitting the environment: private package installation executes
+trusted dependency/build code. Fork PRs are not given the token and cannot count a
+skipped backend as success. The preflight exits 2 on a foreign/missing repository
+identity or missing token. Missing/expired access is an environment gate, not a
+passing backend test result. Do not promote untrusted fork code merely to obtain
+credentials.
+
+Stable job names are `backend-lint-and-locked-tests` and
+`frontend-typecheck-vitest-build`. This patch does not configure branch rules or
+claim these jobs ran. Before merge, inspect both checks on **the current PR #1 head**.
+
+## Fresh producer and running UI remain separate
+
+The normal suite retains explicit skip reasons. A skipped
+`tests/test_kev_laya_pipeline.py` is not a successful Layev pipeline. For issue #3,
+provide the separately authorized fresh producer export, isolated registry/cache
+paths, unskipped pipeline receipt, the actual report JSON, browser errors and
+populated UI screenshot. Do not inject a fixture or use hosted CI green as a
+substitute. Run the full suite after supplying that prerequisite on the supported
+local environment. Confirm which checkout the running service uses, preserving
+unrelated files and existing run identities. Never terminate WSL.
+
+## Source references
+
+- Repository dependency source: `pyproject.toml` at
+  `11f32c597f3763137b041e7d867d346e782b3bf9`.
+- uv named-index environment authentication:
+  https://docs.astral.sh/uv/concepts/indexes/#authentication
+- uv existing subprocess keyring route:
+  https://docs.astral.sh/uv/concepts/authentication/http/#keyring-providers
+
+Local source/preflight tests are not the locked Overwatch backend suite.

docs/WANDB_RESPONSE_GUARDS.md

@@ -0,0 +1,56 @@
+# W&B response guards
+
+This is source-level robustness for Overwatch #5, not evidence that an authorized
+W&B or S3 exchange, complete backend suite, or populated UI has passed.
+
+## Pagination metadata
+
+The collector requires `pageInfo` to be an object and `hasNextPage` to be an
+actual JSON Boolean. It must not coerce `null`, numbers, strings, arrays, or
+objects to a completion decision. The Cursor Connections specification defines
+`hasNextPage` as a non-null Boolean:
+https://relay.dev/graphql/connections.htm#sec-undefined.PageInfo.Fields
+
+The Boolean and, for a continuing page, the nonempty, nonrepeating string cursor
+and nonempty edge list are validated before snapshots from that page are
+accepted. A malformed page is an error, not a successfully refreshed source.
+Already accepted snapshots from earlier valid pages remain available. This does
+not promise transactional validation of every other field in a page.
+
+An invalid response does not revoke registration: `configured_ids` and
+`registry_complete` describe the validated registry, while `complete`, source
+`status`, and source `refreshed_at` describe this collection attempt. Do not
+replace an older successful refresh with a failed attempt's timestamp.
+
+## Response ownership
+
+`WandbPages.fetch_snapshot_page` owns the opened response until it returns the
+`Body`. A header-extraction or Content-Length conversion exception closes that
+response without reading it, then propagates the exception. Valid responses
+transfer ownership to the collector, whose existing `finally` closes the body.
+The existing length interpretation and byte-budget rules are unchanged.
+
+No request retries, redirect allowance, credential fallback, run creation,
+run finishing, or additional payload reads are introduced. The deadline remains
+cooperative: these changes do not cancel an in-flight network/credential call.
+
+## Regressions
+
+`tests/test_kev_laya_wandb_response_guards.py` uses the real collector and the
+standalone telemetry validator, synthetic in-memory pages, and an injected
+opener. Socket connection attempts are forbidden in this suite. It covers
+invalid completion types and page metadata, invalid/repeated cursors, previous
+valid-page retention, valid terminal/paginated pages, encoded summary JSON,
+known/unknown lengths, source and request/page budgets, identity mismatch,
+GraphQL errors, header-error cleanup, ownership transfer, and no added retry.
+
+In a supported, authenticated checkout, run these tests and the unchanged full
+CI gates described in [CI_ACCEPTANCE.md](CI_ACCEPTANCE.md). No synthetic result
+satisfies the live acceptance boxes in [issue #5](https://github.com/CompleteDotTech/Overwatch/issues/5)
+or the fresh producer-to-visible-UI gate in [issue #3](https://github.com/CompleteDotTech/Overwatch/issues/3).
+
+Live qualification must still record the actual endpoint/query shape, bounded
+S3/W&B bytes and request/page/read counts, timeout behavior, malformed/outage and
+budget-exhaustion cache preservation, and source-registration/freshness
+separation, using existing authorized non-production targets. Keep credentials
+and private payloads out of source, receipts, logs, and issue comments.

scripts/ci_prerequisites.py

@@ -0,0 +1,39 @@
+"""Secret-safe CI prerequisite checks. No credential lookup or network requests."""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from collections.abc import Mapping
+
+
+def blockers(version: tuple[int, ...], environ: Mapping[str, str], *, ci: bool) -> list[str]:
+    """Return public diagnostic codes only; never include environment values."""
+    result = []
+    if not (version[:2] == (3, 13) and version[2] >= 12):
+        result.append("unsupported_python_requires_3_13_12_to_before_3_14")
+    if ci:
+        repository = environ.get("GITHUB_REPOSITORY", "")
+        head = environ.get("OVERWATCH_CI_HEAD_REPOSITORY", "")
+        if not repository or not head or head != repository:
+            result.append("credentialed_backend_requires_reviewed_same_repository_code")
+        if not environ.get("UV_INDEX_TSCORE_PASSWORD", ""):
+            result.append("missing_authorized_tscore_index_token")
+    return result
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--ci", action="store_true")
+    args = parser.parse_args()
+    reasons = blockers(tuple(sys.version_info[:3]), os.environ, ci=args.ci)
+    for reason in reasons:
+        print("BLOCKED: " + reason)
+    if reasons:
+        return 2
+    print("Prerequisite shape checks passed; authentication and locked install remain unverified.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/overwatch/cached_metrics.py

@@ -680,3 +680,10 @@ def cloudwatch_event_order(event: dict[str, Any]) -> tuple[int, str]:
 def raw_cache_manifest() -> dict[str, Any]:
     """Load the collector manifest used for query status and cache provenance."""
     return read_json(raw_cache_path("manifest.json"), {"sources": {}, "cache_root": str(RAW_CACHE_ROOT)})
+
+
+def cached_kev_laya_runs() -> dict[str, Any]:
+    """Read model-run snapshots strictly from the versioned collector cache."""
+    from overwatch.raw_cache import read_model_runs_cache
+
+    return read_model_runs_cache()

src/overwatch/collector.py

@@ -57,6 +57,7 @@ class CollectorOptions:
     limit: int = 20
     entity: str | None = None
     gcp_billing_table: str | None = None
+    local_models_only: bool = False
 
 
 
@@ -194,6 +195,29 @@ async def collect_source(
         }
         return value
 
+    # Explicit offline/local mode: use the REAL collector and cache without cloud I/O.
+    # Default behavior and existing Flow/resource records remain unchanged.
+    if options.local_models_only:
+        if job_id is not None:
+            raise ValueError("local-models-only cannot refresh a SkyPilot job")
+        result = await collect_source("kev_laya", collect_kev_laya_cache)
+        if result is not None:
+            source_status["kev_laya"]["raw"] = result
+            if result["warnings"]:
+                source_status["kev_laya"]["status"] = "warning"
+        for key in ("sky_jobs", "sky_clusters", "aws_billing", "gcp_billing", "wandb", "cloudwatch"):
+            source_status[key] = {"status": "skipped", "duration_seconds": 0.0,
+                                  "error": None, "reason": "explicit local-models-only"}
+        manifest = {
+            "cache_spec_version": RAW_CACHE_SPEC_VERSION, "cache_root": str(RAW_CACHE_ROOT),
+            "updated_at": isoformat(datetime.now(UTC)),
+            "duration_seconds": round(perf_counter() - collection_started, 3),
+            "scope": {"job_id": None, "full_logs": False, "local_models_only": True},
+            "sources": source_status,
+        }
+        write_json_atomically(manifest_path, manifest)
+        return manifest
+
     # A scoped job refresh reuses inventory and only advances that job's raw log cursor.
     if job_id is not None:
         job = read_json(raw_cache_path("jobs", str(job_id), "sky.json"))
@@ -253,6 +277,12 @@ async def collect_source(
                 "error": None,
             }
 
+        kev_laya_result = await collect_source("kev_laya", collect_kev_laya_cache)
+        if kev_laya_result is not None:
+            source_status["kev_laya"]["raw"] = kev_laya_result
+            if kev_laya_result["warnings"]:
+                source_status["kev_laya"]["status"] = "warning"
+
         # W&B stays optional and is captured as raw hydrated run records.
         wandb_started = perf_counter()
         try:
@@ -393,10 +423,35 @@ async def run_bounded_cloudwatch_operation(
     return manifest
 
 
+
+def collect_kev_laya_cache() -> dict[str, Any]:
+    """The collector alone owns Kev-Laya raw-cache writes."""
+    from overwatch.constants import KEV_LAYA_RETENTION_DAYS
+    from overwatch.kev_laya_adapter import merge_snapshots
+    from overwatch.kev_laya_lock import collector_lock
+    from overwatch.providers.kev_laya import collect_snapshots
+    from overwatch.raw_cache import model_runs_cache_path
+
+    path = model_runs_cache_path()
+    collected = collect_snapshots()
+    with collector_lock(path.with_suffix(".lock")):
+        from overwatch.raw_cache import read_model_runs_cache
+        envelope = merge_snapshots(read_model_runs_cache(), collected,
+                                   retention_days=KEV_LAYA_RETENTION_DAYS)
+        write_json_atomically(path, envelope)
+    return {"runs": len(envelope["records"]), "warnings": len(envelope["warnings"]),
+            "complete": collected.get("complete", False),
+            "accounting": collected.get("accounting", {}),
+            "source_status": collected.get("sources", []),
+            "collection_warnings": collected.get("warnings", [])}
+
+
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Collect raw Overwatch metrics")
     parser.add_argument("--job-id", type=int)
     parser.add_argument("--full-logs", action="store_true")
+    parser.add_argument("--local-models-only", action="store_true",
+                        default=os.environ.get("OVERWATCH_LOCAL_MODELS_ONLY") == "1")
     parser.add_argument("--limit", type=int, default=20)
     parser.add_argument("--entity")
     parser.add_argument(
@@ -411,6 +466,7 @@ def main() -> int:
         limit=args.limit,
         entity=args.entity,
         gcp_billing_table=args.gcp_billing_table,
+        local_models_only=args.local_models_only,
     )
     manifest = asyncio.run(
         collect_raw_metrics(options, job_id=args.job_id, full_logs=args.full_logs)

src/overwatch/constants.py

@@ -60,3 +60,7 @@
     r"\[(?P<elapsed>[^<\]]+)<(?P<remaining>.+?)\s+[\d.eE+-]+ex/s "
     r"(?P<tokens_per_second>[\d,]+)tok/s"
 )
+
+# Kev-Laya adapter policy; does not change Flow/cloud log defaults.
+KEV_LAYA_STALE_SECONDS = 120
+KEV_LAYA_RETENTION_DAYS = 14

src/overwatch/frontend/App.tsx

@@ -1,3 +1,4 @@
+import { ModelRuns } from "./ModelRunsView";
 import {
   ActionIcon,
   Alert,
@@ -1009,7 +1010,7 @@ export function App() {
             </Stack>
           </Center>
         )}
-        {report && page === "resources" && <ResourcesPage report={report} />}
+        {report && page === "resources" && <><ResourcesPage report={report} /><ModelRuns runs={report.model_runs ?? []} warnings={report.model_run_warnings ?? []} /></>}
         {report && page === "billing" && <BillingPage report={report} />}
         {report && page === "cost-waste" && <CostWastePage report={report} />}
         {report && page === "run" && runJobId != null && <RunPage report={report} jobId={runJobId} />}