From 05001155db5ee19d76a0189d3b01413e80cf0bc9 Mon Sep 17 00:00:00 2001 From: Harish Seshadri Date: Fri, 25 Sep 2026 08:36:01 -0700 Subject: [PATCH] feat(fleet): cover agentic consumers and fail on uncovered ones The fleet scan only checked repositories listed by hand in repository_expectations. agentic-saga and agentic-context-service pin hseshadr/ci modules but were never listed, so no fleet rule ran on them. - Add both to repository_expectations with the full consumer contract (sole Dagger check, conversation resolution, no rollout exception; both already declare shared foundation). - New fleet_coverage module: list every public hseshadr repository, read its default-branch dagger.json, and report uncovered-consumer for any active repository that pins a github.com/hseshadr/ci module but is missing from the list. Listing or config read errors fail closed. - scan_repository now turns FleetAccessError into an evidence-unreadable finding, so one unreadable repository (agentic-context-service has no branch protection on main) no longer hides every later result. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_015oBArfm762nN1r4F4Fst5a --- .dagger/src/ci/fleet.py | 32 +++- .dagger/src/ci/fleet_coverage.py | 101 ++++++++++++ .dagger/tests/test_fleet.py | 4 +- .dagger/tests/test_fleet_coverage.py | 237 +++++++++++++++++++++++++++ CHANGELOG.md | 4 + README.md | 5 +- docs/dagger-modules.md | 16 ++ 7 files changed, 390 insertions(+), 9 deletions(-) create mode 100644 .dagger/src/ci/fleet_coverage.py create mode 100644 .dagger/tests/test_fleet_coverage.py diff --git a/.dagger/src/ci/fleet.py b/.dagger/src/ci/fleet.py index c35aea9..d6d9c3f 100644 --- a/.dagger/src/ci/fleet.py +++ b/.dagger/src/ci/fleet.py @@ -5,8 +5,11 @@ from dataclasses import dataclass from datetime import date -from ci.fleet_policy import PolicyFinding, RepositoryExpectation, validate_repository -from ci.github_fleet import GitHubHttpTransport, read_repository +from ci.fleet_coverage import coverage_results +from ci.fleet_policy import PolicyFinding, RepositoryExpectation, finding, validate_repository +from ci.github_fleet import FleetAccessError, GitHubHttpTransport, GitHubTransport, read_repository + +OWNER = "hseshadr" @dataclass(frozen=True) @@ -21,6 +24,8 @@ class RepositoryResult: def repository_expectations(include_central: bool) -> tuple[RepositoryExpectation, ...]: """Return the reviewed fleet contract for the current rollout phase.""" consumers = ( + expectation("agentic-context-service"), + expectation("agentic-saga"), expectation("almamesh", grandfathered_until=date(2026, 12, 15)), expectation("aml-filter", grandfathered_until=date(2026, 12, 31)), expectation("assay", linear_history=True, grandfathered_until=date(2026, 11, 30)), @@ -57,15 +62,28 @@ def expectation_for(name: str) -> RepositoryExpectation: def scan_fleet(token: str, include_central: bool) -> tuple[RepositoryResult, ...]: """Read and evaluate each repository from authoritative exact-main evidence.""" - transport = GitHubHttpTransport(token) + return scan_fleet_with(GitHubHttpTransport(token), include_central) + + +def scan_fleet_with( + transport: GitHubTransport, include_central: bool +) -> tuple[RepositoryResult, ...]: + """Prove coverage of every discovered consumer, then evaluate each reviewed one.""" + reviewed = tuple(item.name for item in repository_expectations(True)) + uncovered = coverage_results(transport, OWNER, reviewed) + coverage = tuple(RepositoryResult(item.name, "", item.findings) for item in uncovered) expectations = repository_expectations(include_central) - return tuple(scan_repository(transport, item) for item in expectations) + return coverage + tuple(scan_repository(transport, item) for item in expectations) def scan_repository( - transport: GitHubHttpTransport, expectation_: RepositoryExpectation + transport: GitHubTransport, expectation_: RepositoryExpectation ) -> RepositoryResult: - """Evaluate one exact-main repository against its reviewed contract.""" - snapshot = read_repository(transport, "hseshadr", expectation_.name) + """Evaluate one repository, turning unreadable evidence into a failing finding.""" + try: + snapshot = read_repository(transport, OWNER, expectation_.name) + except FleetAccessError as error: + unreadable = finding("evidence-unreadable", "github", str(error)) + return RepositoryResult(expectation_.name, "", (unreadable,)) findings = validate_repository(snapshot, expectation_) return RepositoryResult(snapshot.name, snapshot.sha, findings) diff --git a/.dagger/src/ci/fleet_coverage.py b/.dagger/src/ci/fleet_coverage.py new file mode 100644 index 0000000..2389e28 --- /dev/null +++ b/.dagger/src/ci/fleet_coverage.py @@ -0,0 +1,101 @@ +"""Discover every Dagger consumer of the central modules and prove fleet coverage. + +The reviewed fleet contract (`repository_expectations`) is a hand-maintained list, so a +new consumer silently escapes every fleet check until someone remembers to add it. This +module closes that gap: it lists the owner's repositories, reads each default-branch +`dagger.json`, and fails the scan for any consumer the contract does not name. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final + +from pydantic.dataclasses import dataclass as validated_dataclass + +from ci.fleet_policy import PolicyFinding, finding +from ci.github_fleet import ( + BOUNDARY_CONFIG, + HTTP_NOT_FOUND, + GitHubTransport, + decode_source, + parse_content_response, + parse_dagger_config, + read_model, +) + +UNCOVERED_CODE: Final = "uncovered-consumer" +CENTRAL_PREFIXES: Final = ("github.com/hseshadr/ci/", "github.com/hseshadr/ci@") +PAGE_SIZE: Final = 100 + + +@validated_dataclass(config=BOUNDARY_CONFIG) +class OwnedRepositoryPayload: + """One entry of the owner's public repository listing.""" + + name: str + archived: bool + default_branch: str + + +@dataclass(frozen=True) +class UncoveredConsumer: + """One discovered consumer that the reviewed fleet contract does not name.""" + + name: str + findings: tuple[PolicyFinding, ...] + + +def discover_consumers(transport: GitHubTransport, owner: str) -> tuple[str, ...]: + """Return every active repository whose default-branch dagger.json pins hseshadr/ci.""" + return tuple( + repository.name + for repository in list_repositories(transport, owner) + if not repository.archived and consumes_central(transport, owner, repository) + ) + + +def list_repositories(transport: GitHubTransport, owner: str) -> tuple[OwnedRepositoryPayload, ...]: + """Read every page of the owner's repository listing, failing closed on any error.""" + collected: list[OwnedRepositoryPayload] = [] + page = 1 + while True: + path = f"users/{owner}/repos?type=owner&per_page={PAGE_SIZE}&page={page}" + batch = read_model(transport, path, tuple[OwnedRepositoryPayload, ...]) + collected.extend(batch) + if len(batch) < PAGE_SIZE: + return tuple(collected) + page += 1 + + +def consumes_central( + transport: GitHubTransport, owner: str, repository: OwnedRepositoryPayload +) -> bool: + """Return whether one default-branch dagger.json declares a central module.""" + base = f"repos/{owner}/{repository.name}" + path = f"{base}/contents/dagger.json?ref={repository.default_branch}" + response = transport.get(path) + if response.status == HTTP_NOT_FOUND: + return False + source = decode_source(parse_content_response(response, path), "dagger.json", base) + config = parse_dagger_config(source) + return any(item.source.startswith(CENTRAL_PREFIXES) for item in config.dependencies) + + +def uncovered_consumers(discovered: tuple[str, ...], reviewed: tuple[str, ...]) -> tuple[str, ...]: + """Return discovered consumers absent from the reviewed fleet contract.""" + return tuple(name for name in discovered if name not in reviewed) + + +def coverage_results( + transport: GitHubTransport, owner: str, reviewed: tuple[str, ...] +) -> tuple[UncoveredConsumer, ...]: + """Build one failing result per discovered consumer the fleet scan would skip.""" + missing = uncovered_consumers(discover_consumers(transport, owner), reviewed) + return tuple(UncoveredConsumer(name, (uncovered_finding(name),)) for name in missing) + + +def uncovered_finding(name: str) -> PolicyFinding: + """Name the exact fix for one consumer that escapes the fleet scan.""" + message = f"{name} pins hseshadr/ci modules but is missing from repository_expectations" + return finding(UNCOVERED_CODE, "dagger.json", message) diff --git a/.dagger/tests/test_fleet.py b/.dagger/tests/test_fleet.py index 5d72633..a3164a2 100644 --- a/.dagger/tests/test_fleet.py +++ b/.dagger/tests/test_fleet.py @@ -10,8 +10,10 @@ def test_should_enforce_exact_consumer_set_when_central_is_not_main() -> None: # When the immutable fleet expectations are selected expectations = repository_expectations(include_central) - # Then exactly the seven migrated consumers require sole Dagger + # Then exactly the nine migrated consumers require sole Dagger assert tuple(item.name for item in expectations) == ( + "agentic-context-service", + "agentic-saga", "almamesh", "aml-filter", "assay", diff --git a/.dagger/tests/test_fleet_coverage.py b/.dagger/tests/test_fleet_coverage.py new file mode 100644 index 0000000..1a2710a --- /dev/null +++ b/.dagger/tests/test_fleet_coverage.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import base64 +import json +from dataclasses import dataclass + +import pytest +from test_github_fleet import FakeTransport as ExactMainTransport +from test_github_fleet import _responses as exact_main_responses + +import ci.fleet +from ci.fleet import ( + expectation, + expectation_for, + repository_expectations, + scan_fleet, + scan_fleet_with, + scan_repository, +) +from ci.fleet_coverage import ( + UNCOVERED_CODE, + coverage_results, + discover_consumers, + uncovered_consumers, +) +from ci.github_fleet import FleetAccessError, HttpResponse + +OWNER = "hseshadr" +CI_SOURCE = "github.com/hseshadr/ci/modules/portfolio-foundation@" + "a" * 40 +OTHER_SOURCE = "github.com/dagger/dagger/modules/wolfi@" + "b" * 40 +LISTING = "users/hseshadr/repos?type=owner&per_page=100&page={page}" + +# The live 2026-09-25 scan's exact set of hseshadr repos whose default-branch +# dagger.json references github.com/hseshadr/ci. A new consumer must be added here +# and to repository_expectations in the same change. +KNOWN_CONSUMERS = ( + "agentic-context-service", + "agentic-saga", + "almamesh", + "aml-filter", + "assay", + "edge-proc", + "edge-reco", + "edgeproc-core", + "privacy-core", +) + + +@dataclass(frozen=True) +class FakeTransport: + """Return exact fixture responses; every unknown path is a 404.""" + + responses: dict[str, HttpResponse] + + def get(self, path: str) -> HttpResponse: + return self.responses.get(path, HttpResponse(status=404, body="{}")) + + +def _json(value: object, status: int = 200) -> HttpResponse: + return HttpResponse(status=status, body=json.dumps(value)) + + +def _repo(name: str, *, archived: bool = False) -> dict[str, object]: + return {"name": name, "archived": archived, "default_branch": "main"} + + +def _config(name: str, *sources: str) -> HttpResponse: + dependencies = [{"name": f"dep{index}", "source": item} for index, item in enumerate(sources)] + sdk = {"source": "python"} + config = {"name": name, "engineVersion": "v0.21.8", "sdk": sdk, "dependencies": dependencies} + text = json.dumps(config) + encoded = base64.b64encode(text.encode()).decode() + return _json({"type": "file", "path": "dagger.json", "encoding": "base64", "content": encoded}) + + +def _config_path(name: str) -> str: + return f"repos/hseshadr/{name}/contents/dagger.json?ref=main" + + +def _fleet_responses() -> dict[str, HttpResponse]: + first_page = [_repo(f"filler-{index}") for index in range(98)] + first_page += [_repo("consumer"), _repo("unrelated")] + second_page = [_repo("no-dagger"), _repo("retired", archived=True), _repo("covered")] + return { + LISTING.format(page=1): _json(first_page), + LISTING.format(page=2): _json(second_page), + _config_path("consumer"): _config("consumer", CI_SOURCE), + _config_path("unrelated"): _config("unrelated", OTHER_SOURCE), + _config_path("retired"): _config("retired", CI_SOURCE), + _config_path("covered"): _config("covered", OTHER_SOURCE, CI_SOURCE), + } + + +def test_should_discover_every_active_ci_consumer_across_listing_pages() -> None: + # Given a two-page owner listing with consumers, non-consumers and an archived consumer + transport = FakeTransport(_fleet_responses()) + + # When consumers are discovered from default-branch dagger.json evidence + discovered = discover_consumers(transport, OWNER) + + # Then only active repositories that pin hseshadr/ci modules are returned + assert discovered == ("consumer", "covered") + + +def test_should_report_consumer_missing_from_fleet_expectations() -> None: + # Given a discovered consumer that the reviewed fleet contract does not name + transport = FakeTransport(_fleet_responses()) + reviewed = ("assay", "covered") + + # When coverage is evaluated against discovery + results = coverage_results(transport, OWNER, reviewed) + + # Then only the uncovered consumer fails the scan, with an actionable finding + assert [(item.name, [f.code for f in item.findings]) for item in results] == [ + ("consumer", [UNCOVERED_CODE]) + ] + assert results[0].findings[0].path == "dagger.json" + assert "repository_expectations" in results[0].findings[0].message + + +def test_should_fail_closed_when_owner_listing_is_unreadable() -> None: + # Given a listing endpoint that refuses the read + transport = FakeTransport({LISTING.format(page=1): _json({}, status=403)}) + + # When discovery runs + # Then the scan cannot claim complete coverage + with pytest.raises(FleetAccessError, match="403"): + discover_consumers(transport, OWNER) + + +def test_should_fail_closed_when_consumer_config_is_unreadable() -> None: + # Given a repository whose dagger.json read fails with something other than 404 + responses = { + LISTING.format(page=1): _json([_repo("flaky")]), + _config_path("flaky"): _json({}, status=500), + } + + # When discovery runs + # Then the unreadable config is an error, never a silent non-consumer + with pytest.raises(FleetAccessError, match="500"): + discover_consumers(FakeTransport(responses), OWNER) + + +def test_should_cover_every_known_ci_consumer_in_reviewed_expectations() -> None: + # Given the live-discovered consumer set (stubbed; the hosted scan rediscovers it) + names = tuple(item.name for item in repository_expectations(True)) + + # When coverage is computed against the reviewed fleet contract + missing = uncovered_consumers(KNOWN_CONSUMERS, names) + + # Then no consumer escapes the fleet scan + assert missing == () + + +@pytest.mark.parametrize("name", ["agentic-context-service", "agentic-saga"]) +def test_should_hold_agentic_consumers_to_the_sole_dagger_contract(name: str) -> None: + # Given an agentic consumer that already declares shared foundation + # When its reviewed expectation is selected + item = expectation_for(name) + + # Then it gets the full consumer contract with no rollout exception + assert item.required_contexts == ("Dagger",) + assert item.conversation_resolution is True + assert item.linear_history is False + assert item.shared_foundation_required is True + assert item.grandfathered_until is None + + +def test_should_report_unreadable_repository_as_finding_and_keep_scanning() -> None: + # Given a consumer whose protection endpoint 404s (for example, unprotected main) + transport = FakeTransport({}) + + # When that one repository is scanned + result = scan_repository(transport, expectation_for("agentic-context-service")) + + # Then the failure is a named finding, so later repositories are still evaluated + assert result.name == "agentic-context-service" + assert [item.code for item in result.findings] == ["evidence-unreadable"] + assert "404" in result.findings[0].message + + +def test_should_fail_hosted_scan_for_uncovered_consumer_before_reviewed_repositories() -> None: + # Given a live listing where one consumer is unknown to the reviewed contract + responses = { + LISTING.format(page=1): _json([_repo("newcomer"), _repo("assay")]), + _config_path("newcomer"): _config("newcomer", CI_SOURCE), + _config_path("assay"): _config("assay", CI_SOURCE), + } + + # When the whole fleet scan runs (reviewed repositories are unreadable here) + results = scan_fleet_with(FakeTransport(responses), include_central=False) + + # Then the uncovered consumer is reported and every reviewed repository still runs + assert results[0].name == "newcomer" + assert [item.code for item in results[0].findings] == [UNCOVERED_CODE] + reviewed = tuple(item.name for item in repository_expectations(False)) + assert tuple(item.name for item in results[1:]) == reviewed + assert all(item.findings for item in results) + + +def test_should_evaluate_readable_repository_against_its_contract() -> None: + # Given complete exact-main evidence for one reviewed repository + transport = ExactMainTransport(exact_main_responses()) + + # When that repository is scanned + result = scan_repository(transport, expectation("example")) + + # Then its identity is the exact main SHA and no evidence error is reported + assert (result.name, result.sha) == ("example", "a" * 40) + assert "evidence-unreadable" not in {item.code for item in result.findings} + + +def test_should_reject_unknown_fleet_repository_name() -> None: + # Given a name the reviewed contract does not contain + # When its expectation is requested + # Then the caller is told instead of receiving a default contract + with pytest.raises(ValueError, match="unknown fleet repository: nope"): + expectation_for("nope") + + +def test_should_scan_hosted_fleet_through_authenticated_transport( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Given a hosted run that supplies only a token + seen: list[object] = [] + + def fake_scan(transport: object, include_central: bool) -> tuple[()]: + seen.append((type(transport).__name__, include_central)) + return () + + monkeypatch.setattr(ci.fleet, "scan_fleet_with", fake_scan) + + # When the fleet is scanned + scan_fleet("token", include_central=True) + + # Then the real GitHub transport performs discovery and every repository read + assert seen == [("GitHubHttpTransport", True)] diff --git a/CHANGELOG.md b/CHANGELOG.md index ced7115..a864b5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,10 @@ evidence. - Optional typed Git authorization for exact private-repository history in `portfolio-foundation`, kept inside Dagger's secret boundary. +- Fleet coverage: `agentic-saga` and `agentic-context-service` join the fleet scan, and every + scan now discovers `hseshadr/ci` consumers from default-branch `dagger.json` and fails with + `uncovered-consumer` for any that are not listed. Unreadable repositories become an + `evidence-unreadable` finding, so they no longer stop the scan. ### Changed diff --git a/README.md b/README.md index 9185a41..aed4e20 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ dagger call fleet --github-token=env:GITHUB_TOKEN --include-central The first command runs central quality and security checks. The second reads exact `main` state from GitHub for: +- `agentic-context-service` +- `agentic-saga` - `almamesh` - `aml-filter` - `assay` @@ -32,7 +34,8 @@ The first command runs central quality and security checks. The second reads exa - `privacy-core` - `ci` -Any inaccessible or incomplete evidence is an error. A scan that inspected nothing +It also fails if any other `hseshadr` repository pins a `github.com/hseshadr/ci` module but +is missing from that list. Any inaccessible or incomplete evidence is an error. A scan that inspected nothing cannot report success. ## Reuse the Dagger legos diff --git a/docs/dagger-modules.md b/docs/dagger-modules.md index 653d8bb..a908597 100644 --- a/docs/dagger-modules.md +++ b/docs/dagger-modules.md @@ -372,6 +372,22 @@ authoritative fleet evidence before merge, followed by exact-main evidence after the merged SHA against `^[0-9a-f]{40}$` and record it in the durable release ledger; a temporary file alone is not release evidence. +## Fleet coverage + +The fleet scan checks only the repositories named in `repository_expectations` +(`.dagger/src/ci/fleet.py`). That list is written by hand, so a new consumer would otherwise +escape every fleet check. To close that gap, each hosted scan first lists every public +`hseshadr` repository, reads its default-branch `dagger.json`, and reports +`uncovered-consumer` for any active repository that pins a `github.com/hseshadr/ci` module but is +missing from the list. The scan then fails. + +- **When you onboard a consumer, add it to `repository_expectations` in the same PR.** Also add + it to `KNOWN_CONSUMERS` in `.dagger/tests/test_fleet_coverage.py`. +- Archived repositories are skipped. Private repositories are not listed, so they are not + discovered. +- A repository whose evidence cannot be read (for example, `main` has no branch protection) + gets an `evidence-unreadable` finding. The scan keeps going and still fails. + ## Release status Shipped in this central change: