From 0e0277411a9b8755ff82a087c8b77874629728d7 Mon Sep 17 00:00:00 2001 From: Harish Seshadri Date: Fri, 25 Sep 2026 22:26:48 -0700 Subject: [PATCH 1/2] docs: rewrite README in plain English; add architecture and getting-started docs The README now says what the repo is (shared Dagger modules and a daily cross-repo check for Harish's own repos), shows how a real consumer pins a module, and pastes real output from running one. Technical detail moved to docs/ARCHITECTURE.md; docs/GETTING_STARTED.md walks a new developer from a fresh clone to a first change, with measured times. A new README contract test pins the first line, section order, module names, links to every doc, and a banned-jargon check. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_015oBArfm762nN1r4F4Fst5a --- .dagger/tests/test_readme_contract.py | 105 +++++++++++ README.md | 254 ++++++++++++-------------- docs/ARCHITECTURE.md | 144 +++++++++++++++ docs/GETTING_STARTED.md | 133 ++++++++++++++ 4 files changed, 503 insertions(+), 133 deletions(-) create mode 100644 .dagger/tests/test_readme_contract.py create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/GETTING_STARTED.md diff --git a/.dagger/tests/test_readme_contract.py b/.dagger/tests/test_readme_contract.py new file mode 100644 index 0000000..3b9d8ab --- /dev/null +++ b/.dagger/tests/test_readme_contract.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).parents[2] +README = ROOT / "README.md" + +SECTIONS = ( + "## How a repo uses it", + "## Try it", + "## How it works", + "## What it does not do", + "## Develop", + "## More detail", + "## License", +) +MODULES = ("portfolio-foundation", "cloudflare-pages", "python-package") +BANNED = ( + "northstar", + "seam", + "lego", + "trust envelope", + "fail-closed", + "fail closed", + "fleet", + "gate", + "portfolio", + "production-ready", + "robust", + "blazing", + "enterprise-grade", + "seamless", + "tl;dr", +) + + +def _prose(text: str) -> str: + without_blocks = re.sub(r"```.*?```", "", text, flags=re.DOTALL) + without_code = re.sub(r"`[^`]*`", "", without_blocks) + return re.sub(r"\]\([^)]*\)", "]", without_code).lower() + + +def test_should_open_with_name_and_plain_audience_sentence() -> None: + # Given the repository landing page + lines = README.read_text().splitlines() + + # When a reader sees only the first three lines + title, blank, sentence = lines[0], lines[1], lines[2] + + # Then it names the repo and says plainly who it is for and what it holds + assert title == "# hseshadr/ci" + assert blank == "" + assert "Harish Seshadri's own repositories" in sentence + assert "Dagger modules" in sentence + + +def test_should_keep_sections_in_the_agreed_order() -> None: + # Given the landing page headings + text = README.read_text() + + # When the required sections are located + positions = [text.find(heading + "\n") for heading in SECTIONS] + + # Then every section exists, in order, with technical links above the first one + assert all(position >= 0 for position in positions) + assert positions == sorted(positions) + assert 0 <= text.find("**Technical docs:**") < positions[0] + + +def test_should_state_true_facts_about_modules_and_checks() -> None: + # Given the landing page + text = README.read_text() + + # When the reader looks for what the repo ships and how to run it + required = (*MODULES, "dagger call ci", "Dagger 0.21.8", "dagger.json", '"pin"') + + # Then every shared module and the real local command are named + assert all(fragment in text for fragment in required) + + +def test_should_link_every_technical_document() -> None: + # Given every document under docs/ plus the changelog + text = README.read_text() + documents = sorted(path.relative_to(ROOT).as_posix() for path in ROOT.glob("docs/**/*.md")) + + # When the README link targets are collected + targets = set(re.findall(r"\]\(([^)#]+)", text)) + + # Then nothing technical is orphaned + expected = {*documents, "docs/architecture/index.html", "CHANGELOG.md", "LICENSE"} + assert "docs/ARCHITECTURE.md" in documents + assert "docs/GETTING_STARTED.md" in documents + assert expected <= targets + + +def test_should_avoid_internal_jargon_and_hype_in_prose() -> None: + # Given the README with code and link targets removed + prose = _prose(README.read_text()) + + # When banned vocabulary is searched as whole words + found = [word for word in BANNED if re.search(rf"\b{re.escape(word)}s?\b", prose)] + + # Then the prose reads as plain English + assert found == [] diff --git a/README.md b/README.md index 9185a41..bd8d554 100644 --- a/README.md +++ b/README.md @@ -1,168 +1,156 @@ # hseshadr/ci -**TL;DR:** GitHub delivers events; Dagger owns execution. This repository contains the -fleet policy used to prove that all eight repositories follow that boundary, plus typed -reusable Dagger modules for repository safety and Cloudflare Pages delivery. It does not -publish reusable workflows, composite actions, or copyable CI templates. - -A **Dagger lego** is a typed reusable Dagger module installed at an immutable commit SHA. -The exact commit makes the shared behavior reviewable and prevents a consumer from changing -when central `main` moves. - -## Run it now - -Prerequisites: Docker, Dagger 0.21.8, and a GitHub token that can read the fleet. - -```bash -cd /path/to/ci -export GITHUB_TOKEN="$(gh auth token)" -dagger call ci --github-token=env:GITHUB_TOKEN -dagger call fleet --github-token=env:GITHUB_TOKEN --include-central +The shared CI code for Harish Seshadri's own repositories: three Dagger modules they install, and a daily check that each repo still runs its CI the same way. + +**Try it without cloning:** `dagger -m github.com/hseshadr/ci/modules/portfolio-foundation@faf55ac51dfeb6bad274988941e9215117e5259e functions` + +[Dagger](https://dagger.io) runs CI steps as code inside containers, so the same steps run on +a laptop and in GitHub Actions. Harish's projects (almamesh, aml-filter, assay, edge-proc, +edge-reco and others) all need the same few things: check the source is safe, build a +Python package, deploy a static site to Cloudflare Pages. Copying that code into each repo +means many copies drifting apart. This repo holds one copy, as Dagger modules each repo +installs at an exact commit. + +It also watches the other repos. Once a day it reads the `main` branch of each repo on its +list from GitHub, and fails if a workflow runs anything outside Dagger, branch protection +has drifted, or the latest `main` commit is not green. + +**Technical docs:** [Architecture](docs/ARCHITECTURE.md) · [Getting started](docs/GETTING_STARTED.md) · [Using the modules](docs/dagger-modules.md) + +## How a repo uses it + +A repo adds a module to its own `dagger.json`, pinned to one commit of this repo. This is +aml-filter's, as it is today: + +```json +"dependencies": [ + { + "name": "cloudflare-pages", + "source": "github.com/hseshadr/ci/modules/cloudflare-pages@dd19871486588b1582e432b7bc1f2cfffb296340", + "pin": "dd19871486588b1582e432b7bc1f2cfffb296340" + }, + { + "name": "foundation", + "source": "github.com/hseshadr/ci/modules/portfolio-foundation@dd19871486588b1582e432b7bc1f2cfffb296340", + "pin": "dd19871486588b1582e432b7bc1f2cfffb296340" + } +] ``` -The first command runs central quality and security checks. The second reads exact -`main` state from GitHub for: - -- `almamesh` -- `aml-filter` -- `assay` -- `edge-proc` -- `edge-reco` -- `edgeproc-core` -- `privacy-core` -- `ci` - -Any inaccessible or incomplete evidence is an error. A scan that inspected nothing -cannot report success. - -## Reuse the Dagger legos - -The shared modules are: - -- `portfolio-foundation`: exact source identity, full-history repository guard, deterministic - artifact envelopes, envelope verification, and exact-current-`main` GitHub evidence; -- `cloudflare-pages`: fail-closed Pages preflight, one pinned Wrangler direct upload, - deployment/live convergence bound to the created deployment ID, and an opt-in compiler for - authenticated Pages Functions sources that stage validated advanced-mode module trees. -- `python-package`: frozen dependency audit, non-root pure-Python wheel and sdist build, - metadata-derived tag verification, and a Foundation envelope for a separate source-free - official PyPA publisher job. The module never publishes to a registry. +Its own Dagger code then calls the modules like any other function, for example +`dag.foundation().guard(...)` before a build and `dag.cloudflare_pages().deploy(...)` to +ship the site. Its GitHub workflow stays tiny: check out the code, then call Dagger. -Start with the [exact-SHA consumer quickstart](docs/dagger-modules.md#quickstart). It captures -the central `main` SHA, validates all 40 lowercase hexadecimal characters, and commits that -literal dependency. The guide also includes a realistic Python composition, typed secret and -GitHub Environment rules, Python/TypeScript fixture proofs, and cold-engine verification. +The three modules: -The modules and their cross-language composition fixtures are implemented in this repository. -The guarded central merge establishes the remotely installable SHA. EdgeReco adoption and its -production canary are separate pending rollout steps; this change does not claim a production -deployment or fleet-wide module adoption. - -## Architecture - -Explore the [interactive runtime map](docs/architecture/index.html). - -## Execution model - -Only four workflows remain: - -| Workflow | Event ingress | Dagger function | +| Module | What it does | Used by | |---|---|---| -| `dagger.yml` | pull request, push to `main`, manual | `ci` | -| `consumer-drift.yml` | push to `main`, daily, manual | `fleet` | -| `dagger-security.yml` | weekly, manual | `security` | -| `module-canary.yml` | weekly, manual | `module-fixtures` | +| `portfolio-foundation` | Ties a build to one exact commit, runs secret and workflow scans over the full Git history, and wraps build output with a record of where it came from. | all nine repos | +| `cloudflare-pages` | Deploys a built site to Cloudflare Pages once, then checks the live site is serving that exact deployment. | almamesh, aml-filter, edge-reco | +| `python-package` | Audits dependencies, builds the wheel and sdist, and checks the version matches the tag. It does not upload to PyPI; a separate job does that with PyPI's official action. | edge-proc, edgeproc-core, agentic-context-service, agentic-saga | -Each job has exactly two pinned actions: +Upgrading is a pull request in the consumer that changes the commit in both `source` and +`pin`. Nothing changes for a consumer when this repo's `main` moves. -1. `actions/checkout` with `persist-credentials: false` -2. `dagger/dagger-for-github` pinned to a full commit SHA and Dagger 0.21.8 +## Try it -The module receives source through an explicit typed `dagger.Workspace` and stores an -explicit `dagger.Directory`. Credentials cross public Dagger functions as -`dagger.Secret`. Generated SDK bytes are mounted separately as toolchain data, so they -cannot silently expand the caller-selected source snapshot. +You need Docker and [Dagger 0.21.8](https://docs.dagger.io/install). -## What the central gate proves +1. List what the foundation module offers, straight from GitHub (about 15 seconds): -`dagger call ci` runs: + ```bash + dagger -m github.com/hseshadr/ci/modules/portfolio-foundation@faf55ac51dfeb6bad274988941e9215117e5259e functions + ``` -- Ruff formatting and linting -- strict mypy -- Xenon Grade A complexity -- pytest with at least 90% core coverage -- locked dependency audit -- actionlint -- Zizmor, failing on medium or high findings -- Gitleaks over both the exact source snapshot and complete Git history + ```text + Name Description + envelope Wrap a typed artifact with deterministic evidence. + green-main Resolve exact-green main evidence using a typed secret. + guard Apply repository security checks to a bound source. + source Bind a supplied workspace to a repository identity. + verify-envelope Revalidate and return only a closed envelope's artifact subtree. + ``` -The same graph runs locally and in GitHub. Hosted calls bind full-history scanning to -`${{ github.sha }}`. +2. Ask it whether a repo's `main` is green right now. This is the same call aml-filter + makes before it deploys: -## What fleet policy proves + ```bash + export GITHUB_TOKEN="$(gh auth token)" + dagger -m github.com/hseshadr/ci/modules/portfolio-foundation@faf55ac51dfeb6bad274988941e9215117e5259e \ + call green-main --github-token=env:GITHUB_TOKEN --repository=hseshadr/aml-filter serialization + ``` -For every exact consumer `main`, the scanner requires: + Real output from 25 Sep 2026, trimmed: -- every repository-authored workflow job is thin pinned Dagger ingress; -- source is an explicit typed `Directory` or `Workspace`; -- Dagger-exposed credential arguments are typed `Secret`; -- branch protection is strict and requires only `Dagger`, bound to GitHub Actions app - ID `15368`; -- the required `Dagger` check succeeded on the exact current `main` SHA; -- managed CodeQL default setup is disabled; -- no independent execution app controls the build or deploy path; -- no live workflow executes a retired `hseshadr/ci` reusable control. + ```json + {"app_id":15368,"branch":"main","check_name":"Dagger", + "commit_sha":"4392391bde505a8367fea87723e4ee8ef7bc4895", + "repository":"hseshadr/aml-filter","workflow_path":".github/workflows/dagger.yml", + "workflow_run_id":"36177431244", ...} + ``` -GitGuardian is allowed only as a non-required advisory observer. + `commit_sha` is aml-filter's current `main`, and the `Dagger` check passed on it. If the + check had not passed, the call would fail instead of returning. -### Approved transport exceptions +3. Run this repo's own checks from a clone (about 8 minutes): -The policy recognizes only two non-Dagger transports around a release candidate: + ```bash + git clone https://github.com/hseshadr/ci && cd ci + dagger call ci --github-token=env:GITHUB_TOKEN + ``` -- a pinned `upload-artifact` step after a successful unprivileged Dagger candidate; -- a source-free privileged job that downloads that exact run/SHA artifact, then either - invokes the official PyPI OIDC action with attestations or an exact-SHA remote Dagger - npm publisher with typed GitHub OIDC URL and token inputs. + Success ends with `central Dagger gate passed`. -Publisher bridges reject checkout, setup, install, build, test, free-form shell, mutable -references, excess permissions, wrong artifact identity, and missing provenance. +## How it works -This repository does not publish packages and its CI never dispatches a registry -mutation. +Every workflow in this repo and in the consumer repos does two things: check out the code, +and call one Dagger function. All the real work happens in Dagger. `dagger call ci` runs +linting, strict type checks, tests with a 90% coverage floor, a dependency audit, workflow +security scans, and a secret scan over the full Git history. It also builds two tiny consumer +modules, one in Python and one in TypeScript, to prove the shared modules install and run +from both. `dagger call fleet` reads every consumer repo's `main` through the GitHub API and +applies the rules in `.dagger/src/ci/fleet_policy.py`. If any repo cannot be read, that is a +failure, not a skip. -## Fleet token +## What it does not do -`CONSUMER_DRIFT_TOKEN` must be able to read all eight repositories. The authoritative -reader needs: - -- Contents: read -- Administration: read -- Checks: read -- Pull requests: read - -Administration read is required for effective branch protection and CodeQL default-setup -metadata. Checks read is required for exact-SHA app-bound integration evidence. The -scanner fails closed with a permission-specific message when either is unavailable. +- **It is for Harish's repos.** The modules assume his setup: GitHub, Cloudflare Pages, + PyPI, Dagger 0.21.8. You can read and borrow from them, but there is no support promise. +- **No reusable GitHub workflows or composite actions.** The old ones were removed; they + are in Git history and old tags only. +- **It never publishes anything.** No package, tag or registry upload comes from this repo. + `python-package` builds a package; publishing is a separate job in the consumer. +- **No automatic upgrades.** Consumers stay on their pinned commit until someone opens a PR. +- **Dependabot PRs are never auto-merged.** ## Develop -Use TDD and run the same enforced gate: +The quick local loop (about 1 minute once Dagger has generated the SDK): ```bash +dagger develop uv run --directory .dagger poe gate -uv run --directory .dagger poe audit -export GITHUB_TOKEN="$(gh auth token)" -dagger call ci --github-token=env:GITHUB_TOKEN -dagger call fleet --github-token=env:GITHUB_TOKEN ``` -Core policy lives in `.dagger/src/ci/fleet_policy.py`; GitHub's typed evidence adapter -lives in `.dagger/src/ci/github_fleet.py`. Behavioral tests live in `.dagger/tests/`. +The full check, the same one CI runs, is `dagger call ci --github-token=env:GITHUB_TOKEN`. +Each module under `modules/` has its own `poe gate` too. See +[Getting started](docs/GETTING_STARTED.md) for versions, the code map, and a worked first +change. + +## More detail + +- [Getting started](docs/GETTING_STARTED.md): set up, run the checks, make a first change, + open a PR. +- [Architecture](docs/ARCHITECTURE.md): the four workflows, exactly what `ci` and `fleet` + check, the allowed publishing exceptions, and the token `fleet` needs. +- [Interactive runtime map](docs/architecture/index.html). +- [Using the modules](docs/dagger-modules.md): installing at an exact commit, a full + consumer example, secrets, Pages safety and rollback. +- [Design (2026-08-27)](docs/superpowers/specs/2026-08-27-dagger-lego-architecture-design.md) + and the [first rollout plan](docs/superpowers/plans/2026-08-27-dagger-lego-edge-reco-canary.md): + why the shared modules exist. +- [CHANGELOG](CHANGELOG.md): what changed, including what was removed. -## Scope +## License -This is a control-plane repository, not a template catalog. Consumer-specific application builds -stay local; consumers compose Foundation, Pages, and the closed Python package candidate Lego -instead of copying shared trust mechanics. Privileged publisher jobs stay source-free and use -official registry actions outside Dagger. Dependabot may propose dependency updates, but its pull -requests are never auto-merged. +MIT. See [LICENSE](LICENSE). diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..7dffb52 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,144 @@ +# Architecture + +This repo is the shared CI for Harish's repositories. It has two jobs: + +1. Ship reusable Dagger modules that other repos install at an exact commit. +2. Check, every day, that each of those repos still runs its CI the agreed way. + +The rule behind both: **GitHub Actions only delivers events; Dagger runs everything.** A +workflow file checks out the code and calls one Dagger function. Tests, builds, scans and +deploys all happen inside Dagger, so the same steps run on a laptop and in GitHub. + +For an interactive picture, open the [runtime map](architecture/index.html) (its data is in +[runtime.architecture.json](architecture/runtime.architecture.json)). + +## The pieces + +| Path | What it is | +|---|---| +| `.dagger/src/ci/main.py` | The root Dagger module. Public functions: `ci`, `security`, `fleet`, `module-fixtures`. | +| `.dagger/src/ci/fleet_policy.py` | The rules every consumer repo must follow (pure Python, no network). | +| `.dagger/src/ci/github_fleet.py` | Reads the real state of each repo from the GitHub API and feeds it to the rules. | +| `.dagger/src/ci/fleet.py` | The list of consumer repos and what each must have (branch protection, deadlines). | +| `modules/portfolio-foundation/` | Shared module: exact source identity, repo safety checks, artifact envelopes, "is `main` green at this SHA" evidence. | +| `modules/cloudflare-pages/` | Shared module: checks and deploys a Cloudflare Pages site, then confirms the live site serves that deployment. | +| `modules/python-package/` | Shared module: audits, builds and checks a Python wheel and sdist. It never uploads to PyPI. | +| `tests/dagger/python_consumer/`, `tests/dagger/typescript_consumer/` | Tiny consumer modules that prove the shared modules work from Python and TypeScript code. | +| `.github/workflows/` | Four thin workflows that only call Dagger. | + +## Why consumers pin an exact commit + +A consumer installs a shared module at a full 40-character commit SHA, for example +`github.com/hseshadr/ci/modules/portfolio-foundation@`. Dagger writes both `source` and +`pin` into the consumer's `dagger.json`. The exact commit means a reviewer approved exactly +those bytes, and a consumer does not change behavior when this repo's `main` moves. Upgrading +is a deliberate pull request in the consumer. `main`, `latest`, tags and short SHAs are +rejected by the policy check. + +This repo itself uses its foundation module as a local, same-tree dependency (see +[`dagger.json`](../dagger.json)), so central CI always tests the module bytes in the current +commit. + +The full install steps, a realistic consumer flow, secrets rules and rollback are in +[dagger-modules.md](dagger-modules.md). + +## Workflows + +Only four workflows exist: + +| Workflow | Triggered by | Dagger function | +|---|---|---| +| `dagger.yml` | pull request, push to `main`, manual | `ci` (and `fleet` after a push to `main`) | +| `consumer-drift.yml` | push to `main`, daily, manual | `fleet` | +| `dagger-security.yml` | weekly, manual | `security` | +| `module-canary.yml` | weekly, manual | `module-fixtures` | + +Each job uses exactly two pinned actions: + +1. `actions/checkout` with `persist-credentials: false` +2. `dagger/dagger-for-github` pinned to a full commit SHA and Dagger 0.21.8 + +The module receives source through an explicit typed `dagger.Workspace` and stores an explicit +`dagger.Directory`. Credentials cross public Dagger functions as `dagger.Secret`. Generated SDK +bytes are mounted separately as toolchain data, so they cannot silently expand the +caller-selected source snapshot. + +## What `dagger call ci` checks + +- Ruff formatting and linting +- strict mypy +- Xenon Grade A complexity +- pytest with at least 90% core coverage (and 90% branch coverage) +- locked dependency audit +- actionlint +- Zizmor, failing on medium or high findings +- Gitleaks over both the exact source snapshot and complete Git history + +The same graph runs locally and in GitHub. Hosted calls bind full-history scanning to +`${{ github.sha }}`. + +## What `dagger call fleet` checks + +`fleet` reads the exact current `main` of every consumer repo from GitHub (plus this repo +with `--include-central`). Any unreadable or incomplete evidence is an error: a scan that +inspected nothing cannot report success. For every consumer it requires: + +- every repository-authored workflow job is thin pinned Dagger ingress; +- source is an explicit typed `Directory` or `Workspace`; +- Dagger-exposed credential arguments are typed `Secret`; +- branch protection is strict and requires only `Dagger`, bound to GitHub Actions app ID + `15368`; +- the required `Dagger` check succeeded on the exact current `main` SHA; +- managed CodeQL default setup is disabled; +- no independent execution app controls the build or deploy path; +- no live workflow executes a retired `hseshadr/ci` reusable control. + +GitGuardian is allowed only as a non-required advisory observer. + +The consumer list lives in `.dagger/src/ci/fleet.py`. + +### Approved transport exceptions + +The policy recognizes only two non-Dagger transports around a release candidate: + +- a pinned `upload-artifact` step after a successful unprivileged Dagger candidate; +- a source-free privileged job that downloads that exact run/SHA artifact, then either invokes + the official PyPI OIDC action with attestations or an exact-SHA remote Dagger npm publisher + with typed GitHub OIDC URL and token inputs. + +Publisher bridges reject checkout, setup, install, build, test, free-form shell, mutable +references, excess permissions, wrong artifact identity, and missing provenance. + +This repository does not publish packages and its CI never dispatches a registry mutation. + +## The `fleet` token + +`CONSUMER_DRIFT_TOKEN` must be able to read every consumer repository. It needs: + +- Contents: read +- Administration: read +- Checks: read +- Pull requests: read + +Administration read is required for effective branch protection and CodeQL default-setup +metadata. Checks read is required for exact-SHA app-bound check evidence. The scanner stops +with a permission-specific message when either is unavailable. + +Locally, `gh auth token` usually works for your own repos. + +## Scope + +This is a control-plane repository, not a template catalog. It publishes no reusable +workflows, composite actions or copyable CI templates (older ones were retired; they remain in +Git history and old tags, see [CHANGELOG](../CHANGELOG.md)). Consumer-specific application +builds stay in each consumer; consumers compose the foundation, Pages and Python package +modules instead of copying shared release mechanics. Privileged publisher jobs stay source-free +and use official registry actions outside Dagger. Dependabot may propose dependency updates, +but its pull requests are never auto-merged. + +## Design history + +- [Architecture design (2026-08-27)](superpowers/specs/2026-08-27-dagger-lego-architecture-design.md): + why the shared modules exist and how they are versioned. +- [Foundation and EdgeReco canary plan (2026-08-27)](superpowers/plans/2026-08-27-dagger-lego-edge-reco-canary.md): + the first rollout plan. diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md new file mode 100644 index 0000000..12f1da1 --- /dev/null +++ b/docs/GETTING_STARTED.md @@ -0,0 +1,133 @@ +# Getting started + +From a fresh clone to a green local build and your first change. Every command here was run +from a fresh clone on 25 Sep 2026 (macOS, Apple silicon); times are from that run. + +## 1. Prerequisites + +| Tool | Version | How to get it | +|---|---|---| +| Docker (or another engine Dagger supports) | any recent; tested with 29.2 | [Docker Desktop](https://docs.docker.com/get-docker/) | +| Dagger CLI | **exactly 0.21.8** (CI and every `dagger.json` pin `v0.21.8`) | `curl -fsSL https://dl.dagger.io/dagger/install.sh \| DAGGER_VERSION=0.21.8 BIN_DIR=/usr/local/bin sh` | +| uv | 0.8 or later; tested with 0.8.5 | `curl -LsSf https://astral.sh/uv/install.sh \| sh` | +| Python | 3.13 (uv downloads it for you) | nothing to do | +| GitHub CLI | any; used for a token | `brew install gh`, then `gh auth login` | + +Traps we hit: + +- **`uv run` fails on a fresh clone** with `Failed to generate package metadata for + dagger-io==0.0.0 @ editable+sdk ... Distribution not found at: .../.dagger/sdk`. The Python + projects depend on a generated Dagger SDK in `.dagger/sdk/`, which is not committed. Run + `dagger develop` once in the repo root (and once in each module you work on) first. +- **Dagger prints "A new release of dagger is available".** Ignore it. Do not upgrade past + 0.21.8 unless you are upgrading every pin on purpose. `export DAGGER_NO_NAG=1` hides it. +- **`dagger call ci` fails locally but the plain `poe gate` passed.** `ci` runs on the + committed and uncommitted files in your working tree, including new test files. That is + expected; read the failing test. + +## 2. Clone, set up, run the tests + +```bash +git clone https://github.com/hseshadr/ci +cd ci +dagger develop # generates .dagger/sdk (about 10 s) +uv run --directory .dagger poe gate # lint, types, complexity, tests, coverage +``` + +Success ends with a line like `224 passed` and a coverage table with `TOTAL ... 96%`, and no +`Error:` line. Took 1 min 12 s on the fresh clone (most of it is uv installing packages the +first time). + +## 3. The full check (what CI runs) + +```bash +export GITHUB_TOKEN="$(gh auth token)" +dagger call ci --github-token=env:GITHUB_TOKEN +``` + +This is the same call the `Dagger` job in `.github/workflows/dagger.yml` makes. It runs the +root `poe gate`, the dependency audit, the foundation module's repository checks (actionlint, +Zizmor, Gitleaks over the full history), and both consumer fixtures under `tests/dagger/`. +Success prints `central Dagger gate passed`. Took 8 min 11 s on the fresh clone. + +The daily cross-repo check is `dagger call fleet --github-token=env:GITHUB_TOKEN`. It needs a +token that can read the admin settings of every consumer repo (see +[Architecture](ARCHITECTURE.md#the-fleet-token)), so most contributors never run it locally. + +## 4. Map of the code + +| Path | What it is | +|---|---| +| `.dagger/src/ci/main.py` | Root Dagger module: `ci`, `security`, `fleet`, `module-fixtures`. | +| `.dagger/src/ci/fleet_policy.py` | The rules consumer repos must follow. Pure Python, easy to test. | +| `.dagger/src/ci/github_fleet.py` | Reads each repo's real state from the GitHub API. | +| `.dagger/src/ci/fleet.py` | The list of consumer repos and their expected branch protection. | +| `.dagger/tests/` | Tests for all of the above, plus contract tests for docs and workflows. | +| `modules/portfolio-foundation/` | Shared module: exact-commit identity, repo checks, artifact envelopes, green-main evidence. | +| `modules/cloudflare-pages/` | Shared module: Cloudflare Pages deploy, live check and rollback. | +| `modules/python-package/` | Shared module: Python wheel and sdist build and checks. | +| `tests/dagger/*_consumer/` | Tiny Python and TypeScript modules that install the shared modules, to prove they work from both languages. | +| `.github/workflows/` | Four thin workflows; each only checks out and calls Dagger. | + +Each module under `modules/` is its own Python project with its own `.dagger/pyproject.toml`, +tests and `poe gate`. + +## 5. Make your first change + +A typical change is a small fix inside one shared module. PR #53 is a real example: it taught +`python-package` to accept ZIP64 wheels. It touched two files: + +- `modules/python-package/.dagger/src/python_package/distribution_probe.py` +- `modules/python-package/.dagger/tests/test_distribution_probe.py` + +The steps: + +1. Branch: `git switch -c fix/python-package-`. +2. Set up the module once (about 10 s): + + ```bash + (cd modules/python-package && dagger develop) + ``` + +3. Write the failing test first in the module's `tests/` folder, and run just that file: + + ```bash + uv run --directory modules/python-package/.dagger pytest tests/test_distribution_probe.py -q --no-cov + ``` + + Watch it fail for the reason you expect, then make the smallest change in `src/` that + makes it pass. A single file runs in under a second. + +4. Run the module's full check (about 1 minute): + + ```bash + uv run --directory modules/python-package/.dagger poe gate + ``` + + Each module has a 90% line and branch coverage floor and Radon grade A complexity, like + the root. + +5. Run the root check, which also builds the Python and TypeScript consumer fixtures against + your changed module: `dagger call ci --github-token=env:GITHUB_TOKEN`. + +6. After your PR merges, consumers do **not** pick it up automatically. In each consumer repo + that needs the change, reinstall the module at the new `main` commit so both `source` and + `pin` in its `dagger.json` change, then open a PR there. The exact commands are in + [Using the modules](dagger-modules.md#quickstart). + +Changing a rule for consumer repos works the same way, with the code in +`.dagger/src/ci/fleet_policy.py` and the tests in `.dagger/tests/test_fleet_policy.py`. + +## 6. Open a PR + +- **Branch names** follow the change type: `feat/...`, `fix/...`, `docs/...`, `ci/...`. + Keep the branch short-lived; delete it after merge. +- **Commit and PR titles** use Conventional Commits with the module as scope, for example + `fix(python-package): accept ZIP64 wheels`. +- **CI** runs one required check, `Dagger` (`dagger call ci`). Branch protection requires it + to pass. After merge, `main` also runs the cross-repo check. +- **Reviewers look for:** a test that failed before the change; no drop in coverage or + complexity grade; typed inputs (`dagger.Directory`, `dagger.Secret`) at every public Dagger + function; no new workflow steps outside Dagger; and a CHANGELOG line for anything a + consumer would notice. +- Dependabot PRs are reviewed by hand and never auto-merged. From 60ac65b2bbb2c18c2b3e1d00822c98cdbba62efb Mon Sep 17 00:00:00 2001 From: Harish Seshadri Date: Fri, 25 Sep 2026 22:32:46 -0700 Subject: [PATCH 2/2] docs: explain --commit-sha for branch runs of dagger call ci Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_015oBArfm762nN1r4F4Fst5a --- README.md | 1 + docs/GETTING_STARTED.md | 22 +++++++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index bd8d554..e64ca55 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,7 @@ uv run --directory .dagger poe gate ``` The full check, the same one CI runs, is `dagger call ci --github-token=env:GITHUB_TOKEN`. +On a branch, push first and add `--commit-sha=$(git rev-parse HEAD)`. Each module under `modules/` has its own `poe gate` too. See [Getting started](docs/GETTING_STARTED.md) for versions, the code map, and a worked first change. diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index 12f1da1..f40e24e 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -21,9 +21,12 @@ Traps we hit: `dagger develop` once in the repo root (and once in each module you work on) first. - **Dagger prints "A new release of dagger is available".** Ignore it. Do not upgrade past 0.21.8 unless you are upgrading every pin on purpose. `export DAGGER_NO_NAG=1` hides it. -- **`dagger call ci` fails locally but the plain `poe gate` passed.** `ci` runs on the - committed and uncommitted files in your working tree, including new test files. That is - expected; read the failing test. +- **`dagger call ci` on a branch fails with `SourceMismatchError: workspace does not match + exact commit`.** Without `--commit-sha`, the history scan checks your files against + GitHub's `main`, so any change fails. Commit, push your branch, then pass + `--commit-sha=$(git rev-parse HEAD)`. The working tree must be clean and match that commit. +- **`ci` also runs files you have not committed yet**, including new tests. A test that + fails there but not in your editor is usually a new file you forgot about. ## 2. Clone, set up, run the tests @@ -45,6 +48,9 @@ export GITHUB_TOKEN="$(gh auth token)" dagger call ci --github-token=env:GITHUB_TOKEN ``` +On a clean clone of `main` that is all you need. On a branch, commit and push first, then +add `--commit-sha=$(git rev-parse HEAD)` (see the traps above). + This is the same call the `Dagger` job in `.github/workflows/dagger.yml` makes. It runs the root `poe gate`, the dependency audit, the foundation module's repository checks (actionlint, Zizmor, Gitleaks over the full history), and both consumer fixtures under `tests/dagger/`. @@ -107,8 +113,14 @@ The steps: Each module has a 90% line and branch coverage floor and Radon grade A complexity, like the root. -5. Run the root check, which also builds the Python and TypeScript consumer fixtures against - your changed module: `dagger call ci --github-token=env:GITHUB_TOKEN`. +5. Commit, push the branch, and run the root check. It also builds the Python and + TypeScript consumer fixtures against your changed module (about 4 minutes with a warm + cache): + + ```bash + git push -u origin HEAD + dagger call ci --github-token=env:GITHUB_TOKEN --commit-sha=$(git rev-parse HEAD) + ``` 6. After your PR merges, consumers do **not** pick it up automatically. In each consumer repo that needs the change, reinstall the module at the new `main` commit so both `source` and