diff --git a/.github/actions/install-cartesi-machine/action.yml b/.github/actions/install-cartesi-machine/action.yml new file mode 100644 index 00000000..b09c4617 --- /dev/null +++ b/.github/actions/install-cartesi-machine/action.yml @@ -0,0 +1,51 @@ +name: "Install cartesi-machine" +description: "Install the pinned machine-emulator .deb and export LIBCARTESI_PATH / INCLUDECARTESI_PATH for the Rust bindings." + +inputs: + version: + description: "cartesi-machine release tag" + required: true + sha256-amd64: + description: "SHA-256 for the amd64 .deb" + required: true + sha256-arm64: + description: "SHA-256 for the arm64 .deb" + required: true + +runs: + using: composite + steps: + - name: Install cartesi-machine + shell: bash + run: | + set -euo pipefail + ARCH="$(dpkg --print-architecture)" + VERSION="${{ inputs.version }}" + + case "${ARCH}" in + amd64) + DEB_SHA256="${{ inputs.sha256-amd64 }}" + ;; + arm64) + DEB_SHA256="${{ inputs.sha256-arm64 }}" + ;; + *) + echo "unsupported arch for machine-emulator: ${ARCH}" >&2 + exit 1 + ;; + esac + + wget -O /tmp/machine-emulator.deb "https://github.com/cartesi/machine-emulator/releases/download/${VERSION}/machine-emulator_${ARCH}.deb" + echo "${DEB_SHA256} /tmp/machine-emulator.deb" | sha256sum --check + sudo apt-get install -y /tmp/machine-emulator.deb + rm -f /tmp/machine-emulator.deb + cartesi-machine --version + + # testsi's cartesi-machine bindings (cartesi/dave) link this prebuilt + # emulator; they build only with LIBCARTESI_PATH set (external_cartesi). + # The static archive references libslirp, so callers must also install + # libslirp-dev. + test -f /usr/lib/libcartesi.a + test -f /usr/include/cartesi-machine/cm.h + echo "LIBCARTESI_PATH=/usr/lib" >> "${GITHUB_ENV}" + echo "INCLUDECARTESI_PATH=/usr/include/cartesi-machine" >> "${GITHUB_ENV}" diff --git a/.github/actions/setup-guest-toolchain/action.yml b/.github/actions/setup-guest-toolchain/action.yml index 972153dc..3b95829d 100644 --- a/.github/actions/setup-guest-toolchain/action.yml +++ b/.github/actions/setup-guest-toolchain/action.yml @@ -80,29 +80,11 @@ runs: sudo apt-get install -y /tmp/xgenext2fs.deb - name: Install cartesi-machine - shell: bash - run: | - set -euo pipefail - ARCH="$(dpkg --print-architecture)" - VERSION="${{ inputs.cartesi-machine-version }}" - - case "${ARCH}" in - amd64) - DEB_SHA256="${{ inputs.cartesi-machine-sha256-amd64 }}" - ;; - arm64) - DEB_SHA256="${{ inputs.cartesi-machine-sha256-arm64 }}" - ;; - *) - echo "unsupported arch for machine-emulator: ${ARCH}" >&2 - exit 1 - ;; - esac - - wget -O /tmp/machine-emulator.deb "https://github.com/cartesi/machine-emulator/releases/download/${VERSION}/machine-emulator_${ARCH}.deb" - echo "${DEB_SHA256} /tmp/machine-emulator.deb" | sha256sum --check - sudo apt-get install -y /tmp/machine-emulator.deb - cartesi-machine --version + uses: ./.github/actions/install-cartesi-machine + with: + version: ${{ inputs.cartesi-machine-version }} + sha256-amd64: ${{ inputs.cartesi-machine-sha256-amd64 }} + sha256-arm64: ${{ inputs.cartesi-machine-sha256-arm64 }} - name: Set up QEMU uses: docker/setup-qemu-action@v4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ae4ff6b..cb433a43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,8 +28,20 @@ jobs: libfaketime \ lua5.4 \ liblua5.4-dev \ + lua-check \ + libcurl4-openssl-dev \ libslirp-dev + # canonical-test links the emulator through testsi's cartesi-machine + # bindings, so the workspace build needs the pinned CM .deb and the + # LIBCARTESI_PATH / INCLUDECARTESI_PATH this action exports. + - name: Install cartesi-machine + uses: ./.github/actions/install-cartesi-machine + with: + version: ${{ env.CARTESI_MACHINE_VERSION }} + sha256-amd64: ${{ env.CARTESI_MACHINE_SHA256_AMD64 }} + sha256-arm64: ${{ env.CARTESI_MACHINE_SHA256_ARM64 }} + - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: @@ -53,11 +65,13 @@ jobs: - name: Clippy run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings - - name: Watchdog Lua tests - run: lua5.4 watchdog/tests/run.lua + - name: Watchdog Lua lint + run: cd watchdog && luacheck --no-color *.lua tests/*.lua - - name: Watchdog divergence drill - run: bash scripts/test-watchdog-divergence-drill.sh + - name: Watchdog Lua unit tests + run: | + bash scripts/watchdog-lua-deps.sh + lua5.4 watchdog/tests/run.lua - name: Test timeout-minutes: 15 @@ -132,9 +146,8 @@ jobs: - name: Run rollups E2E tests (Rust and C hosts) run: just test-rollups-e2e - # Runs after the e2e step so the canonical machine image is already built; - # exercises the in-process machine_cartesi binding incl. store -> reload -> advance, - # which the Rust harness never loads (its compare passes only load the genesis image). + # Runs after the e2e step so the canonical images already exist. Builds the + # watchdog test guest and checks the executor against the reference CLI. - name: Watchdog Lua CM e2e run: just test-watchdog-e2e diff --git a/AGENTS.md b/AGENTS.md index 500dcdd6..36324df6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -162,6 +162,7 @@ signing, and SSZ batch encoding. - `examples/canonical-app/` — on-chain scheduler reference implementation. - `examples/canonical-test/` — e2e test harness for the canonical app. - `sdk/rust-client/` — Rust client library for the sequencer API. +- `sdk/guest/` — canonical-guest SDK: `trolley` (guest rollup API over `libcmt-sys`), `rollups-types` (rollups ABI encodings), and `testsi` (host-side guest image tests); provenance in its README. - `tests/{benchmarks,e2e,harness}/` — test infrastructure. ### Sequencer module layout @@ -370,7 +371,9 @@ Prefer black-box tests around `POST /tx` and commit outcomes for integration. Some `sequencer` tests use Anvil (Foundry). They run by default and fail with a clear message if `anvil` is not on PATH. Use the configured Nix/direnv environment -or install Foundry. `canonical-test` additionally needs libslirp. +or install Foundry. `canonical-test` additionally needs the Cartesi Machine +library named by `LIBCARTESI_PATH`/`INCLUDECARTESI_PATH` (the devshell exports +both) and libslirp. ## Shell and Commands @@ -453,5 +456,6 @@ work reaches another boundary. | Trust boundaries, provider behavior, or hostile L1 input | [Threat model](docs/threat-model/README.md) — actor assumptions, supported failures, and residual risks. | | Submission fees or oracle pricing | [L1 fee policy](docs/l1-fee-policy.md) — estimation and replacement limits; [threat-model actor table](docs/threat-model/README.md#actors-and-trust) — oracle source and outage assumptions. | | Command setup or deployment configuration | [Running](README.md#running) and [config.rs](sequencer/src/commands/config.rs) — invocation, identity pinning, defaults, and validation. | -| Watchdog development or operation | [Architecture](docs/watchdog/README.md); [local dev](docs/watchdog/getting-started.md) for Anvil; [operator deployment](docs/watchdog/operator-deployment.md) for Sepolia/mainnet. | +| Watchdog development or operation | [Watchdog README](docs/watchdog/README.md) — contract, state sources, commands, and state; [incident runbook](docs/watchdog/incident-runbook.md) — what a latched divergence requires; [local dev](docs/watchdog/getting-started.md) for Anvil; [operator deployment](docs/watchdog/operator-deployment.md) for Sepolia/mainnet. | +| Canonical guest, machine images, or a Cartesi Machine bump | [Cartesi Machine facts](docs/cartesi-machine.md) — version pinning, host semantics, storage, memory ranges, and the guest side. | | A new mechanism, simplification, or work spanning an active track | Owning design and [invariants](docs/invariants.md) — reasons and assumptions; [review register](docs/review/register.md) — unresolved work; [coordination tracks](docs/plans/2026-07-coordination-tracks.md) — priorities and dependencies. Follow the [review lifecycle](docs/review/README.md) when recording conclusions. | diff --git a/Cargo.lock b/Cargo.lock index bfd831a5..c05bf0b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -758,17 +758,11 @@ dependencies = [ "alloy-sol-types", "ethereum_ssz", "ethereum_ssz_derive", + "rollups-types", "sequencer-core", "tracing", - "types", ] -[[package]] -name = "ar" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d67af77d68a931ecd5cbd8a3b5987d63a1d1d1278f7f6a60ae33db485cdebb69" - [[package]] name = "ark-ff" version = "0.3.0" @@ -1437,9 +1431,9 @@ dependencies = [ "alloy-primitives", "app-core", "k256", + "rollups-types", "sequencer-core", "trolley", - "types", ] [[package]] @@ -1450,15 +1444,15 @@ dependencies = [ "canonical-app", "ethereum_ssz", "k256", + "rollups-types", "sequencer-core", "testsi", - "types", ] [[package]] name = "cartesi-machine" version = "2.0.0" -source = "git+https://github.com/cartesi/dave?branch=feature%2Fbump-emulator-2-coutinho#317a88a32a1b4a6002d561f37bb833843df55cba" +source = "git+https://github.com/cartesi/dave?rev=79874c0a1b40678465daf533acc976c88301c6d4#79874c0a1b40678465daf533acc976c88301c6d4" dependencies = [ "base64", "cartesi-machine-sys", @@ -1472,13 +1466,11 @@ dependencies = [ [[package]] name = "cartesi-machine-sys" version = "2.0.0" -source = "git+https://github.com/cartesi/dave?branch=feature%2Fbump-emulator-2-coutinho#317a88a32a1b4a6002d561f37bb833843df55cba" +source = "git+https://github.com/cartesi/dave?rev=79874c0a1b40678465daf533acc976c88301c6d4#79874c0a1b40678465daf533acc976c88301c6d4" dependencies = [ "bindgen", - "cfg-if", - "hex-literal 1.1.0", "link-cplusplus", - "sha1", + "sha2", ] [[package]] @@ -2484,18 +2476,6 @@ dependencies = [ "arrayvec", ] -[[package]] -name = "hex-literal" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" - -[[package]] -name = "hex-literal" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e712f64ec3850b98572bffac52e2c6f282b29fe6c5fa6d42334b30be438d95c1" - [[package]] name = "hmac" version = "0.12.1" @@ -2593,7 +2573,6 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots", ] [[package]] @@ -2984,16 +2963,8 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libcmt-sys" version = "0.1.0" -source = "git+https://github.com/GCdePaula/cartesi-tools-rs?rev=ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22#ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22" dependencies = [ - "ar", "bindgen", - "bytes", - "hex-literal 0.4.1", - "reqwest 0.12.28", - "sha2", - "tar", - "xz2", ] [[package]] @@ -3085,17 +3056,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "macro-string" version = "0.2.0" @@ -3763,28 +3723,22 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64", "bytes", - "futures-channel", "futures-core", "futures-util", "http", "http-body", "http-body-util", "hyper", - "hyper-rustls", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -3794,7 +3748,6 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", ] [[package]] @@ -3915,6 +3868,14 @@ dependencies = [ "toml", ] +[[package]] +name = "rollups-types" +version = "0.1.0" +dependencies = [ + "alloy-primitives", + "alloy-sol-types", +] + [[package]] name = "rsqlite-vfs" version = "0.1.1" @@ -4036,7 +3997,6 @@ checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "aws-lc-rs", "once_cell", - "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -4293,6 +4253,7 @@ dependencies = [ "sequencer-rust-client", "serde", "serde_json", + "sha2", "tar", "tempfile", "thiserror 2.0.19", @@ -4734,20 +4695,18 @@ dependencies = [ [[package]] name = "testsi" version = "0.1.0" -source = "git+https://github.com/GCdePaula/cartesi-tools-rs?rev=ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22#ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22" dependencies = [ "cartesi-machine", "inventory", "libtest-mimic", + "rollups-types", "testsi-macros", "thiserror 1.0.69", - "types", ] [[package]] name = "testsi-macros" version = "0.1.0" -source = "git+https://github.com/GCdePaula/cartesi-tools-rs?rev=ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22#ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22" dependencies = [ "proc-macro2", "quote", @@ -5136,10 +5095,9 @@ dependencies = [ [[package]] name = "trolley" version = "0.1.0" -source = "git+https://github.com/GCdePaula/cartesi-tools-rs?rev=ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22#ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22" dependencies = [ "libcmt-sys", - "types", + "rollups-types", ] [[package]] @@ -5187,15 +5145,6 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" -[[package]] -name = "types" -version = "0.1.0" -source = "git+https://github.com/GCdePaula/cartesi-tools-rs?rev=ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22#ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22" -dependencies = [ - "alloy-primitives", - "alloy-sol-types", -] - [[package]] name = "ucd-trie" version = "0.1.7" @@ -5419,6 +5368,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "watchdog-test-guest" +version = "0.1.0" +dependencies = [ + "libc", + "libcmt-sys", + "trolley", +] + [[package]] name = "web-sys" version = "0.3.103" @@ -5448,15 +5406,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "webpki-roots" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "winapi-util" version = "0.1.11" @@ -5656,15 +5605,6 @@ dependencies = [ "rustix", ] -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index 4846cc62..dd1c208c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,12 @@ members = [ "tests/benchmarks", "tests/harness", "tests/e2e", + "watchdog/test-guest", + "sdk/guest/libcmt-sys", + "sdk/guest/trolley", + "sdk/guest/testsi", + "sdk/guest/testsi/testsi-macros", + "sdk/guest/rollups-types", ] default-members = ["sequencer", "examples/wallet-sequencer"] @@ -33,12 +39,11 @@ authors = [ ] [workspace.dependencies] -# ── Git-pinned Cartesi tooling ─────────────────────────────── -libcmt-sys = { version = "0.1", git = "https://github.com/GCdePaula/cartesi-tools-rs", rev = "ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22" } -trolley = { version = "0.1", git = "https://github.com/GCdePaula/cartesi-tools-rs", rev = "ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22" } -testsi = { version = "0.1", git = "https://github.com/GCdePaula/cartesi-tools-rs", rev = "ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22" } -types = { version = "0.1", git = "https://github.com/GCdePaula/cartesi-tools-rs", rev = "ed14b98ecfe9796dc3ca7c9b96bfdbf0ef9baf22" } -cartesi-machine = { version = "2", git = "https://github.com/cartesi/dave", rev = "317a88a32a1b4a6002d561f37bb833843df55cba", features = ["download_uarch"] } +# ── Guest SDK: canonical-guest crates (see sdk/guest/README.md) ── +libcmt-sys = { version = "0.1", path = "sdk/guest/libcmt-sys" } +trolley = { version = "0.1", path = "sdk/guest/trolley" } +testsi = { version = "0.1", path = "sdk/guest/testsi" } +rollups-types = { version = "0.1", path = "sdk/guest/rollups-types" } # ── Contracts (exact pin — the single place the version lives) ── cartesi-rollups-contracts = "=3.0.0-alpha.6" diff --git a/README.md b/README.md index acdbaec5..db0278ad 100644 --- a/README.md +++ b/README.md @@ -427,10 +427,14 @@ api split lands). - `GET /finalized_state/inclusion_block` — cheap JSON the watchdog polls: `{ "inclusion_block": , "executed_input_count": }`. +- `GET /finalized_state/digest` — the SHA-256 of the accepted checkpoint's + comparison file, as lowercase hex, hashed under the same lease: + `{ "inclusion_block": , "executed_input_count": , "sha256": "" }`. + The watchdog replays to that block and compares digests. - `GET /finalized_state` — streams the accepted checkpoint's comparison file (`application/octet-stream`), with `X-Inclusion-Block`, `X-Executed-Input-Count`, and `ETag: "block-"` (`If-None-Match` supports `304`). - The watchdog compares at the end of that L1 block. + The watchdog downloads it only as divergence evidence. - `GET /latest_snapshot` — streams a restorable tar archive of the newest valid batch-close snapshot, or the era baseline. Includes immutable `info.toml` and the application's opaque `state` file or directory. @@ -444,7 +448,7 @@ Streaming holds the lease until the response ends or the client disconnects. The accepted endpoints return `404` until a comparable checkpoint exists: genesis is comparable at block zero; a rebuilt baseline is restorable but only a later accepted batch establishes a comparison point. Known divergence makes -all three finalized endpoints return `503 UNAVAILABLE`, including conditional +all four finalized endpoints return `503 UNAVAILABLE`, including conditional state requests. The check shares the checkpoint-selection transaction, before any lease or archive is created. See [snapshot lifecycle](docs/snapshots/lifecycle.md). diff --git a/bindings/c-app-engine/include/application-engine.h b/bindings/c-app-engine/include/application-engine.h index e332ad9f..f1f951f7 100644 --- a/bindings/c-app-engine/include/application-engine.h +++ b/bindings/c-app-engine/include/application-engine.h @@ -372,9 +372,10 @@ APPLICATION_ENGINE_API void application_engine_progress(const ApplicationEngine /// Deletion must leave other checkpoints and independently restored engines usable. Filesystem /// CoW sharing is allowed when writes remain isolated. /// -/// The file named by application_engine_state_file_in_dump must byte-equal the canonical build's -/// deterministic inspection or designated state-drive representation for the same logical state. -/// The bridge does not verify this cross-build equivalence. +/// The file named by application_engine_state_file_in_dump must byte-equal what the canonical +/// machine yields after the same inputs, also for an engine restored from a dump: its whole labeled +/// state range, or its `state` inspect report (docs/protocol/application-contract.md, "Comparison +/// bytes"). The bridge does not verify this cross-build equivalence. APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_create_dump(ApplicationEngine *engine, const char *prefix) APPLICATION_ENGINE_NOEXCEPT; @@ -385,8 +386,7 @@ APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_create_dump(Ap /// file sits follows from the shape the engine gives a dump, which is why the engine answers /// rather than a host assuming. An engine whose dump is a directory answers with a file inside /// it, and one whose dump is the state image itself answers with the prefix unchanged. -/// Its bytes must match the independent canonical build's deterministic comparison representation -/// for the same logical state, as required by application_engine_create_dump. +/// Its bytes are the comparison bytes that application_engine_create_dump requires. /// /// The storage is engine owned and thread local, overwritten by the next call on the same /// thread, so copy rather than retain the pointer. Being fallible, it also clears the last error diff --git a/docs/cartesi-machine.md b/docs/cartesi-machine.md new file mode 100644 index 00000000..288dbebf --- /dev/null +++ b/docs/cartesi-machine.md @@ -0,0 +1,164 @@ +# Cartesi Machine Facts We Rely On + +The canonical application runs inside the Cartesi Machine (CM). The watchdog +drives it, the canonical guest is built for it, and cockroach recovery exports +from it. This page records the CM behavior those components depend on, +especially behavior that is easy to assume wrongly. Everything here holds for +the pinned version (`CARTESI_MACHINE_VERSION` in +[`toolchain-pins.env`](../toolchain-pins.env), currently v0.21.0) and was +checked against the emulator source at that tag. Re-check it on every bump. + +## Versions are all-or-nothing + +A stored machine records a config archive version and the emulator's machine +identifiers; loading rejects any mismatch. A CM bump therefore invalidates every +stored image, release tarball, and watchdog checkpoint: rebuild the canonical +images and re-`init` watchdog state directories. The kernel +(`machine-linux-image`) and guest tools (`machine-guest-tools`) are paired with +the emulator release and move with it. The development shell pins the same +emulator through the parent flake. + +**A deployed application is pinned to its emulator version.** Its on-chain +template hash is the root hash of a machine built with one emulator release, +and the machine identifiers are part of that hash. A machine rebuilt with +another release has a different root hash, and the old release's stored +machines do not load. Every canonical machine for a deployment, the +watchdog's included, must run the release its template was built with; moving +to a new emulator release means a new application deployment. + +## Rollup host semantics + +The reference host loop is the `cartesi-machine` CLI (`run_advance_state_epoch` +in `src/cartesi-machine.lua`); Dave's PRT client implements the same rules. +Any other host, including the watchdog, must match it exactly: + +| Guest outcome after an input | Host action | +|---|---| +| `RX_ACCEPTED` manual yield | commit the input's state | +| `RX_REJECTED` manual yield | restore the pre-input snapshot | +| `TX_EXCEPTION`, halt, any other manual yield, or mcycle overflow | fixed point: the machine stays there forever, with no revert | + +- The host reads the root hash, snapshots, then sends the input together with + that root as the *revert root hash*. The emulator refuses an advance unless + the machine sits at an `RX_ACCEPTED` yield and the revert root hash equals the + current root, so a host cannot silently continue from a rejected state. +- Each input gets an mcycle budget of 2^48 (`imcyclemax`). Exhausting it is a + fixed point, not a reject. At realistic speeds this takes days, so a stuck + guest shows up as a hung host long before the budget ends. +- An inspect query is legal only at an `RX_ACCEPTED` yield and runs on a + snapshot the host discards. Inspect execution is never part of canonical + history, and its single report is bounded by the 2 MiB CMIO TX buffer. + +A fixed point is permanent: once the canonical machine reaches one, no later +input is ever processed for that application. + +## Storage, sharing, and snapshots + +- **Loading** defaults to `SHARING_NONE`: every backing file is mapped + privately and never modified, under a shared `flock`. `SHARING_ALL` maps the + files shared and runs the machine in place on disk, under an exclusive + `flock`, so one directory cannot be open both ways at once. +- **`store`** writes every range in full as sparse files, refuses an existing + directory, and does **not** fsync. Durable publication uses `sync_stored` + (fsyncs files, the directory, and its parent), then `rename_stored` (atomic, + no-replace, fsyncs both parents). `remove_stored` also syncs the parent. +- **`clone_stored`** reflinks writable files (`clonefile` on macOS APFS, + `FICLONE` on Linux btrfs/XFS), hardlinks read-only ones, and falls back to a + sparse copy when the filesystem cannot link (`ENOTSUP`, `EXDEV`, `EPERM`). It + always works; it is cheap only on copy-on-write filesystems. +- **Snapshots come in two families.** An in-place (`SHARING_ALL`) machine + snapshots by closing, cloning its directory, and reopening; this is the CLI's + `--revert-mode=stored` and Dave's sling node. The CLI's default + `--revert-mode=fork` instead forks a JSON-RPC machine server process, which + refuses to fork a machine with shared ranges (both processes would see the + writes). We run the machine in-process and use the clone family only. + +Measured on APFS (Apple M5 Max) with a machine holding a 1 GiB NVRAM, 768 MiB +of it non-zero: + +| Operation | Cost | +|---|---| +| Write 768 MiB in place and compute the root hash | 0.68 s | +| Clone the whole stored machine | ~0.1 s, 64 KiB of new disk | +| Per input: root hash, close, clone, reopen, dirty 16 pages, commit or revert | ~50 ms; 100 inputs grew disk by ~40 MB | + +Every revert in that run restored exactly the recorded pre-input root hash. On +a filesystem without reflinks, each clone is a full sparse copy of the +machine. + +## Memory ranges for application state + +An application can keep canonical state in a labeled memory range instead of +answering an inspect query: + +- **NVRAM** (`--nvram=label:,...`) appears in the guest as `/dev/uioN`. It + has no filesystem layer and no page cache: guest writes through `mmap` are + immediately in the machine state. It requires the paired kernel with UIO + (ctsi-2 and later). +- **Flash drives** appear as `/dev/pmemN`. Guest writes go through the page + cache, so the application must flush (`fsync`, `msync(MS_SYNC)`, or + `O_DIRECT`) before finishing each input; otherwise the drive holds a stale + mix of pages. +- **A comparison range must be raw.** The CLI formats a flash drive that has no + data file (`mke2fs`) and mounts one that has; either puts filesystem metadata + the native engine cannot reproduce into the range. Declare state drives with + `mke2fs:false,mount:false`, or use an NVRAM. +- **Labels.** User labels are stored in the machine config (`flash_drive[i].label`, + `nvram[i].label`). The guest's device tree aliases both the user label and the + automatic name (`flashdriveN`, `nvramN`), but the config holds only the user + label, and `cartesi.util.find_drive` matches user labels only. +- Read a range with `machine:read_memory(start, length)`. The stored file layout + (`-.bin`) is an internal format; do not depend on it. + +## The guest side + +- **The guest–host interface is unchanged from v0.20.** Yield reasons, CMIO + buffer addresses, and ioctl numbers are the same; only the names moved to + `HTIF_*`. The revert root hash and the per-input cycle budget are host-side. + libcmt 0.18 bundles its ioctl header, so the guest build needs no kernel + headers. +- **Some guest tools are glibc builds.** The `nvram`/`flashdrive` label helpers, + which the CLI's generated init script calls for every NVRAM and for flash + drives it formats, mounts, or chowns, and `xhalt`, which `cartesi-init` calls + to halt with the entrypoint's exit status, do not run on a musl (Alpine) + rootfs. Without `xhalt`, a failing entrypoint halts with payload 0. The + watchdog test guest ships musl stand-ins for both; the canonical wallet image + does not, so its panics halt with exit code 0. +- **trolley has no exception call.** A guest that must raise a CMIO exception + does it through `libcmt-sys` directly (see `watchdog/test-guest`). +- **The last input stays in the machine.** The CMIO receive buffer keeps it, + and it is part of the machine state: root hashes agree only for byte-identical + inputs, metadata included. + +## Hashing + +- `cartesi.sha256(data)` and `cartesi.keccak256(data)` hash a single Lua string + in one shot, in place; neither the Lua module nor the C API exports a + streaming hasher. +- `machine:read_memory(start, length)` fills a native buffer and copies it into + a new Lua string, so reading a whole range holds it twice for a moment. It + takes any start and length, so a range can be read in chunks. +- The machine's own hash tree uses keccak256 by default (sha256 is + configurable), with 32-byte leaves and 4 KiB pages. `get_node_hash(start, + ceil_log2(length))` returns a range's Merkle root at the cost of rehashing + dirty pages only, and `get_proof` ties it to the machine root hash. Any party + comparing against that root must reproduce the same tree. + +## The CLI as a test oracle + +The CLI is a script, not a module: it parses the global `arg` and ends in +`os.exit`, so it cannot be `require`d. Its building blocks can (`cartesi`, +`cartesi.util`, `cartesi.hash-tree`, `cartesi.jsonrpc`). When tests run the CLI +as a reference: + +- `--cmio-advance-state` checks the outputs Merkle root from genesis by default; + resuming from a mid-history checkpoint needs `check_outputs_merkle_root:false`. +- `--max-mcycle` reached mid-input exits 0 and stores the mid-input state. +- Exit codes do not distinguish outcomes (a halt with payload 0 exits 0); load + the stored machine and read its state instead, cross-checked with + `--final-hash`. +- An inspect query is delivered even to a machine at an exception yield, and + its own outcome is not reported. +- `--remote-spawn` leaves machine servers running unless `--remote-shutdown` is + also given. `--no-rollback` no longer exists; use `--no-revert`, which cannot + get past a rejected input: the emulator refuses the next advance. diff --git a/docs/plans/2026-09-watchdog-v2.md b/docs/plans/2026-09-watchdog-v2.md new file mode 100644 index 00000000..b549ca35 --- /dev/null +++ b/docs/plans/2026-09-watchdog-v2.md @@ -0,0 +1,48 @@ +# Watchdog v2 + +**Status:** implemented (2026-09-22). The design lives with its owners; this +file keeps the remaining work. + +| Topic | Owner | +|---|---| +| Runtime contract, state sources, commands, configuration, state directory, metrics, and why it is built this way | [Watchdog README](../watchdog/README.md) | +| Divergence playbook and drills | [Incident runbook](../watchdog/incident-runbook.md) | +| Emulator behavior and measurements | [Cartesi Machine facts](../cartesi-machine.md) | +| Comparison bytes an application must produce | [Application contract §6](../protocol/application-contract.md#6-checkpoint-lifecycle) | +| `GET /finalized_state/digest` | [README API](../../README.md#operator-snapshot-endpoints-internal-only) | + +## Remaining work + +- **CI on GitHub.** The workflow changes (emulator install for the `rust` + job, native module build, `luacheck`, the test guest build in the + `rollups-e2e` job) have not run on GitHub; their commands pass locally. +- **Staging drill.** Run the divergence drill once against staging with the + operators who will be paged ([runbook](../watchdog/incident-runbook.md#drills)). +- **Measure at scale.** Per-input snapshot and per-tick clone costs on the + deployment filesystem with a DEX-sized state (reflink XFS or btrfs; on ext4 + every snapshot copies the machine's allocated data), and the tick's peak + memory (about twice the range, from `read_memory`). If memory binds, stream + the same flat SHA-256 over chunked reads, which needs a vendored incremental + hasher ([why](../watchdog/README.md#why-it-is-built-this-way)). +- **Restart and recovery comparisons.** A range-source application needs an + end-to-end test that restarts the sequencer partway through its history + (`from_dump`, then replay), and one that recovers it from an extracted + range, each followed by a watchdog comparison. The wallet's sorted SSZ file + cannot catch a layout that depends on history. +- **Guest exit codes.** The canonical image lacks a musl `xhalt`, so a + panicking canonical application halts with payload 0 and the watchdog reports + exit code 0. The test guest's `xhalt` can be shared. +- **trolley exceptions.** Add an exception call to `sdk/guest/trolley`; the + test guest raises its exception through `libcmt-sys` meanwhile. + +## Open questions for the DEX integration + +- Is `M` exactly the whole labeled NVRAM or drive, zero tail included, with the + same length as the native comparison file? +- Is `M` byte-identical between native and guest execution after the same + inputs, including after a sequencer restart and after recovery from an + extracted `M`? Does any layout-affecting state (allocator free lists, hash + seeds, capacity policy) live outside `M` or depend on the process? +- Which Cartesi Machine version does the DEX image pin? +- Does the guest ever reject an input? The watchdog handles it either way. +- How large is `M` expected to grow? diff --git a/docs/protocol/application-contract.md b/docs/protocol/application-contract.md index f9af109a..7776b9e3 100644 --- a/docs/protocol/application-contract.md +++ b/docs/protocol/application-contract.md @@ -179,12 +179,30 @@ machine checkpoint may instead contain a separate app-state projection. |---|---|---| | Wallet | SSZ wallet state | The same SSZ file, also returned by canonical inspect | | Cartesi Machine wrapper | Full multi-file machine state | Deterministic app-state projection stored alongside it | -| Native DEX design | Fixed-memory state `M` plus required resumable metadata | Canonical `M`, matching the designated drive in the canonical machine | +| Native DEX design | Fixed-memory state `M` plus required resumable metadata | `M` exactly as the canonical machine's labeled state range holds it | The DEX row describes an integration requirement, not a verified private -implementation. The [watchdog guide](../watchdog/README.md) owns comparison -transport and support; the [wallet format](../snapshots/format.md) owns its SSZ -representation. +implementation. The [watchdog guide](../watchdog/README.md#state-sources) owns +how the comparison is transported and which state source a deployment uses; +the [wallet format](../snapshots/format.md) owns its SSZ representation. + +**Comparison bytes.** The comparison file must byte-equal what the canonical +machine yields after the same inputs, through the deployment's state source, +whether the engine ran them uninterrupted or was restored with `from_dump` or +rebuilt by recovery partway. For a range source, anything that shapes the +range's layout (allocator state, hash seeds, capacity policy) belongs to the +checkpoint and must not depend on the process: + +- **Range source**: the entire flash drive or NVRAM carrying the configured + user label, all of its configured length including any unused zero tail, + raw: no filesystem, header, or trimming. The guest must have written the + state into the range before it finishes each input. NVRAM has no page cache; + a flash drive goes through the guest page cache, so the guest must flush + (`fsync`, `msync(MS_SYNC)`, or `O_DIRECT`) first. The + [Cartesi Machine facts](../cartesi-machine.md#memory-ranges-for-application-state) + explain why the range must be raw. +- **Inspect source**: the single report the canonical application returns for + the inspect query `state`. - `create_dump(&mut self, prefix)` creates a checkpoint at an absent path, which may become a file or directory. On `Ok`, all files and directory @@ -197,9 +215,7 @@ representation. checkpoint or another restored instance. The restored engine must remain usable after the source checkpoint is garbage-collected. - `state_file_in_dump(prefix)` is a pure path function naming a single file, - possibly `prefix` itself. Its bytes match the canonical application's - deterministic comparison representation, whether obtained through inspect - or from a designated state drive. + possibly `prefix` itself. Its bytes are the comparison bytes above. - All checkpoint-owned artifacts reside at or below `prefix`, including the canonical comparison file. Disposal requires only ordinary filesystem deletion: the sequencer removes the enclosing directory, including its own diff --git a/docs/recovery/cockroach.md b/docs/recovery/cockroach.md index 34be82c6..c714065e 100644 --- a/docs/recovery/cockroach.md +++ b/docs/recovery/cockroach.md @@ -82,8 +82,10 @@ latest possible sound checkpoint is not a prerequisite for recovery. 1. **Stop and preserve.** Stop the sequencer and prevent automatic restarts. Preserve its data directory, logs, and available archives before rebuilding. - Pause watchdog ticks while copying its selected checkpoint, manifest, - `head.json`, and configuration. Preserve affected client databases and their + Copy the watchdog state directory's `config.json`, its head under + `checkpoints/`, and its `incident/` evidence once `status` no longer shows + the evidence collection `running`; a latched watchdog does no other work, + so this needs no pause. Preserve affected client databases and their checkpoint metadata separately. 2. **Establish the cause and reference.** Check deployment identity, canonical machine image, bootstrap boundary, and the reported comparison boundary. @@ -93,7 +95,12 @@ latest possible sound checkpoint is not a prerequisite for recovery. 3. **Select a sound application checkpoint.** The watchdog's durable head is its last successful comparison checkpoint, or its operator-trusted initial checkpoint if no comparison succeeded. A failed comparison does not replace - it; initialization and idle ticks are not successful comparisons. Use that + it; initialization and idle ticks are not successful comparisons. A latched + `state_mismatch` usually also keeps `incident/canonical` (its evidence + lists it), the canonical machine at the latched block, derived + independently of the sequencer, and + `sequencer-watchdog replay` re-derives one at any block from a trusted + machine ([incident runbook](../watchdog/incident-runbook.md)). Use that canonical state, or independently validate a retained candidate at the same exact L1 boundary. Current-state equality can establish a usable application state without establishing that all earlier executions were correct. @@ -116,8 +123,8 @@ The watchdog stores a whole CM, including scheduler state; it does not save a native `/finalized_snapshot` archive. Use the application's rehearsed canonical export command to obtain the native bundle, or a retained native archive whose state and resume metadata can be validated against the canonical reference. -The mapping is required even when it is a direct extraction of a designated -drive. The generic watchdog does not implement that application-specific command. +The mapping is required even when it is a direct extraction of the labeled +state range. The generic watchdog does not implement that application-specific command. A comparison file alone need not contain everything an engine requires to restore. Retain verified native archives outside sequencer GC if they are the intended diff --git a/docs/snapshots/lifecycle.md b/docs/snapshots/lifecycle.md index 1cf78d96..903c3cec 100644 --- a/docs/snapshots/lifecycle.md +++ b/docs/snapshots/lifecycle.md @@ -91,7 +91,8 @@ These endpoints are operator-only and require network isolation: opaque `state` artifact. It may describe optimistic state. - `/finalized_state` streams only the canonical comparison file for the latest accepted checkpoint. `/finalized_state/inclusion_block` provides its block and - executed-input count for the watchdog. + executed-input count for the watchdog's idle poll, and + `/finalized_state/digest` adds the file's SHA-256 under the same lease. - `/finalized_snapshot` streams a complete recovery tar archive containing `info.toml`, `state`, and a generated `checkpoint.toml` acceptance receipt. The receipt supplies the accepted inclusion block and next batch nonce. diff --git a/docs/threat-model/README.md b/docs/threat-model/README.md index 3fb1bf00..1641ee5f 100644 --- a/docs/threat-model/README.md +++ b/docs/threat-model/README.md @@ -75,7 +75,7 @@ blocking production diagnostics would require revisiting that assumption entered by restarting after a terminal exit (including supervisors that restart regardless of exit status), and it is bounded by backstops that never depended on a boot gate: - rollbackable soft confirmations, the watchdog byte-compare, and the I15 + rollbackable soft confirmations, the watchdog's digest comparison, and the I15 divergence freeze. - **Adversarial mempool:** reorder, delay, drop, selective inclusion by builders - **Zombie transactions:** a submitted batch may sit in a private mempool indefinitely and land long after we believed it was gone. Two load-bearing defenses: the recovery flusher consumes every wallet-nonce slot this deployment ever used (anchored by the persisted watermark, I14) so zombies cannot claim them; and the content-identity check (I9/I15) compares every *simulated-accepted* landing strictly after baseline block `C` against the valid closed batch we sealed at that nonce. The complete prefix through `C` is opaque and is never reinterpreted using the recovered nonce. A foreign or byte-different landing records divergence when it becomes safe and is ingested, freezes the accepted frontier, and requires cockroach recovery. This is trust-boundary validation of external input (the mempool replaying our own stale transactions at times we don't control), not defense-in-depth against self-bugs or a general canonical-state oracle. In cockroach recovery the watermark does not survive the wipe, so that flush is best-effort by construction; the content-identity check is what keeps the residual zombie detected-and-frozen rather than silent (see [the flush boundary](../recovery/cockroach.md#flush-and-stopping-block)). diff --git a/docs/watchdog/README.md b/docs/watchdog/README.md index 2e34627a..26ce66a2 100644 --- a/docs/watchdog/README.md +++ b/docs/watchdog/README.md @@ -1,386 +1,340 @@ # Watchdog -The watchdog independently replays L1 inputs in the canonical Cartesi Machine -and compares its application-state bytes with the sequencer's accepted -checkpoint at the same L1 block boundary. The wallet's comparison format is SSZ; -the watchdog itself only compares bytes. - -The `/finalized_state` name refers to the sequencer's latest **safe, accepted -batch checkpoint**, not Ethereum's `finalized` tag or a state the watchdog has -already verified. [Snapshot lifecycle](../snapshots/lifecycle.md#acceptance-and-comparison) -owns checkpoint selection and why that application state is comparable at a -whole L1 block boundary. - -## Documentation - -| Doc | Audience | -|-----|----------| -| **[`operator-deployment.md`](operator-deployment.md)** | **Production-like** — Sepolia and mainnet: internal snapshot API, live L1, checkpoints (Sepolia = mainnet dress rehearsal) | -| **[`getting-started.md`](getting-started.md)** | **Local dev only** — Anvil + `sequencer-devnet`, harness smoke, two-terminal flow | -| This file | Architecture, modules, runtime contract, checkpoints, test commands | -| [`staging-drills.md`](staging-drills.md) | Webhook smoke, synthetic alarms, staging compare daemon | -| [`design-notes.md`](design-notes.md) | Detection boundaries and checkpoint crash model | -| [`sepolia.md`](sepolia.md) | Redirect → [`operator-deployment.md`](operator-deployment.md) | - -### Quick start (pick your environment) - -**Sepolia / mainnet (operator):** [`operator-deployment.md`](operator-deployment.md) — shared checklist, internal URL, Sepolia CM image, mainnet notes. - -**Local devnet:** - -One-time setup, then either a single automated check or an interactive run: - -```bash -just setup && just canonical-build-machine-image && just watchdog-lua-deps - -# Path A — full smoke (Anvil + sequencer + CM + compare), one command: -just test-watchdog-compare-harness - -# Path B — two terminals: stack prints CARTESI_WATCHDOG_* exports, then init + tick: -just devnet-for-watchdog # terminal 1 — leave running -# terminal 2: paste exports, then: -export CARTESI_WATCHDOG_LUA_ROOT="$(pwd)" -export CARTESI_WATCHDOG_LUA_BIN=lua5.4 -export CARTESI_WATCHDOG_LUA_DEPS=.deps/lua -./watchdog/sequencer-watchdog init -./watchdog/sequencer-watchdog tick +The watchdog independently re-derives the application's canonical state from +L1 and checks that the sequencer's accepted checkpoint holds the same state. It +runs the canonical Cartesi Machine over the InputBox inputs, extracts the +application state, and compares its SHA-256 with the digest the sequencer +serves for the same L1 block. Its value is independence: the sequencer serves +state from its own native execution, while the watchdog replays the canonical +program with the rollup's own host semantics. + +The sequencer's "finalized" routes serve its latest **safe, accepted batch +checkpoint**, not Ethereum's `finalized` tag. +[Snapshot lifecycle](../snapshots/lifecycle.md#acceptance-and-comparison) owns +checkpoint selection and why that state is comparable at a whole L1 block. + +| Document | Audience | +|---|---| +| This file | How the watchdog works: contract, commands, configuration, state, metrics | +| [`incident-runbook.md`](incident-runbook.md) | What to do when it latches a divergence, and the drills that practice it | +| [`operator-deployment.md`](operator-deployment.md) | Deploying on a live chain (Sepolia, mainnet) | +| [`getting-started.md`](getting-started.md) | Running it locally against a devnet | +| [Cartesi Machine facts](../cartesi-machine.md) | Emulator behavior the watchdog depends on | + +## The tick + +`tick` runs one compare cycle and exits; a scheduler (systemd timer, +Kubernetes CronJob) runs it periodically. There is no daemon and no in-tick +retry: a failed tick's retry is the next scheduled one. + +1. If a divergence is latched, exit 2 without work. +2. The head is the newest checkpoint under `checkpoints/`. +3. Poll `GET /finalized_state/inclusion_block`. Unchanged: exit 0 (idle). + Lower than the head: latch `inclusion_block_regressed`, unless the head is + still the bootstrap machine. A regression is relative to a block the + sequencer agreed with; a sequencer behind a never-agreed bootstrap block has + simply not reached it, and the tick idles. +4. `GET /finalized_state/digest`. Its block B is the replay target, fixed + before any replay, so a long catch-up never chases a moving target. +5. Check that the L1 RPC serves the configured chain and that its `safe` head + has reached B. +6. Replay the InputBox inputs of blocks `(head, B]` on a clone of the head + (below), proving the L1 view complete (below). +7. Extract the application state from the resulting machine and hash it. +8. Equal digests: publish the machine as `checkpoints/-` and + remove the previous head. Different: latch `state_mismatch`. A canonical + machine that stopped for good during replay latches + `canonical_machine_dead`. A divergence is latched and reported before its + evidence is collected ([below](#divergence-latch)). + +### Canonical execution + +The watchdog drives the machine in-process through the `cartesi` Lua module, +with the reference rollup host semantics the `cartesi-machine` CLI and Dave +implement ([details](../cartesi-machine.md#rollup-host-semantics)). It clones +the head into a working directory and runs the machine in place on it; before +each input it clones the working directory as a snapshot: + +- `RX_ACCEPTED`: the snapshot is discarded. +- `RX_REJECTED`: the snapshot replaces the working directory, and its root + hash must equal the input's revert root hash. +- exception, halt, any other manual yield, or cycle overflow: a permanent + fixed point. Nothing after it can ever run; the watchdog latches. + +On copy-on-write filesystems (APFS, btrfs, XFS with reflink) a clone costs +metadata only; elsewhere it is a full sparse copy of the machine, which makes +multi-GiB states slow. Conformance tests run the same inputs through the +watchdog and the reference CLI and require equal root hashes. + +### L1 completeness + +A provider that silently drops logs must fail the tick, not produce a false +comparison. The reader requires InputBox indices to run contiguously from the +head's input count, and checks the count it reached against +`InputBox.getNumberOfInputs(app)` pinned at block B. Every input's payload must +name the configured application and chain. Long log ranges are split on the +provider error codes that mean "range too large", exactly like the Rust reader +(shared vector: `tests/fixtures/l1_partition_vector.json`). + +**Scan floor.** The Rust reader starts at the application's deployment block, +which is sound only because it also witnesses that the InputBox rejects inputs +for undeployed applications. The watchdog starts at its bootstrap block and +proves completeness from the InputBox count instead; do not copy the +deployment-block floor without the witness. + +The reader holds one provider response at a time: each successful log range +is decoded and fed to the machine before the next is fetched. + +## State sources + +`init` persists how the application state is extracted +(`CARTESI_WATCHDOG_STATE_SOURCE`): + +- **`range: