diff --git a/packages/vantio-install/artifacts/IMPLEMENTATION-REPORT.json b/packages/vantio-install/artifacts/IMPLEMENTATION-REPORT.json index db3e28ec..efab084e 100644 --- a/packages/vantio-install/artifacts/IMPLEMENTATION-REPORT.json +++ b/packages/vantio-install/artifacts/IMPLEMENTATION-REPORT.json @@ -91,9 +91,9 @@ "optics_cli_sha256": "82fe13ad6fc916ac67a670bd95fbf18b24389ecb383d81246e1d52cb96712a1f", "agent_sdk_npm": "0.2.4", "agent_sdk_py": "3.1.0", - "pe_source_commit": "fab81efc08110506ff90847495197e7051a253b5", - "pe_archive_sha256": "72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128", - "pe_manifest_digest": "sha256:4d932b93bf4c20983142d5f9bff1ea060d9407a19a5e8c9f59db29f7a4122553", + "pe_source_commit": "06696d5020700693b0154c59d0e072a24f648378", + "pe_archive_sha256": "e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e", + "pe_manifest_digest": "sha256:8b40aec5c125043ec4278a14170677474c9ca31a7ae78e8496c40dffa69d0e19", "cli_package_tree_mutated": false }, "tests": { diff --git a/packages/vantio-install/docs/ARTIFACT-PATH.md b/packages/vantio-install/docs/ARTIFACT-PATH.md index 5b630e5b..3be600d0 100644 --- a/packages/vantio-install/docs/ARTIFACT-PATH.md +++ b/packages/vantio-install/docs/ARTIFACT-PATH.md @@ -37,23 +37,25 @@ Optional Python sdist (only if included): - Path: `artifacts/optics/vantio_agent_sdk-3.1.0.tar.gz` - SHA-256: `9f991291d5e44a23e17a9b0d7db24f6e7048d4c76cf0a9c37e35ccbcfe999c4f` -## Sealed Phantom Engine archive (customer staging name) +## Sealed Phantom Engine archive -Use a **customer staging** filename that does **not** embed internal lab aliases: +Place the sealed tar under this basename. Apply opens that path and checks this SHA-256. -- Path: `artifacts/phantom-engine/vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar` -- SHA-256: `72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128` -- Source commit: `fab81efc08110506ff90847495197e7051a253b5` -- Manifest digest: `sha256:4d932b93bf4c20983142d5f9bff1ea060d9407a19a5e8c9f59db29f7a4122553` +- Path: `artifacts/phantom-engine/vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar` +- SHA-256: `e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e` +- Source commit: `06696d5020700693b0154c59d0e072a24f648378` +- Manifest digest: `sha256:8b40aec5c125043ec4278a14170677474c9ca31a7ae78e8496c40dffa69d0e19` -**Naming rule (GAP-CB-DEP-002):** the customer-visible archive basename is locked to the staging name above. Customer docs and checksums use that basename only. Do not substitute an internal lab alias, a shorter untagged alias, or any absolute path. +That manifest digest is the digest inside the sealed tar. On Ubuntu 24.04, apply writes a temporary load archive when a layer is gzip and the manifest calls it an uncompressed tar, then `docker load` records the corrected manifest. The sealed file hash stays `e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e`. + +**Naming rule:** customer docs and checksums use the basename above. Do not substitute a shorter untagged alias or any absolute path. `artifacts/phantom-engine/PHANTOM-ARTIFACT-MANIFEST.json` must record the same commit, archive hash, and manifest digest. `SHA256SUMS` lists every file in the bundle. A mismatch stops `plan`. ### Example SHA256SUMS line ``` -72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128 artifacts/phantom-engine/vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar +e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e artifacts/phantom-engine/vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar ``` ## GHCR / registries @@ -78,7 +80,7 @@ Rollback, uninstall, status, and verify-removal semantics remain as documented i | Field | Value | | --- | --- | -| pe_source_commit | `fab81efc08110506ff90847495197e7051a253b5` | -| pe_archive_sha256 | `72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128` | +| pe_source_commit | `06696d5020700693b0154c59d0e072a24f648378` | +| pe_archive_sha256 | `e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e` | | published | `false` | | claim_ceiling | `INTERNAL_CLEAN_HOST_PROOF` | diff --git a/packages/vantio-install/docs/INSTALL.md b/packages/vantio-install/docs/INSTALL.md index 99053bd3..d1a2c78f 100644 --- a/packages/vantio-install/docs/INSTALL.md +++ b/packages/vantio-install/docs/INSTALL.md @@ -22,4 +22,8 @@ A customer apply on the host that will keep the node uses both gates and the pla `` is the SHA-256 of that `PLAN.json` file. The installer recomputes the hash, checks the transaction id, and checks the artifact bytes again before each change. A mismatch stops the command. The sealed Phantom Engine tip, archive hash, and manifest digest stay the ones recorded for this package. -Apply is finished only when `state` is `HEALTHY` or `DEGRADED` and `HEALTH.json` records that result. `APPLIED` means the steps ran and health is not confirmed yet. A process exit of 0 from Docker, npm, or pip is not that result. After the Optics npm install, the host check requires `/bin/vantio` and the pinned CLI version (`vantio --version`, or the installed package manifest when the binary does not print a version). After the Agent SDK npm install, the host check requires `/lib/node_modules/@vantio/agent-sdk` and the pinned version in that package manifest. After the Agent SDK pip install, the host check requires the `vantio` module under the prefix and the pinned version. Debian and Ubuntu pip write that module at `local/lib/python3.X/dist-packages`. An upstream prefix layout writes it at `lib/python3.X/site-packages`. When the module does not declare a version, the check reads the matching dist-info metadata. `install_agent_sdks` is checkpointed only after both SDK checks pass. Before the observe-only Phantom Engine container starts, the installer loads AppArmor profile `vantio-pe-observe` and passes `--security-opt apparmor=vantio-pe-observe`. The same start bind-mounts host `/sys/fs/bpf` and host `/sys/kernel/tracing`. Preflight records the tracing mount as PF-TRACEFS and blocks when that directory is missing or empty. The host check for that container requires the process to be running under that profile. +Apply loads the sealed Phantom Engine OCI tar with `docker load`. On Ubuntu 24.04 the default Docker image store reads the layer media type and unpacks from that. When the manifest says a layer is an uncompressed tar and the bytes are gzip, that unpack stops with `archive/tar: invalid tar header`, and `docker load` can still exit 0. Apply writes a temporary archive in the stage directory with the media type set to match the bytes, then loads that file. The sealed file you checked stays the file you hashed. Docker keeps the storage driver Ubuntu installed. Preflight records this as `PF-OCI-LOAD`. A `docker load` that prints an unpack error fails the install, even when docker exits 0. + +Apply is finished only when `state` is `HEALTHY` or `DEGRADED` and `HEALTH.json` records that result. `APPLIED` means the steps ran and health is not confirmed yet. A process exit of 0 from Docker, npm, or pip is not that result. After the Optics npm install, the host check requires `/bin/vantio` and the pinned CLI version (`vantio --version`, or the installed package manifest when the binary does not print a version). After the Agent SDK npm install, the host check requires `/lib/node_modules/@vantio/agent-sdk` and the pinned version in that package manifest. After the Agent SDK pip install, the host check requires the `vantio` module under the prefix and the pinned version. Debian and Ubuntu pip write that module at `local/lib/python3.X/dist-packages`. An upstream prefix layout writes it at `lib/python3.X/site-packages`. When the module does not declare a version, the check reads the matching dist-info metadata. `install_agent_sdks` is checkpointed only after both SDK checks pass. Before the observe-only Phantom Engine container starts, the installer loads AppArmor profile `vantio-pe-observe` and passes `--security-opt apparmor=vantio-pe-observe`. The same start bind-mounts host `/sys/fs/bpf` and host `/sys/kernel/tracing`. Preflight records the tracing mount as PF-TRACEFS and blocks when that directory is missing or empty. The host check for that container reads the container after `docker run -d` returns. That command exits 0 when the daemon accepts the container, including when the process has already stopped. A stopped container is not verified. A detached container that is still starting is read again until the loader process is visible, the known bpffs pins are present, and clsact is on the interface, or until that wait ends. A detached container that reaches that state is verified. A container that stays up without those facts is not verified. + +On Ubuntu 24.04 the `nodejs` package does not include npm. When npm is already on `PATH`, `ensure_node` runs `npm --version`. When npm is missing and the installer is root, `ensure_node` installs the Ubuntu `npm` package before the Optics CLI install. When npm is missing and the installer is not root, `plan` stops and `PLAN.json` names that package. diff --git a/packages/vantio-install/docs/LIMITATIONS.md b/packages/vantio-install/docs/LIMITATIONS.md index da781c48..9a340ef2 100644 --- a/packages/vantio-install/docs/LIMITATIONS.md +++ b/packages/vantio-install/docs/LIMITATIONS.md @@ -20,7 +20,7 @@ The sealed Phantom Engine archive, manifest digest, and Optics package versions Phantom Engine media are sealed-byte placement only, as `ARTIFACT-PATH.md` describes. They are not fetched, and they are not published. `published` stays false. GHCR is not a Phantom Engine source. The claim ceiling stays `INTERNAL_CLEAN_HOST_PROOF`. -Live changes on a host run only when you set `VANTIO_INSTALL_ALLOW_LIVE=1` and pass `--i-accept-live-mutations` on `apply`, `rollback`, or `uninstall`. Either one alone stops before any host change and the command returns `FAILED_SAFE`. With both set, the command still stops unless the plan hash matches, the sealed artifacts match, the host is x86_64 with kernel BTF, the mode is observe-only, and rollback and residual checks are in the plan. The command runs an allowlisted argv list. It does not use a shell string. Exit 0 from one of those commands is not enough; the installer reads the host again before it records the step as verified. +Live changes on a host run only when you set `VANTIO_INSTALL_ALLOW_LIVE=1` and pass `--i-accept-live-mutations` on `apply`, `rollback`, or `uninstall`. Either one alone stops before any host change and the command returns `FAILED_SAFE`. With both set, the command still stops unless the plan hash matches, the sealed artifacts match, the host is x86_64 with kernel BTF, the mode is observe-only, and rollback and residual checks are in the plan. The command runs an allowlisted argv list. It does not use a shell string. Exit 0 from one of those commands is not enough; the installer reads the host again before it records the step as verified. For the observe container, that read distinguishes a stopped process from a detached container that is still starting. The check waits until the loader is running with the known bpffs pins and clsact on the interface, or until the wait ends. A stopped process is not verified. On Ubuntu 24.04, `ensure_node` installs the Ubuntu `npm` package when Node.js 18 or newer is present, npm is missing, and the process is root. Otherwise a missing npm binary stops the plan and the plan names that package. The allowlist adds `apt-get` only for `apt-get install -y --no-install-recommends npm`. The live privilege check accepts effective uid 0 only. `privilege_mode` `sudo` with `sudo` on `PATH`, and `privilege_mode` `docker_group` with write access to `/var/run/docker.sock`, are recorded facts and are not a live grant. `privilege_mode` `UNKNOWN` blocks `PF-DOCKER-PERM`. When the effective uid is not 0, the live command returns `FAILED_SAFE` and the message `Live mutations need effective root. sudo on PATH is not privilege.` `PREFLIGHT.md` records the symptoms. `sudo` is not an allowlisted executable. Raw `docker` and raw `sudo docker` outside the installer argv list are forbidden. The installer does not insert `sudo` in front of `docker`. A live residual check reports `RESIDUAL_FOUND` when something from that scope is still on the host. From that state, the same dual-gated rollback, or uninstall with `--scope optics` or `--scope all`, removes a leftover Optics CLI or Agent SDK tree under the prefix. Rollback, or uninstall with `--scope pe` or `--scope all`, also unlinks the known bpffs pin names when they are still present. Apply stays refused until `verify-removal` reports an empty residual list. diff --git a/packages/vantio-install/docs/NETWORK-TRANSFER-ALLOWLIST.md b/packages/vantio-install/docs/NETWORK-TRANSFER-ALLOWLIST.md index 4de8356a..c146c903 100644 --- a/packages/vantio-install/docs/NETWORK-TRANSFER-ALLOWLIST.md +++ b/packages/vantio-install/docs/NETWORK-TRANSFER-ALLOWLIST.md @@ -8,19 +8,19 @@ Recorded status: `published` is false. The claim ceiling is `INTERNAL_CLEAN_HOST ## Origin -Origin is the sealed archive already on the operator workstation, in a directory you choose on that workstation. The source commit recorded for that archive is `fab81efc08110506ff90847495197e7051a253b5`. That commit is an identity pin. This page does not tell you to obtain the bytes from a repository checkout or from a container registry. +Origin is the sealed archive already on the operator workstation, in a directory you choose on that workstation. The source commit recorded for that archive is `06696d5020700693b0154c59d0e072a24f648378`. That commit is an identity pin. This page does not tell you to obtain the bytes from a repository checkout or from a container registry. This repository does not contain the archive. If the file is not already on the workstation, stop. Do not upload the archive to create a copy, and do not publish a download URL for it. ## Identity -Use this basename exactly. Do not shorten it, and do not drop `customer-staging` from the name. Do not substitute another file and keep this name. +Use this basename exactly. Do not shorten it. Do not substitute another file and keep this name. -- Basename: `vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar` -- Relative path after placement: `artifacts/phantom-engine/vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar` -- SHA-256: `72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128` -- Source commit: `fab81efc08110506ff90847495197e7051a253b5` -- Manifest digest: `sha256:4d932b93bf4c20983142d5f9bff1ea060d9407a19a5e8c9f59db29f7a4122553` +- Basename: `vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar` +- Relative path after placement: `artifacts/phantom-engine/vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar` +- SHA-256: `e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e` +- Source commit: `06696d5020700693b0154c59d0e072a24f648378` +- Manifest digest: `sha256:8b40aec5c125043ec4278a14170677474c9ca31a7ae78e8496c40dffa69d0e19` - `published` is false This procedure does not change frozen versions. Optics CLI stays 0.3.24. Agent SDK npm stays 0.2.4. Agent SDK Python stays 3.1.0. The source commit, SHA-256, and manifest digest stay the values above. @@ -28,13 +28,13 @@ This procedure does not change frozen versions. Optics CLI stays 0.3.24. Agent S The staging checksum line is the digest, two spaces, then the basename: ```text -72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128 vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar +e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar ``` The bundle checksum line is the same digest, two spaces, then the relative path: ```text -72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128 artifacts/phantom-engine/vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar +e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e artifacts/phantom-engine/vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar ``` ## Hash verify @@ -42,11 +42,11 @@ The bundle checksum line is the same digest, two spaces, then the relative path: On the workstation, in the directory that already holds the archive and `SHA256SUMS`, verify before you open a network allow. Both commands must exit 0. The first command checks the locked digest from this page. The second checks the staging `SHA256SUMS`. `sha256sum` is the last command in the pipeline, so a mismatch still exits non-zero. ```bash -echo "72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128 vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar" | sha256sum -c - +echo "e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar" | sha256sum -c - sha256sum -c SHA256SUMS ``` -Each output must show `vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar: OK`. The staging `SHA256SUMS` must contain the staging line in the identity section. A checksum file that matches the bytes but not the locked digest is a hash mismatch. +Each output must show `vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar: OK`. The staging `SHA256SUMS` must contain the staging line in the identity section. A checksum file that matches the bytes but not the locked digest is a hash mismatch. If the archive is missing, stop. If `SHA256SUMS` is missing, stop. If the digest does not match, that is a hash mismatch. Stop. Do not open the allow, and do not copy the file. @@ -62,7 +62,7 @@ The same boundary works on a firewall that is not AWS. Allow TCP port 22, or the ## Destination -The destination is the bundle directory already on the customer host. This page calls that directory `./bundle/` relative to the login directory you use for the copy. The archive lands at `artifacts/phantom-engine/vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar` under that bundle. The bundle `SHA256SUMS` stays at the bundle root. +The destination is the bundle directory already on the customer host. This page calls that directory `./bundle/` relative to the login directory you use for the copy. The archive lands at `artifacts/phantom-engine/vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar` under that bundle. The bundle `SHA256SUMS` stays at the bundle root. Do not replace the bundle `SHA256SUMS` with the staging checksum file. The staging file names the basename beside the archive. The bundle file names the relative path. Both lines carry the same digest. If the bundle file is missing, stop. If the bundle line is missing, or the digest on that line differs, stop. Do not write a new digest to force a match, and do not create `MANIFEST.json` from this page. @@ -91,21 +91,21 @@ aws ec2 authorize-security-group-ingress \ # 2. COPY ssh -i SSH_IDENTITY ubuntu@CUSTOMER_HOST 'mkdir -p ./vantio-sealed-media' scp -i SSH_IDENTITY \ - ./vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar \ + ./vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar \ ./SHA256SUMS \ ubuntu@CUSTOMER_HOST:./vantio-sealed-media/ # 3. VERIFY ssh -i SSH_IDENTITY ubuntu@CUSTOMER_HOST \ 'cd ./vantio-sealed-media && \ - echo "72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128 vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar" | sha256sum -c - > ./transfer-verify.txt && \ + echo "e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar" | sha256sum -c - > ./transfer-verify.txt && \ sha256sum -c SHA256SUMS >> ./transfer-verify.txt' # 4. PLACE ssh -i SSH_IDENTITY ubuntu@CUSTOMER_HOST \ - 'grep -F "72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128 artifacts/phantom-engine/vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar" ./bundle/SHA256SUMS && \ + 'grep -F "e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e artifacts/phantom-engine/vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar" ./bundle/SHA256SUMS && \ mkdir -p ./bundle/artifacts/phantom-engine && \ - mv ./vantio-sealed-media/vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar \ + mv ./vantio-sealed-media/vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar \ ./bundle/artifacts/phantom-engine/' # 5. REVOKE @@ -114,7 +114,7 @@ aws ec2 revoke-security-group-ingress \ --protocol tcp --port 22 --cidr OPERATOR_PUBLIC_IP/32 ``` -After verify exits 0, read `./vantio-sealed-media/transfer-verify.txt` on the customer host. It must contain `vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar: OK` for the locked digest and for `SHA256SUMS`. If verify exits non-zero, do not place the archive. Keep the transcript if it was written. Revoke the temporary allow. The checksum command stays at the end of its pipeline, and the shell uses `&&`, so a mismatch does not continue. Do not pipe `sha256sum -c` into `tee` or any later command that would hide a non-zero exit. +After verify exits 0, read `./vantio-sealed-media/transfer-verify.txt` on the customer host. It must contain `vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar: OK` for the locked digest and for `SHA256SUMS`. If verify exits non-zero, do not place the archive. Keep the transcript if it was written. Revoke the temporary allow. The checksum command stays at the end of its pipeline, and the shell uses `&&`, so a mismatch does not continue. Do not pipe `sha256sum -c` into `tee` or any later command that would hide a non-zero exit. Place checks the bundle checksum line before it moves the file. If that grep exits non-zero, the archive stays in `./vantio-sealed-media/` and you do not treat the handoff as complete. Revoke the temporary allow. @@ -124,7 +124,7 @@ Each case below stops the handoff. You do not place the archive into `artifacts/ - The archive is missing. Stop. Do not copy an empty path. - `SHA256SUMS` is missing on the workstation or on the customer host. Stop. -- The digest does not match `72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128`. That is a hash mismatch. Stop. A `SHA256SUMS` file that matches some other bytes, and not the locked digest, stops the handoff the same way. +- The digest does not match `e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e`. That is a hash mismatch. Stop. A `SHA256SUMS` file that matches some other bytes, and not the locked digest, stops the handoff the same way. - A different source commit, a different SHA-256, or a different manifest digest is the wrong version. Optics CLI 0.3.24, Agent SDK npm 0.2.4, and Agent SDK Python 3.1.0 stay as they are. Stop. Do not rename the other file to this basename. - An archive whose name is not linux-amd64, or a host that is not x86_64, is the wrong architecture. Stop. Do not rename the file so the name says linux-amd64. - The copy is incomplete. The destination file is shorter than the sealed original, or `sha256sum -c` did not exit 0. Stop. Do not place an incomplete file. @@ -147,11 +147,11 @@ Do not edit the archive, splice it, or pad it. Do not change the digest so the c On the customer host, verify writes `./vantio-sealed-media/transfer-verify.txt`. That file is the checksum transcript. Keep it. Also write `transfer-evidence.txt` next to it with these lines: ```text -basename: vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar -relative: artifacts/phantom-engine/vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar -sha256: 72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128 -source_commit: fab81efc08110506ff90847495197e7051a253b5 -manifest_digest: sha256:4d932b93bf4c20983142d5f9bff1ea060d9407a19a5e8c9f59db29f7a4122553 +basename: vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar +relative: artifacts/phantom-engine/vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar +sha256: e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e +source_commit: 06696d5020700693b0154c59d0e072a24f648378 +manifest_digest: sha256:8b40aec5c125043ec4278a14170677474c9ca31a7ae78e8496c40dffa69d0e19 published: false claim_ceiling: INTERNAL_CLEAN_HOST_PROOF gap: GAP-CB-DEP-013 diff --git a/packages/vantio-install/docs/PREFLIGHT.md b/packages/vantio-install/docs/PREFLIGHT.md index 6c156bd5..aa1aea7b 100644 --- a/packages/vantio-install/docs/PREFLIGHT.md +++ b/packages/vantio-install/docs/PREFLIGHT.md @@ -20,8 +20,12 @@ Root is effective uid 0. The probe does not store the string `root` in `privileg A live command returns `FAILED_SAFE` with the message `Live mutations need effective root. sudo on PATH is not privilege.` when the effective uid is not 0. A passing `sudo` or `docker_group` fact does not change that. -The installer is the only program on this path that runs Docker. It uses an argv list and `shell` is false. A shell string is refused. The executables it may run are `mkdir`, `npm`, `python3`, `docker`, `tc`, and `apparmor_parser`. `sudo`, `su`, and a shell are refused as the executable. An argument that contains a shell metacharacter is refused. An argv list that differs from the catalog entry for that step is refused. +The installer is the only program on this path that runs Docker. It uses an argv list and `shell` is false. A shell string is refused. The executables it may run are `mkdir`, `npm`, `python3`, `docker`, `tc`, `apparmor_parser`, and `apt-get`. `apt-get` is allowlisted only as `apt-get install -y --no-install-recommends npm`. `sudo`, `su`, and a shell are refused as the executable. An argument that contains a shell metacharacter is refused. An argv list that differs from the catalog entry for that step is refused. + +`PF-NODE` records whether Node.js 18 or newer is on the host. `PF-NPM` records whether `npm` is on `PATH`. On Ubuntu 24.04 the `nodejs` package does not include the npm binary. When Node.js 18 or newer is present, npm is missing, and the installer is root, the plan stays ready and `ensure_node` installs the Ubuntu `npm` package. The plan shows that package and the exact argv. The installer does not run `apt-get update`. When npm is missing and the installer is not root, `PF-NPM` is `BLOCKED`, the plan state stays off `PLANNED`, and `PLAN.json` still shows the prerequisite `Install the Ubuntu npm package. The nodejs package does not include the npm binary.` Rollback does not remove that package if a later step fails after it is installed. Raw `docker`, raw `sudo docker`, and a direct call on `docker.sock` are forbidden for customer operators. So is changing the socket mode by hand. When the effective uid is not 0, rerun `vantio-install` under `sudo` so the process is root. A principal that can already write the socket is still not a live grant until that process is root. The allowlist does not insert `sudo` in front of `docker`. A host check can still fail when `docker` runs as a user who cannot open the socket. +`PF-OCI-LOAD` reads the sealed Phantom Engine OCI tar. It passes when the archive is not an OCI layout, when every layer's media type already matches its bytes, or when a mismatch can be corrected at apply. The correction writes a temporary load archive. The sealed file stays the file you hashed. Docker keeps the storage driver Ubuntu installed. The check blocks when the file is an OCI layout the installer cannot correct, for example a missing layer blob. Restore the sealed archive in that case. + `proof_state` stays `NOT_PROVED`. The proof ceiling stays `INTERNAL_CLEAN_HOST_PROOF`. diff --git a/packages/vantio-install/docs/STAGE-B-ARTIFACT-PATH.md b/packages/vantio-install/docs/STAGE-B-ARTIFACT-PATH.md index e99c4bfe..ca20f3c2 100644 --- a/packages/vantio-install/docs/STAGE-B-ARTIFACT-PATH.md +++ b/packages/vantio-install/docs/STAGE-B-ARTIFACT-PATH.md @@ -1,6 +1,6 @@ # Stage B artifact path -Customer operators use `ARTIFACT-PATH.md` for sealed-byte placement. This note uses the same customer-staging archive basename and the same frozen digests. +Customer operators use `ARTIFACT-PATH.md` for sealed-byte placement. This note uses the same archive basename and the same frozen digests. This note is for the operator who will place sealed bytes on a host before a later live run. The second-lab gate is closed. `proof_state` stays `NOT_PROVED`. This file does not publish a download URL, a registry token, or a public image name. @@ -28,10 +28,10 @@ Optional Python sdist, only if you include it: Sealed Phantom Engine archive: -- Path: `artifacts/phantom-engine/vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar` -- SHA-256: `72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128` -- Source commit: `fab81efc08110506ff90847495197e7051a253b5` -- Manifest digest: `sha256:4d932b93bf4c20983142d5f9bff1ea060d9407a19a5e8c9f59db29f7a4122553` +- Path: `artifacts/phantom-engine/vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar` +- SHA-256: `e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e` +- Source commit: `06696d5020700693b0154c59d0e072a24f648378` +- Manifest digest: `sha256:8b40aec5c125043ec4278a14170677474c9ca31a7ae78e8496c40dffa69d0e19` `artifacts/phantom-engine/PHANTOM-ARTIFACT-MANIFEST.json` records that commit, that archive hash, and that manifest digest. `SHA256SUMS` lists every file in the bundle. A mismatch stops `plan`. diff --git a/packages/vantio-install/tests/test_live_executor.py b/packages/vantio-install/tests/test_live_executor.py index defd4930..77ba949b 100644 --- a/packages/vantio-install/tests/test_live_executor.py +++ b/packages/vantio-install/tests/test_live_executor.py @@ -44,7 +44,7 @@ ) from vantio_install.optics_cli import observed_optics_cli_version # noqa: E402 from vantio_install.util import sha256_file # noqa: E402 -from vantio_install.commands import observe_apparmor_opt # noqa: E402 +from vantio_install.commands import apt_install_npm_argv, observe_apparmor_opt # noqa: E402 from vantio_install.pe_apparmor import pe_apparmor_profile_path, profile_text # noqa: E402 ENV = "VANTIO_INSTALL_ALLOW_LIVE" @@ -58,6 +58,7 @@ def __init__(self) -> None: self.running = False self.cmd: list[str] = [] self.npm = False + self.npm_cli = False self.pip = False self.clsact: list[str] = [] self.verified: list[str] = [] @@ -84,6 +85,8 @@ def runner(self, argv: list[str], timeout: int) -> ExecResult: self.cmd = list(argv) elif argv[:2] == ["npm", "install"]: self.npm = True + elif argv == ["npm", "--version"] or (argv[:2] == ["apt-get", "install"] and argv[-1] == "npm"): + self.npm_cli = True elif argv[:3] == ["python3", "-m", "pip"]: self.pip = True elif argv[:3] == ["tc", "qdisc", "replace"]: @@ -104,6 +107,8 @@ def verify(self, op_type: str, grant) -> str: return "VERIFIED" if grant.stage.is_dir() else "NOT_VERIFIED" if op_type == "mkdir_evidence": return "VERIFIED" if grant.evidence.is_dir() else "NOT_VERIFIED" + if op_type == "ensure_npm": + return "VERIFIED" if self.npm_cli else "NOT_VERIFIED" if op_type == "install_optics_cli": return "VERIFIED" if self.npm else "NOT_VERIFIED" if op_type == "install_agent_sdk_npm": @@ -986,6 +991,34 @@ def observed_delta(self, op_type, grant): self.assertEqual(caught.exception.failure_class, "ROLLBACK_REQUIRED") self.assertNotEqual(caught.exception.state, "HEALTHY") + def test_docker_load_exit_zero_with_unpack_error_is_not_success(self) -> None: + harness = self.planned() + self.set_env("1") + grant = self.grant_for(harness) + self.arm(harness) + + def runner(argv, timeout): + return ExecResult( + 0, + False, + "Loaded image: vantio-phantom-engine:example\n" + "Error unpacking image: archive/tar: invalid tar header\n", + "", + ) + + class Observer: + def verify(self, op_type, grant): + raise AssertionError("an unpack error must not be verified") + + def observed_delta(self, op_type, grant): + return {} + + with self.assertRaises(InstallError) as caught: + dispatch(grant, "docker_load", catalog_argv("docker_load", grant), runner=runner, observer=Observer()) + self.assertEqual(caught.exception.failure_class, "ROLLBACK_REQUIRED") + self.assertIn("unpack error", str(caught.exception)) + self.assertIn("storage driver", str(caught.exception)) + def test_live_timeout_is_interrupted(self) -> None: harness = self.planned() self.set_env("1") @@ -1670,6 +1703,7 @@ def _stage_grant(self, stage: Path) -> LiveGrant: sdk_wheel=root / "sdk.whl", container_name="vantio-pe-test", observe_config=root / "observe-config.json", + npm_action="present", ) def test_remove_stage_refuses_symlink_and_does_not_follow_it(self) -> None: @@ -1733,6 +1767,92 @@ def test_fixture_remove_stage_refuses_symlink(self) -> None: self.assertEqual(secret.read_text(encoding="utf-8"), "keep\n") self.assertTrue(stage.is_symlink()) + def test_detached_observe_waits_until_the_loader_is_healthy(self) -> None: + harness = self.planned() + self.set_env("1") + grant = self.grant_for(harness) + pins = list(constants.BPF_PINS) + samples = [ + { + "lifecycle": "detached", + "security_ok": True, + "pins": pins[:1], + "pins_error": False, + "loader": False, + "clsact": True, + }, + { + "lifecycle": "detached", + "security_ok": True, + "pins": pins, + "pins_error": False, + "loader": True, + "clsact": True, + }, + ] + cursor = {"index": 0} + + def sampler(_grant: object) -> dict: + item = samples[min(cursor["index"], len(samples) - 1)] + cursor["index"] += 1 + return item + + times = iter((0.0, 0.5, 1.0)) + observer = ProductionObserver( + observe_sampler=sampler, + clock=lambda: next(times), + sleeper=lambda _seconds: None, + ) + self.assertEqual(observer.verify("start_pe_observe", grant), "VERIFIED") + self.assertGreaterEqual(cursor["index"], 2) + + def test_stopped_observe_is_not_healthy_while_pins_remain(self) -> None: + harness = self.planned() + self.set_env("1") + grant = self.grant_for(harness) + seen = {"count": 0} + + def sampler(_grant: object) -> dict: + seen["count"] += 1 + return { + "lifecycle": "stopped", + "security_ok": False, + "pins": list(constants.BPF_PINS), + "pins_error": False, + "loader": True, + "clsact": True, + } + + observer = ProductionObserver( + observe_sampler=sampler, + clock=lambda: 0.0, + sleeper=lambda _seconds: None, + observe_wait_s=20, + ) + self.assertEqual(observer.verify("start_pe_observe", grant), "NOT_VERIFIED") + self.assertEqual(seen["count"], 1) + + def test_ubuntu_npm_remediation_installs_only_the_npm_package(self) -> None: + harness = self.planned() + self.set_env("1") + self.arm(harness) + host = json.loads(harness.tx_file("HOST-SNAPSHOT.json").read_text(encoding="utf-8")) + host["npm_version"] = None + host["effective_uid"] = 0 + host["node_version"] = "v18.19.1" + grant = self.grant_for(harness, host=host) + self.assertEqual(grant.npm_action, "remediate") + argv = catalog_argv("ensure_npm", grant) + self.assertEqual(argv, apt_install_npm_argv()) + lab = Lab() + delta = dispatch(grant, "ensure_npm", argv, runner=lab.runner, observer=lab) + self.assertEqual(lab.calls, [argv]) + self.assertEqual(delta, {}) + with self.assertRaises(InstallError): + reject_argv(["apt-get", "update"]) + with self.assertRaises(InstallError): + reject_argv(["apt-get", "install", "-y", "--no-install-recommends", "curl"]) + if __name__ == "__main__": unittest.main() diff --git a/packages/vantio-install/tests/test_network_transfer_docs.py b/packages/vantio-install/tests/test_network_transfer_docs.py index 97aaed40..fe968819 100644 --- a/packages/vantio-install/tests/test_network_transfer_docs.py +++ b/packages/vantio-install/tests/test_network_transfer_docs.py @@ -9,12 +9,12 @@ PACKAGE = Path(__file__).resolve().parents[1] DOC_PATH = PACKAGE / "docs" / "NETWORK-TRANSFER-ALLOWLIST.md" -BASENAME = "vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar" +BASENAME = "vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar" RELATIVE = "artifacts/phantom-engine/" + BASENAME -SHA256 = "72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128" -SOURCE_COMMIT = "fab81efc08110506ff90847495197e7051a253b5" -MANIFEST_DIGEST = "sha256:4d932b93bf4c20983142d5f9bff1ea060d9407a19a5e8c9f59db29f7a4122553" -SUPERSEDED_BASENAME = "vantio-phantom-engine-fab81efc0811-linux-amd64.oci.tar" +SHA256 = "e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e" +SOURCE_COMMIT = "06696d5020700693b0154c59d0e072a24f648378" +MANIFEST_DIGEST = "sha256:8b40aec5c125043ec4278a14170677474c9ca31a7ae78e8496c40dffa69d0e19" +SUPERSEDED_BASENAME = "vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar" _ABSOLUTE_PATH = re.compile( r"(?:^|[\s`'\"(])/(?:home|opt|var|usr|Users|root|tmp|mnt|srv|etc)(?:/|\b)" diff --git a/packages/vantio-install/tests/test_npm_preflight.py b/packages/vantio-install/tests/test_npm_preflight.py new file mode 100644 index 00000000..62a3fbf7 --- /dev/null +++ b/packages/vantio-install/tests/test_npm_preflight.py @@ -0,0 +1,86 @@ +"""Ubuntu nodejs without npm is a plan fact, not an apply crash.""" + +from __future__ import annotations + +import json +import sys +import unittest +from pathlib import Path + +PACKAGE = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PACKAGE)) + +from tests.test_stage_a import Harness # noqa: E402 +from vantio_install import constants # noqa: E402 +from vantio_install.commands import apt_install_npm_argv # noqa: E402 +from vantio_install.preflight import NPM_PREREQUISITE # noqa: E402 + +SEAL = "e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e" + + +class NpmPreflightTests(unittest.TestCase): + def make(self) -> Harness: + harness = Harness() + self.addCleanup(harness.close) + return harness + + def test_pins_stay_on_the_run3_seal(self) -> None: + pins = constants.FROZEN_PINS + self.assertEqual(pins["optics_cli_version"], "0.3.24") + self.assertEqual(pins["agent_sdk_py_version"], "3.1.0") + self.assertEqual(pins["pe_archive_sha256"], SEAL) + self.assertIn("PF-OCI-LOAD", constants.PREFLIGHT_ORDER) + self.assertIn("PF-NPM", constants.PREFLIGHT_ORDER) + + def test_present_npm_stays_a_version_check(self) -> None: + harness = self.make() + code, body = harness.run("plan") + self.assertEqual(code, 0, body) + step = next(row for row in body["planned_steps"] if row["id"] == "ensure_node") + self.assertEqual(step["npm_action"], "present") + self.assertFalse(step["mutation"]) + self.assertEqual(step["argv"], ["npm", "--version"]) + self.assertEqual(body["prerequisites"], []) + plan = json.loads(harness.tx_file("PLAN.json").read_text(encoding="utf-8")) + ops = plan["live_operations"] + self.assertLess(ops.index("ensure_npm"), ops.index("install_optics_cli")) + + def test_missing_npm_on_ubuntu_root_is_written_into_the_plan(self) -> None: + harness = self.make() + harness.host["npm_version"] = None + harness.host["node_version"] = "v18.19.1" + harness.host["effective_uid"] = 0 + code, body = harness.run("plan") + self.assertEqual(code, 0, body) + self.assertEqual(body["state"], "PLANNED") + self.assertEqual(body["prerequisites"][0]["text"], NPM_PREREQUISITE) + self.assertEqual(body["prerequisites"][0]["argv"], apt_install_npm_argv()) + step = next(row for row in body["planned_steps"] if row["id"] == "ensure_node") + self.assertTrue(step["mutation"]) + self.assertEqual(step["npm_action"], "remediate") + self.assertEqual(step["prerequisite"], NPM_PREREQUISITE) + self.assertEqual(step["argv"], apt_install_npm_argv()) + preflight = json.loads(harness.tx_file("PREFLIGHT.json").read_text(encoding="utf-8")) + check = next(row for row in preflight["checks"] if row["id"] == "PF-NPM") + self.assertEqual(check["result"], "PASS") + self.assertEqual(check["remediation"], NPM_PREREQUISITE) + self.assertNotIn("PF-NPM", body["failed_or_limiting_checks"]) + + def test_missing_npm_without_root_blocks_before_apply(self) -> None: + harness = self.make() + harness.host["npm_version"] = None + harness.host["effective_uid"] = 1000 + code, body = harness.run("plan") + self.assertEqual(code, 2, body) + self.assertEqual(body["state"], "PREFLIGHT_BLOCKED") + self.assertEqual(body["planned_steps"], []) + self.assertIn("PF-NPM", body["failed_or_limiting_checks"]) + self.assertEqual(body["prerequisites"][0]["text"], NPM_PREREQUISITE) + self.assertNotIn("argv", body["prerequisites"][0]) + plan = json.loads(harness.tx_file("PLAN.json").read_text(encoding="utf-8")) + self.assertEqual(plan["planned_steps"], []) + self.assertEqual(plan["prerequisites"][0]["text"], NPM_PREREQUISITE) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/vantio-install/tests/test_observe_health.py b/packages/vantio-install/tests/test_observe_health.py new file mode 100644 index 00000000..749a9095 --- /dev/null +++ b/packages/vantio-install/tests/test_observe_health.py @@ -0,0 +1,134 @@ +"""Observe host check: a stopped container is not a detached loader.""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +PACKAGE = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PACKAGE)) + +from vantio_install import constants # noqa: E402 +from vantio_install.observe_health import observe_lifecycle, wait_for_observe_host # noqa: E402 +from vantio_install.pe_apparmor import OBSERVE_INSPECT_FORMAT, parse_observe_inspect # noqa: E402 + +CMD = '["--iface","ens5"]' + + +def line(status: str, pid: int, running: str, privileged: str, profile: str = "vantio-pe-observe") -> str: + return f"{status} {pid} {running} {privileged} {profile} {CMD}" + + +def sample(**overrides: object) -> dict: + base = { + "lifecycle": "detached", + "security_ok": True, + "pins": list(constants.BPF_PINS), + "pins_error": False, + "loader": True, + "clsact": True, + } + base.update(overrides) + return base + + +class ObserveHealthTests(unittest.TestCase): + def test_inspect_format_records_status_and_pid(self) -> None: + self.assertIn("{{.State.Status}}", OBSERVE_INSPECT_FORMAT) + self.assertIn("{{.State.Pid}}", OBSERVE_INSPECT_FORMAT) + self.assertLess(OBSERVE_INSPECT_FORMAT.index("Status"), OBSERVE_INSPECT_FORMAT.index("Pid")) + + def test_exit_zero_shape_splits_stopped_from_detached(self) -> None: + stopped = parse_observe_inspect(line("exited", 0, "false", "false")) + detached = parse_observe_inspect(line("running", 4242, "true", "false")) + created = parse_observe_inspect(line("created", 0, "false", "false")) + self.assertEqual(observe_lifecycle(stopped), "stopped") + self.assertEqual(observe_lifecycle(detached), "detached") + self.assertEqual(observe_lifecycle(created), "starting") + self.assertEqual(stopped["pid"], 0) + self.assertGreater(detached["pid"], 0) + + def test_stopped_container_is_not_verified(self) -> None: + calls = {"n": 0} + + def read() -> dict: + calls["n"] += 1 + return sample(lifecycle="stopped", security_ok=False) + + status = wait_for_observe_host( + read, + wait_s=20, + poll_s=0.25, + clock=lambda: 0.0, + sleeper=lambda _seconds: (_ for _ in ()).throw(AssertionError("slept")), + ) + self.assertEqual(status, "NOT_VERIFIED") + self.assertEqual(calls["n"], 1) + + def test_detached_container_is_verified_once_the_loader_is_healthy(self) -> None: + reads = [ + sample(pins=list(constants.BPF_PINS)[:2], loader=False), + sample(), + ] + + def read() -> dict: + return reads.pop(0) + + times = iter((0.0, 0.4)) + status = wait_for_observe_host( + read, + wait_s=20, + poll_s=0.25, + clock=lambda: next(times), + sleeper=lambda _seconds: None, + ) + self.assertEqual(status, "VERIFIED") + self.assertEqual(reads, []) + + def test_created_container_can_become_a_healthy_detached_loader(self) -> None: + reads = [sample(lifecycle="starting", security_ok=False, pins=[], loader=False), sample()] + + def read() -> dict: + return reads.pop(0) + + times = iter((0.0, 1.0)) + status = wait_for_observe_host( + read, + wait_s=20, + poll_s=0.25, + clock=lambda: next(times), + sleeper=lambda _seconds: None, + ) + self.assertEqual(status, "VERIFIED") + + def test_detached_container_that_never_loads_is_not_verified(self) -> None: + def read() -> dict: + return sample(pins=[], loader=False, clsact=False) + + times = iter((0.0, 20.0)) + status = wait_for_observe_host( + read, + wait_s=20, + poll_s=0.25, + clock=lambda: next(times), + sleeper=lambda _seconds: None, + ) + self.assertEqual(status, "NOT_VERIFIED") + + def test_unreadable_pins_stay_unknown(self) -> None: + def read() -> dict: + return sample(pins_error=True, pins=[]) + + status = wait_for_observe_host( + read, + wait_s=20, + poll_s=0.25, + clock=lambda: 0.0, + sleeper=lambda _seconds: (_ for _ in ()).throw(AssertionError("slept")), + ) + self.assertEqual(status, "UNKNOWN") + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/vantio-install/tests/test_oci_load.py b/packages/vantio-install/tests/test_oci_load.py new file mode 100644 index 00000000..64911f3e --- /dev/null +++ b/packages/vantio-install/tests/test_oci_load.py @@ -0,0 +1,186 @@ +"""OCI load correction for containerd's snapshotter. No Docker daemon required.""" + +from __future__ import annotations + +import gzip +import hashlib +import io +import json +import tarfile +import tempfile +import unittest +from pathlib import Path + +from vantio_install.oci_load import ( + LOAD_ARCHIVE_NAME, + describe_archive, + load_output_rejected, + materialize, + plan_load, +) + + +def _sha256(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _gzip(payload: bytes) -> bytes: + buffer = io.BytesIO() + with gzip.GzipFile(fileobj=buffer, mode="wb", mtime=0) as handle: + handle.write(payload) + return buffer.getvalue() + + +def _layer_tar() -> bytes: + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w") as archive: + payload = b"hello" + info = tarfile.TarInfo("hello.txt") + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + return buffer.getvalue() + + +def _pack(members: dict[str, bytes]) -> bytes: + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w") as archive: + for name, payload in members.items(): + info = tarfile.TarInfo(name=name) + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + return buffer.getvalue() + + +def _oci_tar(*, media_type: str, gzip_layer: bool) -> bytes: + layer = _layer_tar() + blob = _gzip(layer) if gzip_layer else layer + blob_digest = _sha256(blob) + config = b'{"architecture":"amd64","os":"linux"}' + config_digest = _sha256(config) + manifest = { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:" + config_digest, + "size": len(config), + }, + "layers": [ + { + "mediaType": media_type, + "digest": "sha256:" + blob_digest, + "size": len(blob), + } + ], + } + manifest_bytes = json.dumps(manifest, separators=(",", ":")).encode("utf-8") + manifest_digest = _sha256(manifest_bytes) + index = { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:" + manifest_digest, + "size": len(manifest_bytes), + "annotations": { + "org.opencontainers.image.ref.name": "example", + "io.containerd.image.name": "docker.io/library/example:example", + }, + } + ], + } + index_bytes = json.dumps(index, separators=(",", ":")).encode("utf-8") + return _pack( + { + "oci-layout": b'{"imageLayoutVersion":"1.0.0"}', + "index.json": index_bytes, + f"blobs/sha256/{config_digest}": config, + f"blobs/sha256/{manifest_digest}": manifest_bytes, + f"blobs/sha256/{blob_digest}": blob, + } + ) + + +def _media_type(path: Path) -> str: + with tarfile.open(path, "r") as archive: + index = json.loads(archive.extractfile("index.json").read()) + digest = index["manifests"][0]["digest"].split(":", 1)[1] + manifest = json.loads(archive.extractfile(f"blobs/sha256/{digest}").read()) + return manifest["layers"][0]["mediaType"] + + +class OciLoadTest(unittest.TestCase): + def test_gzip_bytes_labeled_uncompressed_are_corrected(self) -> None: + raw = _oci_tar(media_type="application/vnd.oci.image.layer.v1.tar", gzip_layer=True) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = root / "sealed.oci.tar" + stage = root / "stage" + stage.mkdir() + source.write_bytes(raw) + before = hashlib.sha256(raw).hexdigest() + plan = plan_load( + source, + stage, + fallback_digest="sha256:" + "a" * 64, + image_tag="vantio-phantom-engine:example", + ) + self.assertTrue(plan.rewrite) + self.assertEqual(plan.load_path.name, LOAD_ARCHIVE_NAME) + self.assertNotEqual(plan.image_digest, "sha256:" + "a" * 64) + materialize(plan) + self.assertEqual(hashlib.sha256(source.read_bytes()).hexdigest(), before) + self.assertEqual( + _media_type(plan.load_path), + "application/vnd.oci.image.layer.v1.tar+gzip", + ) + with tarfile.open(plan.load_path, "r") as archive: + index = json.loads(archive.extractfile("index.json").read()) + self.assertEqual(index["manifests"][0]["digest"], plan.image_digest) + self.assertEqual( + index["manifests"][0]["annotations"]["org.opencontainers.image.ref.name"], + "example", + ) + facts = describe_archive(source) + self.assertTrue(facts["rewrite"]) + self.assertFalse(facts["blocked"]) + self.assertFalse(facts["storage_driver_change"]) + + def test_matching_gzip_media_type_is_loaded_as_sealed(self) -> None: + raw = _oci_tar(media_type="application/vnd.oci.image.layer.v1.tar+gzip", gzip_layer=True) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = root / "sealed.oci.tar" + source.write_bytes(raw) + plan = plan_load(source, root, fallback_digest="sha256:" + "b" * 64) + self.assertFalse(plan.rewrite) + self.assertEqual(plan.image_digest, "sha256:" + "b" * 64) + self.assertEqual(plan.load_path, root / "sealed.oci.tar") + facts = describe_archive(source) + self.assertFalse(facts["rewrite"]) + self.assertEqual(facts["layout"], "oci") + + def test_unrecognized_file_stays_on_the_sealed_path(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = root / "notes.txt" + source.write_text("not an archive", encoding="utf-8") + plan = plan_load(source, root, fallback_digest="sha256:" + "c" * 64) + self.assertFalse(plan.rewrite) + self.assertEqual(plan.layout, "unrecognized") + facts = describe_archive(source) + self.assertFalse(facts["blocked"]) + + def test_unpack_error_text_is_rejected_even_with_loaded_image(self) -> None: + text = ( + "Loaded image: vantio-phantom-engine:pe-residuals-06696d5\n" + "Error unpacking image: failed to extract layer sha256:34467cc9: " + "archive/tar: invalid tar header\n" + ) + self.assertTrue(load_output_rejected(text, "")) + self.assertFalse(load_output_rejected("Loaded image: vantio-phantom-engine:example\n", "")) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/vantio-install/tests/test_pe_apparmor.py b/packages/vantio-install/tests/test_pe_apparmor.py index 902a6109..c7e7cfce 100644 --- a/packages/vantio-install/tests/test_pe_apparmor.py +++ b/packages/vantio-install/tests/test_pe_apparmor.py @@ -65,14 +65,14 @@ def test_profile_keeps_docker_default_denies_and_allows_bpf_pin(self) -> None: def test_inspect_requires_named_profile_and_refuses_privileged(self) -> None: cmd = '["--iface","ens5"]' - good = f"true false vantio-pe-observe {cmd}" + good = f"running 42 true false vantio-pe-observe {cmd}" self.assertTrue(inspect_is_observe_container(good)) - self.assertFalse(inspect_is_observe_container(f"true false docker-default {cmd}")) - self.assertFalse(inspect_is_observe_container(f"true false unconfined {cmd}")) - self.assertFalse(inspect_is_observe_container(f"true true vantio-pe-observe {cmd}")) - self.assertFalse(inspect_is_observe_container(f"false false vantio-pe-observe {cmd}")) - self.assertFalse(inspect_is_observe_container(f"true false vantio-pe-observe {cmd} --enforce")) - self.assertFalse(inspect_is_observe_container("true false vantio-pe-observe")) + self.assertFalse(inspect_is_observe_container(f"running 42 true false docker-default {cmd}")) + self.assertFalse(inspect_is_observe_container(f"running 42 true false unconfined {cmd}")) + self.assertFalse(inspect_is_observe_container(f"running 42 true true vantio-pe-observe {cmd}")) + self.assertFalse(inspect_is_observe_container(f"exited 0 false false vantio-pe-observe {cmd}")) + self.assertFalse(inspect_is_observe_container(f"running 42 true false vantio-pe-observe {cmd} --enforce")) + self.assertFalse(inspect_is_observe_container("running 42 true false vantio-pe-observe")) def test_profile_list_match_is_exact(self) -> None: directory = Path(tempfile.mkdtemp(prefix="vantio-aa-")) diff --git a/packages/vantio-install/tests/test_seal_recipe.py b/packages/vantio-install/tests/test_seal_recipe.py index efda7325..76eab669 100644 --- a/packages/vantio-install/tests/test_seal_recipe.py +++ b/packages/vantio-install/tests/test_seal_recipe.py @@ -99,16 +99,16 @@ def test_customer_docs_use_staging_basename_only(self) -> None: combined = "\n".join((docs / name).read_text(encoding="utf-8") for name in names) artifact = (docs / "ARTIFACT-PATH.md").read_text(encoding="utf-8") stage_b = (docs / "STAGE-B-ARTIFACT-PATH.md").read_text(encoding="utf-8") - basename = "vantio-phantom-engine-customer-staging-fab81efc0811-linux-amd64.oci.tar" + basename = "vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar" self.assertIn(basename, artifact) self.assertIn(basename, stage_b) self.assertIn( - "72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128", + "e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e", artifact, ) - self.assertIn("fab81efc08110506ff90847495197e7051a253b5", artifact) + self.assertIn("06696d5020700693b0154c59d0e072a24f648378", artifact) self.assertIn( - "sha256:4d932b93bf4c20983142d5f9bff1ea060d9407a19a5e8c9f59db29f7a4122553", + "sha256:8b40aec5c125043ec4278a14170677474c9ca31a7ae78e8496c40dffa69d0e19", artifact, ) self.assertIn("INTERNAL_CLEAN_HOST_PROOF", artifact) diff --git a/packages/vantio-install/tests/test_stage_a.py b/packages/vantio-install/tests/test_stage_a.py index 99e12ef5..95cd4671 100644 --- a/packages/vantio-install/tests/test_stage_a.py +++ b/packages/vantio-install/tests/test_stage_a.py @@ -133,9 +133,9 @@ def make(self, files: dict[str, bytes] | None = None) -> Harness: def test_frozen_identities_and_cli_package_untouched(self) -> None: pins = constants.FROZEN_PINS - self.assertEqual(pins["pe_source_commit"], "fab81efc08110506ff90847495197e7051a253b5") - self.assertEqual(pins["pe_archive_sha256"], "72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128") - self.assertEqual(pins["pe_manifest_digest"], "sha256:4d932b93bf4c20983142d5f9bff1ea060d9407a19a5e8c9f59db29f7a4122553") + self.assertEqual(pins["pe_source_commit"], "06696d5020700693b0154c59d0e072a24f648378") + self.assertEqual(pins["pe_archive_sha256"], "e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e") + self.assertEqual(pins["pe_manifest_digest"], "sha256:8b40aec5c125043ec4278a14170677474c9ca31a7ae78e8496c40dffa69d0e19") self.assertEqual(pins["optics_cli_version"], "0.3.24") self.assertEqual(pins["optics_cli_sha256"], "82fe13ad6fc916ac67a670bd95fbf18b24389ecb383d81246e1d52cb96712a1f") self.assertEqual(pins["agent_sdk_npm_version"], "0.2.4") diff --git a/packages/vantio-install/vantio_install/commands.py b/packages/vantio-install/vantio_install/commands.py index 61e7fa26..c2070ed5 100644 --- a/packages/vantio-install/vantio_install/commands.py +++ b/packages/vantio-install/vantio_install/commands.py @@ -83,6 +83,15 @@ def npm_install_argv(tarball: str, prefix: str) -> list[str]: return ["npm", "install", "--global", "--prefix", prefix, tarball] +def npm_version_argv() -> list[str]: + return ["npm", "--version"] + + +def apt_install_npm_argv() -> list[str]: + """Ubuntu npm package only. The nodejs package does not ship the npm binary.""" + return ["apt-get", "install", "-y", "--no-install-recommends", "npm"] + + def pip_wheel_argv(wheel: str, prefix: str) -> list[str]: return [ "python3", diff --git a/packages/vantio-install/vantio_install/constants.py b/packages/vantio-install/vantio_install/constants.py index 8da90653..df636bad 100644 --- a/packages/vantio-install/vantio_install/constants.py +++ b/packages/vantio-install/vantio_install/constants.py @@ -136,28 +136,28 @@ "agent_sdk_py_wheel_sha256": "dcf84cb3c4f144ece21032001657bfd9c91067faeffbefd0fb2ae19d6109dbeb", "agent_sdk_py_sdist": "vantio_agent_sdk-3.1.0.tar.gz", "agent_sdk_py_sdist_sha256": "9f991291d5e44a23e17a9b0d7db24f6e7048d4c76cf0a9c37e35ccbcfe999c4f", - "pe_source_commit": "fab81efc08110506ff90847495197e7051a253b5", - "pe_prior_candidate_commit": "631e435315cd780d83d3259e111893c1d0569bc3", - "pe_archive_name": "vantio-phantom-engine-w3-aws-internal-fab81efc0811-linux-amd64.oci.tar", - "pe_archive_sha256": "72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128", - "pe_manifest_digest": "sha256:4d932b93bf4c20983142d5f9bff1ea060d9407a19a5e8c9f59db29f7a4122553", - "pe_config_digest": "sha256:89f57cb67e2ef160c0304d5dc0519c67b3f90b02b0397f6b64e4d6708f07abf5", - "pe_loader_sha256": "13f80ebb8630eb25b4a2980e2cfca658c59261a4ac6853eca33ad928a42fe3f6", + "pe_source_commit": "06696d5020700693b0154c59d0e072a24f648378", + "pe_prior_candidate_commit": "fab81efc08110506ff90847495197e7051a253b5", + "pe_archive_name": "vantio-phantom-engine-pe-residuals-06696d5-linux-amd64.oci.tar", + "pe_archive_sha256": "e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e", + "pe_manifest_digest": "sha256:8b40aec5c125043ec4278a14170677474c9ca31a7ae78e8496c40dffa69d0e19", + "pe_config_digest": "sha256:1c7bbbd08a87639334b18e841e6b2be67e6de4c9864deaf768dfae723ea517b0", + "pe_loader_sha256": "f19b43f26a4b42671bb8d4f0aeb4b97bac7f52ae78442241f6e30d8ef385694f", "pe_layer_digests": [ "sha256:774043ccc8ccd0d0833a9ee0792142ab7ad93df971e59dd248fbf82db16d0150", - "sha256:2bcb0f32a7bc4b8ae39a1f2f75f736b6505b08db616f8244cf1f92a0f0c56afc", - "sha256:b5a549c61034f3b07933265800ff0f211259878e67a4e71d1e59d7398a5ce6dd", + "sha256:1a6bdb81d7e0937f4806047859066a399a52bfedfca104091f3d1f6a02fd176c", + "sha256:769a1b3da566912c6a7f3766d159125b07fdeeba8a5e156c06c98e31ead5d2db", ], - "pe_local_tag": "vantio-phantom-engine:w3-aws-internal-fab81efc0811", + "pe_local_tag": "vantio-phantom-engine:pe-residuals-06696d5", "pe_platform": "linux/amd64", } # Sealed identity the live gate compares to the pin table at call time. # Archive file bytes are compared to FROZEN_PINS["pe_archive_sha256"]. LIVE_CANONICAL_IDENTITY = { - "pe_source_commit": "fab81efc08110506ff90847495197e7051a253b5", - "pe_archive_sha256": "72719cf4c590805378188da38a0d43c540e6722328268bde3955f07d2c3a9128", - "pe_manifest_digest": "sha256:4d932b93bf4c20983142d5f9bff1ea060d9407a19a5e8c9f59db29f7a4122553", + "pe_source_commit": "06696d5020700693b0154c59d0e072a24f648378", + "pe_archive_sha256": "e0b19d557891b1ee8bbd20e702df11669d175e4083ef5bbe2f7077cf30093b5e", + "pe_manifest_digest": "sha256:8b40aec5c125043ec4278a14170677474c9ca31a7ae78e8496c40dffa69d0e19", } # Observe-only PE container profile. docker-default denies /sys/fs/bpf pin writes. @@ -184,12 +184,14 @@ "PF-ARTIFACT-SDK-NPM", "PF-ARTIFACT-SDK-PY", "PF-ARTIFACT-PE", + "PF-OCI-LOAD", "PF-GHCR-DEFAULT", "PF-DISCLOSURE", "PF-TRANSFER-ALLOWLIST", "PF-ENFORCEMENT-DEFAULT", "PF-TX-ATTACH", "PF-NODE", + "PF-NPM", "PF-ENTERPRISE-CLAIM", "PF-OPERATOR-SSH-ASSUMPTION", ) diff --git a/packages/vantio-install/vantio_install/engine.py b/packages/vantio-install/vantio_install/engine.py index 4c9aaa48..613f5932 100644 --- a/packages/vantio-install/vantio_install/engine.py +++ b/packages/vantio-install/vantio_install/engine.py @@ -203,21 +203,41 @@ def _blank_residual() -> dict: return {"result": "UNKNOWN", "items": [], "scope": None, "probe_errors": []} -def _planned_steps() -> list[dict]: +def _planned_steps(npm: dict) -> list[dict]: rows = [] for index, step_id in enumerate(constants.APPLY_STEPS, start=1): - rows.append( - { - "ord": index, - "id": step_id, - "mutation": step_id in constants.HOST_MUTATION_STEPS, - "mode": "observe-only", - "enforcement": "NOT_ENABLED", - } - ) + row = { + "ord": index, + "id": step_id, + "mutation": step_id in constants.HOST_MUTATION_STEPS, + "mode": "observe-only", + "enforcement": "NOT_ENABLED", + } + if step_id == "ensure_node": + row["npm_action"] = npm["action"] + if npm.get("argv"): + row["argv"] = npm["argv"] + if npm["action"] == "remediate": + row["mutation"] = True + row["prerequisite"] = npm["prerequisite"] + rows.append(row) return rows +def _plan_prerequisites(npm: dict) -> list[dict]: + if not npm.get("prerequisite"): + return [] + row = { + "id": "PF-NPM", + "package": "npm", + "text": npm["prerequisite"], + "action": npm["action"], + } + if npm.get("argv"): + row["argv"] = npm["argv"] + return [row] + + def _digests() -> dict: pin = constants.FROZEN_PINS return { @@ -312,11 +332,12 @@ def _load_host(fixture: Path | None) -> dict: return probe_live() -def _plan_document(tx_id: str, gate: str, missing: list[str], layout: dict[str, Path]) -> dict: +def _plan_document(tx_id: str, gate: str, missing: list[str], layout: dict[str, Path], npm: dict) -> dict: ready = gate in {"PREFLIGHT_READY", "PREFLIGHT_READY_WITH_LIMITATIONS"} return { "transaction_id": tx_id, - "planned_steps": _planned_steps() if ready else [], + "planned_steps": _planned_steps(npm) if ready else [], + "prerequisites": _plan_prerequisites(npm), "artifact_digests": _digests(), "missing_manifest_fields": missing, "layout": {key: str(value) for key, value in layout.items()}, @@ -361,6 +382,7 @@ def plan(ctx: dict) -> tuple[int, dict]: preflight_status=tx.get("preflight_status"), limitations=tx.get("limitations") or [], planned_steps=plan_doc["planned_steps"], + prerequisites=plan_doc.get("prerequisites") or [], artifact_digests=plan_doc["artifact_digests"], replayed=True, ) @@ -403,7 +425,7 @@ def plan(ctx: dict) -> tuple[int, dict]: "UNSUPPORTED": "UNSUPPORTED", }[report["overall"]] _move(tx, gate, stamp) - plan_doc = _plan_document(tx_id, gate, missing, layout) + plan_doc = _plan_document(tx_id, gate, missing, layout, report["npm"]) if gate in {"PREFLIGHT_READY", "PREFLIGHT_READY_WITH_LIMITATIONS"}: _move(tx, "PLANNED", stamp) tx["phase"] = "plan" @@ -429,6 +451,7 @@ def plan(ctx: dict) -> tuple[int, dict]: preflight_status=report["overall"], limitations=report["limitations"], planned_steps=plan_doc["planned_steps"], + prerequisites=plan_doc["prerequisites"], artifact_digests=plan_doc["artifact_digests"], failed_or_limiting_checks=[row["id"] for row in report["failed_or_limiting_checks"]], ) diff --git a/packages/vantio-install/vantio_install/host.py b/packages/vantio-install/vantio_install/host.py index b1c0f424..cedc35b0 100644 --- a/packages/vantio-install/vantio_install/host.py +++ b/packages/vantio-install/vantio_install/host.py @@ -46,6 +46,8 @@ def ready_host(**overrides: object) -> dict: "mem_total_kib": 16 * 1024 * 1024, "ifaces": {"ens5": "up"}, "node_version": "v20.11.0", + "npm_version": "9.2.0", + "effective_uid": 0, } host.update(empty_runtime_state()) host.update(overrides) @@ -98,6 +100,8 @@ def probe_live() -> dict: host["mem_total_kib"] = _mem_total_kib() host["ifaces"] = _ifaces() host["node_version"] = _node_version() + host["npm_version"] = _npm_version() + host["effective_uid"] = os.geteuid() host["probe_note"] = "LIVE_READ_ONLY" return host @@ -185,6 +189,13 @@ def _node_version() -> str | None: return _read_text([node, "--version"]) +def _npm_version() -> str | None: + npm = shutil.which("npm") + if not npm: + return None + return _read_text([npm, "--version"]) + + def _read_text(argv: list[str]) -> str | None: try: completed = subprocess.run(argv, check=False, capture_output=True, text=True, timeout=5) diff --git a/packages/vantio-install/vantio_install/live_executor.py b/packages/vantio-install/vantio_install/live_executor.py index ea67258b..ae020d0a 100644 --- a/packages/vantio-install/vantio_install/live_executor.py +++ b/packages/vantio-install/vantio_install/live_executor.py @@ -11,6 +11,7 @@ import re import shutil import subprocess +import time from collections.abc import Callable from dataclasses import dataclass from pathlib import Path @@ -29,6 +30,7 @@ from vantio_install.commands import ( apparmor_parser_load_argv, apparmor_parser_remove_argv, + apt_install_npm_argv, docker_load_argv, docker_rmi_argv, docker_start_argv, @@ -36,6 +38,7 @@ docker_tag_argv, mkdir_argv, npm_install_argv, + npm_version_argv, observe_apparmor_opt, observe_binds, observe_container_argv, @@ -45,8 +48,8 @@ tc_clsact_del_argv, ) from vantio_install.pe_apparmor import ( + OBSERVE_INSPECT_FORMAT, apparmor_profile_loaded, - inspect_is_observe_container, pe_apparmor_profile_path, profile_text, ) @@ -59,6 +62,7 @@ remove_agent_sdks, ) from vantio_install.errors import InstallError +from vantio_install.oci_load import OciArchiveError, OciLoadPlan, load_output_rejected, materialize, plan_load from vantio_install.host import probe_tracefs_mounted from vantio_install.manifest import artifact_paths, load_manifest from vantio_install.optics_cli import ( @@ -69,7 +73,13 @@ ) from vantio_install.paths import assert_safe_root from vantio_install.state_machine import RESIDUAL_STATES -from vantio_install.preflight import run_preflight +from vantio_install.observe_health import ( + OBSERVE_READY_POLL_S, + OBSERVE_READY_WAIT_S, + observe_sample_from_inspect, + wait_for_observe_host, +) +from vantio_install.preflight import npm_requirement, run_preflight from vantio_install.stage_remove import remove_stage_nofollow from vantio_install.util import read_json, sha256_file, write_json @@ -77,9 +87,10 @@ _IFACE = re.compile(r"^[A-Za-z][A-Za-z0-9_.:-]{0,14}$") _SHELLS = {"sh", "bash", "dash", "zsh", "busybox", "sudo", "su"} _META = (";", "|", "&", "`", "$(", "\n", "\r", ">", "<") -_ALLOWED_EXE = {"mkdir", "npm", "python3", "docker", "tc", "apparmor_parser"} +_ALLOWED_EXE = {"mkdir", "npm", "python3", "docker", "tc", "apparmor_parser", "apt-get"} STEP_OPERATIONS = { + "ensure_node": ("ensure_npm",), "install_optics_cli": ("mkdir_prefix", "install_optics_cli"), "install_agent_sdks": ("install_agent_sdk_npm", "install_agent_sdk_py"), "stage_pe_archive": ("mkdir_stage", "stage_pe_archive"), @@ -191,6 +202,7 @@ class LiveGrant: sdk_wheel: Path container_name: str observe_config: Path + npm_action: str def _fail(message: str, *, failure_class: str = "FAILED_SAFE", state: str = "FAILED_SAFE", exit_code: int = 4) -> None: @@ -215,6 +227,11 @@ def reject_argv(argv: object) -> list[str]: exe = Path(argv[0]).name if exe in _SHELLS or exe not in _ALLOWED_EXE: _fail(f"Executable {exe} is not on the live allowlist.", failure_class="FAILED_SAFE") + if exe == "apt-get" and list(argv) != apt_install_npm_argv(): + _fail( + "apt-get is allowlisted only to install the Ubuntu npm package.", + failure_class="FAILED_SAFE", + ) for item in argv: if any(token in item for token in _META): _fail("An argument contains a shell metacharacter and is refused.", failure_class="FAILED_SAFE") @@ -318,12 +335,37 @@ def _require_apparmor_parser_argv(op_type: str, argv: list[str]) -> None: _fail("The AppArmor parser argv is not the observe profile path.", failure_class="FAILED_SAFE") +def oci_plan_for(grant: LiveGrant) -> OciLoadPlan: + """The docker load file for this grant. The sealed archive stays put.""" + staged = grant.stage / grant.archive.name + source = staged if staged.is_file() else grant.archive + pin = constants.FROZEN_PINS + try: + return plan_load( + source, + grant.stage, + fallback_digest=pin["pe_manifest_digest"], + image_tag=pin["pe_local_tag"], + ) + except OciArchiveError as exc: + _fail(str(exc), failure_class="FAILED_SAFE") + raise AssertionError("oci plan failure always raises") + + def catalog_argv(op_type: str, grant: LiveGrant) -> list[str] | None: """Argv for process operations. None means a confined filesystem operation.""" pin = constants.FROZEN_PINS prefix = str(grant.prefix) iface = grant.iface + if grant.npm_action == "present": + ensure_npm = npm_version_argv() + elif grant.npm_action == "remediate": + ensure_npm = apt_install_npm_argv() + else: + ensure_npm = None + load_plan = oci_plan_for(grant) if op_type in {"docker_load", "docker_tag"} else None mapping: dict[str, list[str] | None] = { + "ensure_npm": ensure_npm, "mkdir_prefix": mkdir_argv(prefix), "mkdir_stage": mkdir_argv(str(grant.stage)), "mkdir_evidence": mkdir_argv(str(grant.evidence)), @@ -331,8 +373,11 @@ def catalog_argv(op_type: str, grant: LiveGrant) -> list[str] | None: "install_agent_sdk_npm": npm_install_argv(str(grant.sdk_npm), prefix), "install_agent_sdk_py": pip_wheel_argv(str(grant.sdk_wheel), prefix), "stage_pe_archive": None, - "docker_load": docker_load_argv(str(grant.stage / grant.archive.name)), - "docker_tag": docker_tag_argv(pin["pe_manifest_digest"], pin["pe_local_tag"]), + "docker_load": docker_load_argv(str(load_plan.load_path if load_plan else grant.stage / grant.archive.name)), + "docker_tag": docker_tag_argv( + load_plan.image_digest if load_plan else pin["pe_manifest_digest"], + pin["pe_local_tag"], + ), "write_observe_config": None, "o7_init": None, "tc_clsact": tc_clsact_argv(iface), @@ -356,6 +401,11 @@ def catalog_argv(op_type: str, grant: LiveGrant) -> list[str] | None: if op_type not in mapping: _fail(f"Operation {op_type} is not on the live allowlist.", failure_class="FAILED_SAFE") argv = mapping[op_type] + if op_type == "ensure_npm" and argv is None: + _fail( + "Install the Ubuntu npm package. The nodejs package does not include the npm binary.", + failure_class="FAILED_SAFE", + ) if argv is not None: reject_argv(argv) _reject_forbidden_live_argv(argv) @@ -422,6 +472,10 @@ def _runtime_tx(grant: LiveGrant) -> dict: def run_allowlisted(argv: list[str], timeout: int, runner) -> ExecResult: checked = reject_argv(argv) if runner is None: + env = None + if Path(checked[0]).name == "apt-get": + env = os.environ.copy() + env["DEBIAN_FRONTEND"] = "noninteractive" try: completed = subprocess.run( checked, @@ -429,10 +483,17 @@ def run_allowlisted(argv: list[str], timeout: int, runner) -> ExecResult: check=False, capture_output=True, timeout=timeout, + env=env, ) except subprocess.TimeoutExpired as exc: return ExecResult(124, True, _captured_text(exc.stdout), _captured_text(exc.stderr)) except OSError as exc: + if Path(checked[0]).name == "apt-get": + _fail( + "The Ubuntu npm package could not be installed because apt-get is not on PATH. " + "Install the Ubuntu npm package. The nodejs package does not include the npm binary.", + failure_class="FAILED_SAFE", + ) _fail(f"The live command could not start: {exc.__class__.__name__}.", failure_class="FAILED_SAFE") return ExecResult( completed.returncode, @@ -590,9 +651,16 @@ def dispatch( if op_type in {"docker_load", "docker_tag", "stage_pe_archive", "start_pe_observe"}: _rehash_archive(grant) _rehash_inputs(op_type, grant) + if op_type == "ensure_npm" and grant.npm_action == "remediate" and runner is None and os.geteuid() != 0: + _fail( + "Install the Ubuntu npm package. The nodejs package does not include the npm binary.", + failure_class="FAILED_SAFE", + ) _append_op(grant, {"op": op_type, "phase": "PENDING", "transaction_id": grant.transaction_id}) result: ExecResult | None = None if expected is not None: + if op_type == "docker_load": + _materialize_load(grant) result = run_allowlisted(expected, timeout, runner) if runner is None: result = _recover_absent_target(op_type, grant, result) @@ -604,6 +672,14 @@ def dispatch( state="INTERRUPTED", exit_code=constants.EXIT_INTERRUPTED, ) + if op_type == "docker_load" and load_output_rejected(result.stdout, result.stderr): + failure = "ROLLBACK_REQUIRED" if result.returncode == 0 else "FAILED_SAFE" + _append_op(grant, {"op": op_type, "phase": failure, "transaction_id": grant.transaction_id}) + _fail( + "docker load printed an unpack error, so the layer is not on the host. " + "The installer does not change Docker's storage driver.", + failure_class=failure, + ) else: _filesystem(op_type, grant) result = ExecResult(0, False) @@ -640,6 +716,16 @@ def dispatch( return delta +def _materialize_load(grant: LiveGrant) -> None: + plan = oci_plan_for(grant) + if not plan.rewrite: + return + try: + materialize(plan) + except OciArchiveError as exc: + _fail(str(exc), failure_class="FAILED_SAFE") + + def execute_step(grant: LiveGrant, step_id: str, kind: str, runner, observer) -> tuple[list[dict], list[list[str]]]: table = STEP_OPERATIONS if kind == "apply" else ROLLBACK_OPERATIONS if step_id not in table: @@ -648,7 +734,8 @@ def execute_step(grant: LiveGrant, step_id: str, kind: str, runner, observer) -> recorded: list[list[str]] = [] for op_type in table[step_id]: argv = catalog_argv(op_type, grant) - delta = dispatch(grant, op_type, argv, runner=runner, observer=observer) + timeout = 180 if op_type == "ensure_npm" else 60 + delta = dispatch(grant, op_type, argv, runner=runner, observer=observer, timeout=timeout) deltas.append(delta) if argv: recorded.append(argv) @@ -879,6 +966,7 @@ def authorize_live( sdk_wheel=paths["agent_sdk_py_wheel"], container_name=f"vantio-pe-{str(tx['transaction_id'])[-12:]}", observe_config=tx_dir / "observe-config.json", + npm_action=npm_requirement(host)["action"], ) @@ -890,11 +978,27 @@ class ProductionObserver: fixture path so they do not touch securityfs. """ - def __init__(self, apparmor_profiles: Path | None = None) -> None: + def __init__( + self, + apparmor_profiles: Path | None = None, + *, + observe_sampler: Callable[[LiveGrant], dict] | None = None, + observe_wait_s: float = OBSERVE_READY_WAIT_S, + observe_poll_s: float = OBSERVE_READY_POLL_S, + clock: Callable[[], float] | None = None, + sleeper: Callable[[float], None] | None = None, + ) -> None: self.apparmor_profiles = apparmor_profiles + self.observe_sampler = observe_sampler + self.observe_wait_s = observe_wait_s + self.observe_poll_s = observe_poll_s + self.clock = clock + self.sleeper = sleeper def verify(self, op_type: str, grant: LiveGrant) -> str: try: + if op_type == "ensure_npm": + return "VERIFIED" if shutil.which("npm") else "NOT_VERIFIED" if op_type == "install_optics_cli": expected = constants.FROZEN_PINS["optics_cli_version"] return "VERIFIED" if optics_cli_verified(grant.prefix, expected) else "NOT_VERIFIED" @@ -960,19 +1064,23 @@ def verify(self, op_type: str, grant: LiveGrant) -> str: if op_type in {"remove_stage", "remove_observe_config", "remove_o7_record"}: return "VERIFIED" if op_type == "docker_load": - return _docker_image_present(constants.FROZEN_PINS["pe_manifest_digest"]) + return _docker_image_present(oci_plan_for(grant).image_digest) if op_type == "docker_tag": return _docker_image_present(grant.tag) if op_type in {"start_pe_observe", "restart_pe_observe"}: - running = _docker_running_observe(grant.container_name) == "VERIFIED" - pins, pin_errors = _host_pins() - loader = _loader_running() - clsact = _tc_has_clsact(grant.iface) == "VERIFIED" - if pin_errors: - return "UNKNOWN" - if running and pins == list(constants.BPF_PINS) and loader and clsact: - return "VERIFIED" - return "NOT_VERIFIED" + + def sample() -> dict: + if self.observe_sampler is not None: + return self.observe_sampler(grant) + return _observe_sample(grant.container_name, grant.iface) + + return wait_for_observe_host( + sample, + wait_s=self.observe_wait_s, + poll_s=self.observe_poll_s, + clock=self.clock or time.monotonic, + sleeper=self.sleeper or time.sleep, + ) if op_type == "unpin_bpf_maps": pins, pin_errors = _host_pins() if pin_errors: @@ -994,6 +1102,11 @@ def verify(self, op_type: str, grant: LiveGrant) -> str: def observed_delta(self, op_type: str, grant: LiveGrant) -> dict: pin = constants.FROZEN_PINS + if op_type == "ensure_npm": + version = _observed_npm_version() + if version: + return {"npm_version": version} + return {} if op_type == "install_optics_cli": version = observed_optics_cli_version(grant.prefix) if version: @@ -1012,7 +1125,11 @@ def observed_delta(self, op_type: str, grant: LiveGrant) -> dict: if op_type == "docker_tag": return { "images": [ - {"tag": pin["pe_local_tag"], "digest": pin["pe_manifest_digest"], "role": "phantom_engine"} + { + "tag": pin["pe_local_tag"], + "digest": oci_plan_for(grant).image_digest, + "role": "phantom_engine", + } ] } if op_type == "start_pe_observe": @@ -1074,18 +1191,32 @@ def _docker_image_present(tag: str) -> str: return "VERIFIED" if text else "NOT_VERIFIED" -_INSPECT_FORMAT = "{{.State.Running}} {{.HostConfig.Privileged}} {{.AppArmorProfile}} {{json .Config.Cmd}}" - - def _docker_inspect_line(name: str) -> str | None: - return _read_only(["docker", "inspect", "--format", _INSPECT_FORMAT, name]) + return _read_only(["docker", "inspect", "--format", OBSERVE_INSPECT_FORMAT, name]) -def _docker_running_observe(name: str) -> str: +def _observe_sample(name: str, iface: str) -> dict: text = _docker_inspect_line(name) - if text and inspect_is_observe_container(text): - return "VERIFIED" - return "NOT_VERIFIED" + lifecycle, security_ok = observe_sample_from_inspect(text) + pins, pin_errors = _host_pins() + return { + "lifecycle": lifecycle, + "security_ok": security_ok, + "pins": pins, + "pins_error": bool(pin_errors), + "loader": _loader_running(), + "clsact": _tc_has_clsact(iface) == "VERIFIED", + } + + +def _observed_npm_version() -> str | None: + npm = shutil.which("npm") + if not npm: + return None + text = _read_only([npm, "--version"]) + if not text: + return None + return text.strip() or None def _docker_stopped(name: str) -> str: diff --git a/packages/vantio-install/vantio_install/mutator.py b/packages/vantio-install/vantio_install/mutator.py index 5f28cc28..3bfdf1b9 100644 --- a/packages/vantio-install/vantio_install/mutator.py +++ b/packages/vantio-install/vantio_install/mutator.py @@ -22,6 +22,7 @@ ) from vantio_install.pe_apparmor import pe_apparmor_profile_path from vantio_install.errors import InstallError +from vantio_install.oci_load import OciArchiveError, materialize, plan_load from vantio_install.stage_remove import remove_stage_nofollow from vantio_install.util import sha256_file, write_json @@ -170,7 +171,18 @@ def _remove_stage(self, ctx: dict) -> None: def _docker_load(self, ctx: dict) -> None: pin = constants.FROZEN_PINS archive = self.stage / pin["pe_archive_name"] - load_argv = docker_load_argv(str(archive)) + try: + plan = plan_load( + archive, + self.stage, + fallback_digest=pin["pe_manifest_digest"], + image_tag=pin["pe_local_tag"], + ) + if plan.rewrite: + materialize(plan) + except OciArchiveError as exc: + raise InstallError(str(exc), exit_code=4, state="FAILED_SAFE") from exc + load_argv = docker_load_argv(str(plan.load_path)) self.recorded_argv.append(load_argv) self.snapshot.setdefault("recorded_argv", []).append(load_argv) images = self.snapshot.setdefault("images", []) @@ -178,7 +190,7 @@ def _docker_load(self, ctx: dict) -> None: images.append( { "tag": pin["pe_local_tag"], - "digest": pin["pe_manifest_digest"], + "digest": plan.image_digest, "role": "phantom_engine", } ) @@ -307,6 +319,15 @@ def __init__(self, grant, runner, observer, snapshot: dict) -> None: self.recorded_argv: list[list[str]] = [] def apply_step(self, step_id: str, ctx: dict) -> None: + if step_id == "ensure_node": + deltas, argv = execute_step(self.grant, step_id, "apply", self.runner, self.observer) + for delta in deltas: + self._merge(delta) + self.recorded_argv.extend(argv) + self.snapshot.setdefault("recorded_argv", []).extend(argv) + if self.grant.npm_action == "remediate": + self.mutation_count += 1 + return self._run(step_id, "apply") def rollback_step(self, step_id: str, ctx: dict) -> None: diff --git a/packages/vantio-install/vantio_install/observe_health.py b/packages/vantio-install/vantio_install/observe_health.py new file mode 100644 index 00000000..b7b091c2 --- /dev/null +++ b/packages/vantio-install/vantio_install/observe_health.py @@ -0,0 +1,86 @@ +"""Host check for the observe container after docker run returns. + +``docker run -d`` exits 0 when the daemon accepts the container. The same exit +is 0 when that process has already stopped. This check reads the container. +A stopped process is not a healthy loader. A detached container that is still +starting is read again until the loader, the known bpffs pins, and clsact are +present, or until the wait ends. +""" + +from __future__ import annotations + +from collections.abc import Callable + +from vantio_install import constants +from vantio_install.pe_apparmor import inspect_is_observe_container, parse_observe_inspect + +OBSERVE_READY_WAIT_S = 20.0 +OBSERVE_READY_POLL_S = 0.25 + +_STOPPED = frozenset({"exited", "dead", "removing"}) +_STARTING = frozenset({"created", "restarting", "paused"}) + + +def observe_lifecycle(parsed: dict | None) -> str: + """Return detached, stopped, starting, or unknown from one inspect parse.""" + if not parsed: + return "unknown" + status = parsed["status"] + if status in _STOPPED: + return "stopped" + if status == "running" and parsed["running"] and parsed["pid"] > 0: + return "detached" + if status in _STARTING: + return "starting" + if parsed["pid"] == 0 and not parsed["running"]: + return "stopped" + return "unknown" + + +def observe_sample_from_inspect(text: str | None) -> tuple[str, bool]: + """Lifecycle and whether the inspect line is a running observe container.""" + parsed = parse_observe_inspect(text or "") + security_ok = bool(text) and inspect_is_observe_container(text or "") + return observe_lifecycle(parsed), security_ok + + +def wait_for_observe_host( + sample_fn: Callable[[], dict], + *, + wait_s: float, + poll_s: float, + clock: Callable[[], float], + sleeper: Callable[[float], None], +) -> str: + """VERIFIED, NOT_VERIFIED, or UNKNOWN. Exit 0 from docker run is not an input.""" + if wait_s < 0: + wait_s = 0.0 + if poll_s <= 0: + poll_s = OBSERVE_READY_POLL_S + started = clock() + limit = int(wait_s / poll_s) + 2 + last_unknown = True + for _ in range(limit): + sample = sample_fn() + if sample.get("pins_error"): + return "UNKNOWN" + life = sample.get("lifecycle") + if life == "stopped": + return "NOT_VERIFIED" + if _healthy(sample): + return "VERIFIED" + last_unknown = life == "unknown" + if clock() - started >= wait_s: + return "UNKNOWN" if last_unknown else "NOT_VERIFIED" + sleeper(poll_s) + return "UNKNOWN" if last_unknown else "NOT_VERIFIED" + + +def _healthy(sample: dict) -> bool: + return ( + sample.get("lifecycle") == "detached" + and sample.get("security_ok") is True + and sample.get("pins") == list(constants.BPF_PINS) + and sample.get("loader") is True + and sample.get("clsact") is True + ) diff --git a/packages/vantio-install/vantio_install/oci_load.py b/packages/vantio-install/vantio_install/oci_load.py new file mode 100644 index 00000000..16b67026 --- /dev/null +++ b/packages/vantio-install/vantio_install/oci_load.py @@ -0,0 +1,362 @@ +"""Load path for a sealed Phantom Engine OCI tar on stock Docker. + +Ubuntu 24.04's default Docker uses the containerd image store. That unpacker +trusts the layer media type. A gzip blob labeled +``application/vnd.oci.image.layer.v1.tar`` fails with +``archive/tar: invalid tar header``. ``docker load`` can still exit 0. + +This module writes a temporary archive whose media type matches the bytes. +It does not modify the sealed file, and it does not change Docker's storage +driver. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import tarfile +from dataclasses import dataclass +from pathlib import Path + +LOAD_ARCHIVE_NAME = "pe-containerd-load.oci.tar" + +_GZIP_MAGIC = b"\x1f\x8b" +_OCI_TAR = "application/vnd.oci.image.layer.v1.tar" +_OCI_GZIP = "application/vnd.oci.image.layer.v1.tar+gzip" +_DOCKER_TAR = "application/vnd.docker.image.rootfs.diff.tar" +_DOCKER_GZIP = "application/vnd.docker.image.rootfs.diff.tar.gzip" +_GZIP_MEDIA = { + _OCI_TAR: _OCI_GZIP, + _DOCKER_TAR: _DOCKER_GZIP, +} +_TAR_MEDIA = {value: key for key, value in _GZIP_MEDIA.items()} +_UNPACK_MARKERS = ( + "invalid tar header", + "Error unpacking", + "apply layer error", + "failed to extract layer", +) + + +class OciArchiveError(Exception): + """The file is an OCI layout this loader cannot correct.""" + + +@dataclass(frozen=True) +class OciLoadPlan: + source: Path + load_path: Path + image_digest: str + rewrite: bool + layout: str + corrections: tuple[str, ...] + image_tag: str | None = None + + +def load_output_rejected(stdout: str, stderr: str) -> bool: + """True when docker load reported an unpack failure, including exit 0.""" + text = f"{stdout}\n{stderr}" + return any(marker in text for marker in _UNPACK_MARKERS) + + +def plan_load( + source: Path, + stage: Path, + *, + fallback_digest: str, + image_tag: str | None = None, +) -> OciLoadPlan: + """Choose the file ``docker load`` should read. + + ``fallback_digest`` is the sealed manifest digest. A corrected load uses + the digest of the temporary manifest instead, because that is the image + id Docker records. The sealed file is not the load path in that case. + """ + passthrough = OciLoadPlan( + source=source, + load_path=stage / source.name, + image_digest=fallback_digest, + rewrite=False, + layout="absent", + corrections=(), + ) + if not source.is_file(): + return passthrough + try: + prepared = _prepare(source) + except OciArchiveError: + raise + except (OSError, tarfile.TarError, json.JSONDecodeError, ValueError): + return OciLoadPlan( + source=source, + load_path=stage / source.name, + image_digest=fallback_digest, + rewrite=False, + layout="unrecognized", + corrections=(), + ) + if prepared is None: + return OciLoadPlan( + source=source, + load_path=stage / source.name, + image_digest=fallback_digest, + rewrite=False, + layout="unrecognized", + corrections=(), + ) + corrections, new_digest = prepared + if not corrections: + return OciLoadPlan( + source=source, + load_path=stage / source.name, + image_digest=fallback_digest, + rewrite=False, + layout="oci", + corrections=(), + ) + return OciLoadPlan( + source=source, + load_path=stage / LOAD_ARCHIVE_NAME, + image_digest="sha256:" + new_digest, + rewrite=True, + layout="oci", + corrections=tuple(corrections), + image_tag=image_tag, + ) + + +def materialize(plan: OciLoadPlan) -> None: + """Write the temporary load archive. The sealed file is only read.""" + if not plan.rewrite: + return + if not plan.source.is_file(): + raise OciArchiveError("The sealed archive is not on disk.") + sealed_before = _sha256(plan.source) + prepared = _prepare(plan.source) + if prepared is None or not prepared[0]: + raise OciArchiveError("The sealed archive no longer needs a load correction.") + _corrections, new_digest = prepared + if plan.image_digest != "sha256:" + new_digest: + raise OciArchiveError("The corrected manifest digest changed while writing the load archive.") + dest = plan.load_path + dest.parent.mkdir(parents=True, exist_ok=True) + partial = dest.with_name(dest.name + ".partial") + _write_corrected(plan.source, partial, new_digest, plan.image_tag) + partial.replace(dest) + if _sha256(plan.source) != sealed_before: + raise OciArchiveError("The sealed archive changed while the load archive was written.") + + +def describe_archive(path: Path) -> dict: + """Preflight facts. A correctable mismatch is not a block.""" + if not path.is_file(): + return { + "layout": "absent", + "rewrite": False, + "blocked": False, + "corrections": [], + "storage_driver_change": False, + } + try: + prepared = _prepare(path) + except OciArchiveError as exc: + return { + "layout": "oci", + "rewrite": False, + "blocked": True, + "reason": str(exc), + "corrections": [], + "storage_driver_change": False, + } + except (OSError, tarfile.TarError, json.JSONDecodeError, ValueError): + return { + "layout": "unrecognized", + "rewrite": False, + "blocked": False, + "corrections": [], + "storage_driver_change": False, + } + if prepared is None: + return { + "layout": "unrecognized", + "rewrite": False, + "blocked": False, + "corrections": [], + "storage_driver_change": False, + } + corrections, _digest = prepared + return { + "layout": "oci", + "rewrite": bool(corrections), + "blocked": False, + "corrections": corrections, + "storage_driver_change": False, + } + + +def _prepare(source: Path) -> tuple[list[str], str] | None: + """Return corrections and the manifest digest docker will record. + + ``None`` means the file is not an OCI layout. An empty correction list + means the media types already match the bytes. The digest in that case + is the sealed manifest digest. + """ + with tarfile.open(source, "r") as archive: + names = set(archive.getnames()) + if "oci-layout" not in names or "index.json" not in names: + return None + index = _read_json(archive, "index.json") + manifests = index.get("manifests") + if not isinstance(manifests, list) or not manifests: + raise OciArchiveError("The OCI index has no manifest.") + corrections: list[str] = [] + # One image. A multi-manifest index is refused rather than half-corrected. + if len(manifests) != 1 or not isinstance(manifests[0], dict): + raise OciArchiveError("The OCI index does not contain exactly one manifest.") + entry = manifests[0] + digest = _digest_hex(entry.get("digest")) + manifest_name = f"blobs/sha256/{digest}" + if manifest_name not in names: + raise OciArchiveError("The OCI manifest blob is missing from the archive.") + manifest = _read_json(archive, manifest_name) + layers = manifest.get("layers") + if not isinstance(layers, list) or not layers: + raise OciArchiveError("The OCI manifest has no layers.") + for layer in layers: + if not isinstance(layer, dict): + raise OciArchiveError("An OCI layer entry is not an object.") + media = str(layer.get("mediaType") or "") + layer_digest = _digest_hex(layer.get("digest")) + blob_name = f"blobs/sha256/{layer_digest}" + if blob_name not in names: + raise OciArchiveError("An OCI layer blob is missing from the archive.") + magic = _read_prefix(archive, blob_name, 2) + replacement = _replacement_media(media, magic) + if replacement is None: + continue + if replacement == media: + continue + layer["mediaType"] = replacement + corrections.append(f"{layer_digest[:12]} {media} -> {replacement}") + manifest_bytes = _canonical(manifest) + new_digest = hashlib.sha256(manifest_bytes).hexdigest() + entry["digest"] = "sha256:" + new_digest + entry["size"] = len(manifest_bytes) + # The returned digest is the sealed one when nothing changed, so callers + # can keep the pin. The rewritten bytes are recomputed in _write_corrected. + if not corrections: + return [], digest + return corrections, new_digest + + +def _write_corrected(source: Path, dest: Path, new_digest: str, image_tag: str | None) -> None: + with tarfile.open(source, "r") as archive: + index = _read_json(archive, "index.json") + entry = index["manifests"][0] + old_digest = _digest_hex(entry.get("digest")) + manifest = _read_json(archive, f"blobs/sha256/{old_digest}") + for layer in manifest["layers"]: + media = str(layer.get("mediaType") or "") + layer_digest = _digest_hex(layer.get("digest")) + magic = _read_prefix(archive, f"blobs/sha256/{layer_digest}", 2) + replacement = _replacement_media(media, magic) + if replacement is not None: + layer["mediaType"] = replacement + manifest_bytes = _canonical(manifest) + if hashlib.sha256(manifest_bytes).hexdigest() != new_digest: + raise OciArchiveError("The corrected manifest does not match the planned digest.") + entry["digest"] = "sha256:" + new_digest + entry["size"] = len(manifest_bytes) + if image_tag: + entry["annotations"] = _annotations(image_tag) + index_bytes = _canonical(index) + with tarfile.open(dest, "w", format=tarfile.USTAR_FORMAT) as out: + for member in archive.getmembers(): + name = member.name + if name.startswith("/") or ".." in Path(name).parts: + raise OciArchiveError("An archive member is outside the tar.") + if name == "index.json": + _add_bytes(out, "index.json", index_bytes) + continue + if name == f"blobs/sha256/{old_digest}": + _add_bytes(out, f"blobs/sha256/{new_digest}", manifest_bytes) + continue + if not member.isfile(): + continue + handle = archive.extractfile(member) + if handle is None: + continue + info = tarfile.TarInfo(name=name) + info.size = member.size + info.mode = 0o644 + info.mtime = 0 + out.addfile(info, handle) + + +def _replacement_media(media: str, magic: bytes) -> str | None: + gzip = magic == _GZIP_MAGIC + if media in _GZIP_MEDIA and gzip: + return _GZIP_MEDIA[media] + if media in _TAR_MEDIA and not gzip: + return _TAR_MEDIA[media] + return None + + +def _annotations(tag: str) -> dict[str, str]: + repo, sep, name = tag.rpartition(":") + ref = name if sep else tag + if repo and "/" in repo: + image_name = tag + else: + image_name = f"docker.io/library/{tag}" + return { + "io.containerd.image.name": image_name, + "org.opencontainers.image.ref.name": ref, + } + + +def _canonical(document: dict) -> bytes: + return json.dumps(document, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + + +def _read_json(archive: tarfile.TarFile, name: str) -> dict: + handle = archive.extractfile(name) + if handle is None: + raise OciArchiveError(f"Missing {name}.") + document = json.loads(handle.read().decode("utf-8")) + if not isinstance(document, dict): + raise OciArchiveError(f"{name} is not a JSON object.") + return document + + +def _read_prefix(archive: tarfile.TarFile, name: str, count: int) -> bytes: + handle = archive.extractfile(name) + if handle is None: + raise OciArchiveError(f"Missing {name}.") + return handle.read(count) + + +def _digest_hex(value: object) -> str: + if not isinstance(value, str) or not value.startswith("sha256:"): + raise OciArchiveError("An OCI digest is missing.") + hex_part = value.split(":", 1)[1] + if len(hex_part) != 64 or any(char not in "0123456789abcdef" for char in hex_part): + raise OciArchiveError("An OCI digest is not a sha256 hex string.") + return hex_part + + +def _add_bytes(archive: tarfile.TarFile, name: str, payload: bytes) -> None: + info = tarfile.TarInfo(name=name) + info.size = len(payload) + info.mode = 0o644 + info.mtime = 0 + archive.addfile(info, io.BytesIO(payload)) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/packages/vantio-install/vantio_install/pe_apparmor.py b/packages/vantio-install/vantio_install/pe_apparmor.py index 8958dddc..253aae57 100644 --- a/packages/vantio-install/vantio_install/pe_apparmor.py +++ b/packages/vantio-install/vantio_install/pe_apparmor.py @@ -108,16 +108,52 @@ def apparmor_profile_loaded(name: str, profiles_path: Path | None = None) -> boo return False +# Status, host pid, running, privileged, AppArmor profile, container cmd. +# docker run -d exits 0 for a detached container and for a process that has +# already exited. Status and pid are what separate those two. +OBSERVE_INSPECT_FORMAT = ( + "{{.State.Status}} {{.State.Pid}} {{.State.Running}} " + "{{.HostConfig.Privileged}} {{.AppArmorProfile}} {{json .Config.Cmd}}" +) + +_OBSERVE_STATUSES = frozenset( + {"created", "restarting", "running", "removing", "paused", "exited", "dead"} +) + + +def parse_observe_inspect(text: str) -> dict | None: + """Parse one observe inspect line. None means the line is not that shape.""" + parts = text.strip().split(" ", 5) + if len(parts) != 6: + return None + status, pid, running, privileged, profile, cmd = parts + status = status.casefold() + if status not in _OBSERVE_STATUSES or not pid.isdigit(): + return None + if running not in {"true", "false"} or privileged not in {"true", "false"}: + return None + return { + "status": status, + "pid": int(pid), + "running": running == "true", + "privileged": privileged == "true", + "profile": profile, + "cmd": cmd, + } + + def inspect_is_observe_container(text: str) -> bool: - """Docker inspect line: running, privileged, AppArmor profile, cmd JSON.""" - parts = text.split(" ", 3) - if len(parts) != 4: + """True when the container is running, unprivileged, and on the observe profile.""" + parsed = parse_observe_inspect(text) + if parsed is None: + return False + if parsed["status"] != "running" or not parsed["running"] or parsed["pid"] <= 0: return False - running, privileged, profile, cmd = parts - if running != "true" or privileged != "false": + if parsed["privileged"]: return False - if profile != constants.PE_OBSERVE_APPARMOR_PROFILE: + if parsed["profile"] != constants.PE_OBSERVE_APPARMOR_PROFILE: return False + cmd = parsed["cmd"] if "--enforce" in cmd or "--privileged" in cmd or "VANTIO_PHANTOM_DENY" in cmd: return False return True diff --git a/packages/vantio-install/vantio_install/preflight.py b/packages/vantio-install/vantio_install/preflight.py index 0b923996..af4e4ec3 100644 --- a/packages/vantio-install/vantio_install/preflight.py +++ b/packages/vantio-install/vantio_install/preflight.py @@ -6,6 +6,8 @@ from pathlib import Path from vantio_install import constants +from vantio_install.commands import apt_install_npm_argv, npm_version_argv +from vantio_install.oci_load import describe_archive from vantio_install.manifest import ( artifact_paths, hash_named, @@ -17,6 +19,10 @@ _FORBIDDEN_CIDRS = {"0.0.0.0/0", "::/0", "0.0.0.0", "*"} +NPM_PREREQUISITE = ( + "Install the Ubuntu npm package. The nodejs package does not include the npm binary." +) + def _check(check_id: str, title: str, result: str, observed: dict, expected: dict, remediation: str) -> dict: return { @@ -98,6 +104,74 @@ def _forbidden_cidr(value: str) -> bool: return False +def _oci_load_check(archive: Path) -> dict: + """Record whether apply must correct the OCI layer media type before docker load.""" + facts = describe_archive(archive) + if facts.get("blocked"): + return _check( + "PF-OCI-LOAD", + "sealed OCI tar can be loaded on the installed Docker", + "BLOCKED", + facts, + {"rewrite": False, "storage_driver_change": False}, + "Restore the sealed archive. The installer does not change Docker's storage driver.", + ) + if facts.get("rewrite"): + remediation = ( + "Apply writes a temporary load archive with the layer media type set to match the bytes. " + "The sealed file stays in place. Docker's storage driver stays as installed." + ) + else: + remediation = "No layer media-type correction is required. Docker's storage driver stays as installed." + return _check( + "PF-OCI-LOAD", + "sealed OCI tar can be loaded on the installed Docker", + "PASS", + facts, + {"storage_driver_change": False}, + remediation, + ) + + +def npm_requirement(host: dict) -> dict: + """Decide whether npm is present, installed by this plan, or a plan blocker. + + Ubuntu 24.04's nodejs package does not ship npm. When Node.js 18 or newer + is already on the host and this process is root, the plan installs the + Ubuntu npm package. Any other missing npm stops the plan and names that + package. + """ + version = host.get("npm_version") + present = isinstance(version, str) and bool(version) and version not in {"ABSENT", "UNKNOWN"} + node = _parse_node(host.get("node_version")) + node_ok = node is not None and node >= (18, 0, 0) + ubuntu = str(host.get("os_id", "")) == "ubuntu" and str(host.get("os_version_id", "")).startswith("24.04") + root = host.get("effective_uid") == 0 + if present: + return { + "result": "PASS", + "action": "present", + "prerequisite": None, + "argv": npm_version_argv(), + "package": "npm", + } + if ubuntu and node_ok and root: + return { + "result": "PASS", + "action": "remediate", + "prerequisite": NPM_PREREQUISITE, + "argv": apt_install_npm_argv(), + "package": "npm", + } + return { + "result": "BLOCKED", + "action": "blocked", + "prerequisite": NPM_PREREQUISITE, + "argv": None, + "package": "npm", + } + + def aggregate(checks: list[dict]) -> str: results = [row["result"] for row in checks] if any(item == "UNSUPPORTED" for item in results): @@ -399,6 +473,7 @@ def run_preflight( "Use the sealed archive for the frozen tip. Tip drift needs a new seal.", ) ) + checks.append(_oci_load_check(paths["pe_archive"])) source = str(config.get("artifact_source", "sealed_archive")) ghcr = source == "ghcr" or "ghcr.io" in source or source.endswith(":0.1.0") @@ -504,6 +579,27 @@ def run_preflight( ) ) + npm = npm_requirement(host) + npm_observed = { + "npm_version_or_absent": host.get("npm_version") or "ABSENT", + "action": npm["action"], + "package": "npm", + } + if npm["prerequisite"]: + npm_observed["prerequisite"] = npm["prerequisite"] + if npm["argv"]: + npm_observed["argv"] = npm["argv"] + checks.append( + _check( + "PF-NPM", + "npm is on PATH, or root on Ubuntu 24.04 will install the Ubuntu npm package", + npm["result"], + npm_observed, + {"npm": "present"}, + npm["prerequisite"] or "npm is already on PATH.", + ) + ) + enterprise = str(config.get("enterprise_inclusion", "OPTIONAL_SOURCE_ONLY_NOT_PACKAGED_AUTHORITY")) claims_packaged = "PACKAGED_AUTHORITY" in enterprise and "NOT_PACKAGED" not in enterprise checks.append( @@ -575,6 +671,7 @@ def run_preflight( return { "overall": overall, "checks": checks, + "npm": npm, "limitations": limitations_from(checks), "failed_or_limiting_checks": [row for row in checks if row["result"] != "PASS"], "proof_ceiling": constants.PROOF_CEILING,